durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3mod cancellation_context;
4mod cancellation_replay_clock;
5mod cancellation_scope;
6mod cancellation_scope_history;
7mod cancellation_scope_replay;
8mod cooperative_cancellation;
9mod local_activity;
10mod runtime_payloads;
11mod runtime_uploads;
12mod sticky_worker;
13mod sticky_workflow_cache;
14mod worker_session;
15
16pub use cancellation_context::{
17    CancellationContext, CancellationLineage, ScopedCancellationContext, ScopedCancellationLineage,
18};
19pub use cancellation_scope::CancellationScopeOpenReceipt;
20#[doc(hidden)]
21pub use cancellation_scope_history::{
22    CancellationScopeDeliveryBudget, CancellationScopeDeliveryReceipt,
23};
24#[doc(hidden)]
25pub use cancellation_scope_replay::CancellationScopeRequested;
26
27pub use cooperative_cancellation::{
28    CancellationCallKind, CancellationDelivery, CancellationDeliveryReceipt,
29    CancellationDeliveryReply, CancellationHistory, CancellationRequest, CancellationShield,
30    CooperativeCancellationOptions, CooperativeCancellationRequested, CooperativeWorkflowTask,
31    CooperativeWorkflowTaskPoll, WorkflowCancellationRequest, WorkflowTaskHeartbeat,
32};
33
34use std::{
35    any::{type_name, Any, TypeId},
36    collections::{BTreeMap, BTreeSet, HashMap},
37    future::Future,
38    io::{self, Read},
39    pin::Pin,
40    sync::{
41        atomic::{AtomicBool, Ordering},
42        Arc, Mutex, OnceLock,
43    },
44    task::{Context as TaskContext, Poll},
45    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
46};
47
48use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
49use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
50use chrono::DateTime;
51use futures_util::{
52    future::OptionFuture, stream::FuturesUnordered, task::noop_waker_ref, StreamExt,
53};
54pub use local_activity::LocalActivityOptions;
55use serde::{
56    de::DeserializeOwned,
57    ser::{SerializeMap, SerializeSeq},
58    Deserialize, Deserializer, Serialize, Serializer,
59};
60pub use serde_json::{json, Value};
61use sha2::{Digest, Sha256};
62pub use sticky_workflow_cache::{StickyCacheMetrics, StickyCacheOptions};
63use thiserror::Error;
64pub use uuid::Uuid;
65pub use worker_session::{WorkerSession, WorkerSessionOptions};
66
67pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
68const WORKFLOW_HISTORY_PAGE_SIZE: usize = 500;
69/// First additive worker protocol that defines portable worker-affinity features.
70pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
71pub const CONTROL_PLANE_VERSION: &str = "2";
72pub const DEFAULT_CODEC: &str = "avro";
73pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
74/// Worker-registration capability for authored condition-wait occurrence identity.
75pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
76    "condition_wait_occurrence_identity";
77/// Worker-registration capability for portable memo upserts.
78pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
79/// Worker-registration capability for server-routed read-only queries.
80pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
81/// Worker-registration capability for canonical typed search attributes.
82pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
83/// Worker-registration capability for synchronous workflow updates.
84pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
85/// Worker-registration capability for durable named input streams.
86pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
87/// Worker-registration capability for persisted first-completion selection.
88pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
89pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
90pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
91pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
92pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
93pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
94/// First additive worker protocol that defines query-task transport.
95pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
96/// First additive worker protocol that defines typed search-attribute upserts.
97pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
98/// First additive worker protocol that defines portable memo upserts.
99pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
100/// First additive worker protocol that preserves declared search-attribute types.
101pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
102/// First additive worker protocol that defines external durable condition waits.
103pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
104/// First additive worker protocol that preserves authored condition-wait occurrences.
105pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
106/// First additive worker protocol that defines durable selection groups.
107pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
108
109pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
110    let Some((major, minor)) = version.split_once('.') else {
111        return false;
112    };
113    major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
114}
115
116fn validate_user_signal_name(signal_name: &str) -> Result<()> {
117    if signal_name == MESSAGE_STREAM_SIGNAL {
118        return Err(Error::Codec(format!(
119            "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
120        )));
121    }
122    Ok(())
123}
124
125const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
126const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
127    "Workflow task waiting for scheduled history.";
128const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
129const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
130const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
131const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
132const MAX_MEMO_ENTRIES: usize = 100;
133const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
134const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
135
136const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
137    "lease_expired",
138    "query_task_not_found",
139    "query_task_not_leased",
140    "query_task_timed_out",
141];
142
143/// Manifest for the default service-worker profile.
144///
145/// Inline local execution is enabled separately by [`Worker::local_activities`].
146/// Local execution, sessions and caching require explicit Worker configuration.
147pub fn portable_worker_affinity_capability_manifest() -> Value {
148    json!({
149        "local_activities": {
150            "supported": false,
151            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
152            "reason": "rust_worker_does_not_execute_record_local_activity",
153        },
154        "worker_sessions": {
155            "supported": false,
156            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
157            "reason": "rust_worker_has_no_typed_session_lifecycle",
158        },
159        "sticky_execution": {
160            "supported": false,
161            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
162            "reason": "rust_worker_uses_complete_durable_history_replay",
163        },
164    })
165}
166
167/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
168pub const AVRO_VALUE_SCHEMA_JSON: &str =
169    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
170pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
171pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
172const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
173
174static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
175static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
176    OnceLock::new();
177
178#[derive(Clone, Copy)]
179enum RequestProtocol {
180    ControlPlane,
181    Worker(&'static str),
182}
183
184pub type Result<T> = std::result::Result<T, Error>;
185
186#[derive(Debug, Error)]
187pub enum Error {
188    #[error("transport error: {0}")]
189    Transport(#[from] reqwest::Error),
190    #[error(
191        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
192    )]
193    InvalidBaseUrl,
194    #[error("json error: {0}")]
195    Json(#[from] serde_json::Error),
196    #[error("http {status}: {body}")]
197    Http {
198        status: reqwest::StatusCode,
199        body: String,
200    },
201    #[error("codec error: {0}")]
202    Codec(String),
203    #[error(transparent)]
204    QueryFailed(QueryFailure),
205    #[error(transparent)]
206    Protocol(ProtocolFailure),
207    #[error(transparent)]
208    NonDeterministicReplay(ReplayFailure),
209    #[error(transparent)]
210    ChildWorkflowFailed(ChildWorkflowFailure),
211    #[error(transparent)]
212    ActivityFailed(ActivityFailure),
213    #[error(transparent)]
214    ParallelFailed(ParallelFailure),
215    #[error(transparent)]
216    SagaCompensationFailed(SagaCompensationFailure),
217    #[error(transparent)]
218    InvalidParallelGroup(ParallelGroupError),
219    #[error(transparent)]
220    DurableOperationCancelled(DurableOperationCancelled),
221    #[error(transparent)]
222    WorkflowCancellationRequested(WorkflowCancellationRequested),
223    #[error(transparent)]
224    CooperativeCancellationRequested(CooperativeCancellationRequested),
225    #[error(transparent)]
226    CancellationScopeRequested(CancellationScopeRequested),
227    #[error(transparent)]
228    WorkflowCommandRejected(WorkflowCommandRejection),
229    #[error(transparent)]
230    WorkflowFailed(WorkflowTerminalOutcome),
231    #[error(transparent)]
232    WorkflowCancelled(WorkflowTerminalOutcome),
233    #[error(transparent)]
234    WorkflowTerminated(WorkflowTerminalOutcome),
235    #[error(transparent)]
236    WorkflowTimedOut(WorkflowTerminalOutcome),
237    #[error(transparent)]
238    ActivityTaskRejected(ActivityTaskRejection),
239    #[error("workflow handler {0:?} is not registered")]
240    WorkflowNotRegistered(String),
241    #[error("activity handler {0:?} is not registered")]
242    ActivityNotRegistered(String),
243    #[error(
244        "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
245    )]
246    HandlerType {
247        handler_kind: HandlerKind,
248        handler_name: String,
249        value_kind: HandlerValueKind,
250        rust_type: &'static str,
251        message: String,
252    },
253    #[error("workflow future yielded without emitting a durable command")]
254    WorkflowYieldedWithoutCommand,
255    #[error(
256        "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id or task_id"
257    )]
258    MissingWorkflowCommandIdentity,
259    #[error("workflow state lock is poisoned")]
260    WorkflowStatePoisoned,
261    #[error("timer duration is too large for the worker protocol")]
262    TimerDurationOverflow,
263    #[error(transparent)]
264    InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
265    #[error(transparent)]
266    InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
267    #[error("operation timed out")]
268    Timeout,
269    #[error(
270        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
271    )]
272    MissingRoleCredentials {
273        role: &'static str,
274        opposite_role: &'static str,
275    },
276    #[error("worker loop error: {0}")]
277    WorkerLoop(String),
278    #[error(
279        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
280    )]
281    UnsupportedUpdateValidators { workflow_type: String },
282    #[error("{primary}; worker deregistration also failed: {deregistration}")]
283    WorkerShutdown {
284        primary: Box<Error>,
285        deregistration: Box<Error>,
286    },
287    #[error("invalid child workflow options: {0}")]
288    InvalidChildWorkflowOptions(String),
289    #[error("invalid workflow memo update: {0}")]
290    InvalidMemoUpdate(String),
291    #[error(
292        "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
293    )]
294    WorkflowMemoUpdatesUnavailable,
295    #[error("cooperative cancellation is unavailable: {0}")]
296    CooperativeCancellationUnavailable(String),
297    #[error("cancellation_scope_execution_not_supported: Rust worker cannot replay scoped cancellation history")]
298    CancellationScopeExecutionUnavailable,
299    #[error("invalid cooperative cancellation: {0}")]
300    InvalidCooperativeCancellation(String),
301    #[error("activity execution no longer owns its claim: {0}")]
302    ActivityExecutionAbandoned(String),
303    #[error(transparent)]
304    InvalidActivityOptions(ActivityOptionsError),
305    #[error(transparent)]
306    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
307    #[doc(hidden)]
308    #[error("workflow requested continue as new")]
309    ContinueAsNew(ContinueAsNewRequest),
310}
311
312/// Handler namespaces checked independently within a Worker.
313#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
314pub enum RegistrationKind {
315    Workflow,
316    Activity,
317    Query,
318    Update,
319}
320
321impl std::fmt::Display for RegistrationKind {
322    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
323        formatter.write_str(match self {
324            Self::Workflow => "workflow",
325            Self::Activity => "activity",
326            Self::Query => "query",
327            Self::Update => "update",
328        })
329    }
330}
331
332/// The authoring adapter and source location of a handler registration.
333#[derive(Clone, Debug, PartialEq, Eq)]
334pub struct HandlerRegistration {
335    pub method: &'static str,
336    pub handler_type: &'static str,
337    pub file: &'static str,
338    pub line: u32,
339    pub column: u32,
340}
341
342impl std::fmt::Display for HandlerRegistration {
343    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
344        write!(
345            formatter,
346            "{} at {}:{}:{} ({})",
347            self.method, self.file, self.line, self.column, self.handler_type
348        )
349    }
350}
351
352/// Ambiguous local configuration, rejected before contacting Server.
353#[derive(Clone, Debug, Error, PartialEq, Eq)]
354#[error("duplicate_registration: {handler_kind} name {handler_name:?} in workflow scope {workflow_type:?}; first {first_definition}; conflicting {second_definition}")]
355pub struct DuplicateRegistrationError {
356    pub handler_kind: RegistrationKind,
357    pub handler_name: String,
358    /// Query/update registrations are scoped to their owning workflow type.
359    pub workflow_type: Option<String>,
360    pub first_definition: HandlerRegistration,
361    pub second_definition: HandlerRegistration,
362}
363
364impl From<DuplicateRegistrationError> for Error {
365    fn from(error: DuplicateRegistrationError) -> Self {
366        Self::WorkerLoop(error.to_string())
367    }
368}
369
370/// Validation failure for a durable condition-wait definition.
371#[derive(Clone, Debug, Error, PartialEq, Eq)]
372pub enum ConditionWaitOptionsError {
373    #[error("condition_key must be non-empty")]
374    EmptyKey,
375    #[error("condition_definition_fingerprint must be non-empty")]
376    EmptyPredicateIdentity,
377    #[error("condition timeout is too large for the worker protocol")]
378    TimeoutOverflow,
379}
380
381/// Stable identity and optional durable timeout for a condition wait.
382///
383/// `predicate_identity` is recorded as the worker protocol's
384/// `condition_definition_fingerprint` and must change whenever predicate
385/// behavior changes. Prefer the [`wait_condition!`] macro when the predicate
386/// is written inline; it derives this identity from the predicate tokens.
387#[derive(Clone, Debug, PartialEq, Eq)]
388pub struct ConditionWaitOptions {
389    condition_key: String,
390    predicate_identity: String,
391    timeout: Option<Duration>,
392}
393
394impl ConditionWaitOptions {
395    pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
396        Self {
397            condition_key: condition_key.into(),
398            predicate_identity: predicate_identity.into(),
399            timeout: None,
400        }
401    }
402
403    pub fn timeout(mut self, timeout: Duration) -> Self {
404        self.timeout = Some(timeout);
405        self
406    }
407
408    fn validate(
409        &self,
410    ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
411        let condition_key = self.condition_key.trim();
412        if condition_key.is_empty() {
413            return Err(ConditionWaitOptionsError::EmptyKey);
414        }
415        let predicate_identity = self.predicate_identity.trim();
416        if predicate_identity.is_empty() {
417            return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
418        }
419        let timeout_seconds = self
420            .timeout
421            .map(|timeout| {
422                timeout
423                    .as_secs()
424                    .checked_add(u64::from(timeout.subsec_nanos() > 0))
425                    .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
426            })
427            .transpose()?;
428
429        Ok(ValidatedConditionWaitOptions {
430            condition_key: condition_key.to_string(),
431            predicate_identity: predicate_identity.to_string(),
432            timeout_seconds,
433        })
434    }
435}
436
437#[derive(Clone, Debug, PartialEq, Eq)]
438struct ValidatedConditionWaitOptions {
439    condition_key: String,
440    predicate_identity: String,
441    timeout_seconds: Option<u64>,
442}
443
444const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
445
446/// Unambiguous terminal result of a durable condition wait.
447#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
448#[serde(rename_all = "snake_case")]
449pub enum ConditionWaitResult {
450    Satisfied,
451    TimedOut,
452}
453
454impl ConditionWaitResult {
455    pub fn is_satisfied(self) -> bool {
456        self == Self::Satisfied
457    }
458
459    pub fn is_timed_out(self) -> bool {
460        self == Self::TimedOut
461    }
462}
463
464/// Build the stable condition definition identity used by [`wait_condition!`].
465#[doc(hidden)]
466pub fn __condition_definition_fingerprint(source: &str) -> String {
467    let mut digest = Sha256::new();
468    digest.update(b"durable-workflow-rust.wait-condition.v1\0");
469    digest.update(source.as_bytes());
470    format!("sha256:{:x}", digest.finalize())
471}
472
473/// Create a durable condition wait whose predicate definition is fingerprinted
474/// from its inline Rust tokens.
475///
476/// The returned [`ConditionWaitCall`] must be awaited. The timeout form is
477/// `wait_condition!(ctx, "approval", timeout: duration, || predicate)`.
478#[macro_export]
479macro_rules! wait_condition {
480    ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
481        $ctx.wait_condition(
482            $crate::ConditionWaitOptions::new(
483                $key,
484                $crate::__condition_definition_fingerprint(concat!(
485                    module_path!(),
486                    "\0",
487                    stringify!($predicate)
488                )),
489            )
490            .timeout($timeout),
491            $predicate,
492        )
493    }};
494    ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
495        $ctx.wait_condition(
496            $crate::ConditionWaitOptions::new(
497                $key,
498                $crate::__condition_definition_fingerprint(concat!(
499                    module_path!(),
500                    "\0",
501                    stringify!($predicate)
502                )),
503            ),
504            $predicate,
505        )
506    }};
507}
508
509const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
510const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
511const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
512const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
513const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
514
515/// Validation failure for a typed workflow search-attribute update.
516#[derive(Clone, Debug, Error, PartialEq, Eq)]
517pub enum SearchAttributeUpdateError {
518    #[error("search-attribute update requires at least one attribute")]
519    Empty,
520    #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
521    InvalidKey(String),
522    #[error("search-attribute update exceeds the limit of 100 attributes")]
523    TooManyAttributes,
524    #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
525    ValueTooLong {
526        key: String,
527        kind: &'static str,
528        limit: usize,
529    },
530    #[error(
531        "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
532    )]
533    EmptyString(String),
534    #[error("search attribute {0:?} has a non-finite float value")]
535    NonFiniteFloat(String),
536    #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
537    InvalidDateTime(String),
538    #[error("search-attribute update exceeds the 65536-byte protocol limit")]
539    PayloadTooLarge,
540}
541
542/// One public typed search-attribute value.
543#[derive(Clone, Debug, PartialEq)]
544pub enum SearchAttributeValue {
545    String(String),
546    Keyword(String),
547    KeywordList(Vec<String>),
548    Int(i64),
549    Float(f64),
550    Bool(bool),
551    DateTime(String),
552    Delete,
553}
554
555impl SearchAttributeValue {
556    fn type_name(&self) -> Option<&'static str> {
557        match self {
558            Self::String(_) => Some("string"),
559            Self::Keyword(_) => Some("keyword"),
560            Self::KeywordList(_) => Some("keyword_list"),
561            Self::Int(_) => Some("int"),
562            Self::Float(_) => Some("float"),
563            Self::Bool(_) => Some("bool"),
564            Self::DateTime(_) => Some("datetime"),
565            Self::Delete => None,
566        }
567    }
568
569    fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
570        let normalize_string = |value: String, kind: &'static str, limit: usize| {
571            let value = value.trim().to_string();
572            if value.is_empty() {
573                return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
574            }
575            if value.len() > limit {
576                return Err(SearchAttributeUpdateError::ValueTooLong {
577                    key: key.to_string(),
578                    kind,
579                    limit,
580                });
581            }
582            Ok(value)
583        };
584
585        match self {
586            Self::String(value) => Ok(Self::String(normalize_string(
587                value,
588                "string",
589                MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
590            )?)),
591            Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
592                value,
593                "keyword",
594                MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
595            )?)),
596            Self::KeywordList(values) => {
597                let values = values
598                    .into_iter()
599                    .map(|value| {
600                        let value = value.trim().to_string();
601                        if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
602                            return Err(SearchAttributeUpdateError::ValueTooLong {
603                                key: key.to_string(),
604                                kind: "keyword-list entry",
605                                limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
606                            });
607                        }
608                        Ok(value)
609                    })
610                    .collect::<std::result::Result<Vec<_>, _>>()?;
611                Ok(Self::KeywordList(values))
612            }
613            Self::Float(value) if !value.is_finite() => {
614                Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
615            }
616            Self::DateTime(value) => {
617                let value =
618                    normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
619                if DateTime::parse_from_rfc3339(&value).is_err() {
620                    return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
621                }
622                Ok(Self::DateTime(value))
623            }
624            value => Ok(value),
625        }
626    }
627
628    fn into_json(self) -> Value {
629        match self {
630            Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
631                Value::String(value)
632            }
633            Self::KeywordList(values) => {
634                Value::Array(values.into_iter().map(Value::String).collect())
635            }
636            Self::Int(value) => json!(value),
637            Self::Float(value) => json!(value),
638            Self::Bool(value) => json!(value),
639            Self::Delete => Value::Null,
640        }
641    }
642}
643
644/// Validated typed workflow-side search-attribute mutation.
645#[derive(Clone, Debug, Default, PartialEq)]
646pub struct SearchAttributeUpdate {
647    attributes: BTreeMap<String, SearchAttributeValue>,
648}
649
650impl SearchAttributeUpdate {
651    pub fn new() -> Self {
652        Self::default()
653    }
654
655    pub fn set(
656        mut self,
657        key: impl Into<String>,
658        value: SearchAttributeValue,
659    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
660        let key = key.into();
661        validate_search_attribute_key(&key)?;
662        if !self.attributes.contains_key(&key)
663            && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
664        {
665            return Err(SearchAttributeUpdateError::TooManyAttributes);
666        }
667        self.attributes.insert(key.clone(), value.normalized(&key)?);
668        self.validate_size()?;
669        Ok(self)
670    }
671
672    pub fn string(
673        self,
674        key: impl Into<String>,
675        value: impl Into<String>,
676    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
677        self.set(key, SearchAttributeValue::String(value.into()))
678    }
679
680    pub fn keyword(
681        self,
682        key: impl Into<String>,
683        value: impl Into<String>,
684    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
685        self.set(key, SearchAttributeValue::Keyword(value.into()))
686    }
687
688    pub fn keyword_list<I, V>(
689        self,
690        key: impl Into<String>,
691        values: I,
692    ) -> std::result::Result<Self, SearchAttributeUpdateError>
693    where
694        I: IntoIterator<Item = V>,
695        V: Into<String>,
696    {
697        self.set(
698            key,
699            SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
700        )
701    }
702
703    pub fn int(
704        self,
705        key: impl Into<String>,
706        value: i64,
707    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
708        self.set(key, SearchAttributeValue::Int(value))
709    }
710
711    pub fn float(
712        self,
713        key: impl Into<String>,
714        value: f64,
715    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
716        self.set(key, SearchAttributeValue::Float(value))
717    }
718
719    pub fn bool(
720        self,
721        key: impl Into<String>,
722        value: bool,
723    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
724        self.set(key, SearchAttributeValue::Bool(value))
725    }
726
727    pub fn datetime(
728        self,
729        key: impl Into<String>,
730        value: impl Into<String>,
731    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
732        self.set(key, SearchAttributeValue::DateTime(value.into()))
733    }
734
735    pub fn delete(
736        self,
737        key: impl Into<String>,
738    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
739        self.set(key, SearchAttributeValue::Delete)
740    }
741
742    fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
743        let (attributes, _) = self.clone().into_wire_parts();
744        if serde_json::to_vec(&attributes)
745            .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
746            .unwrap_or(true)
747        {
748            return Err(SearchAttributeUpdateError::PayloadTooLarge);
749        }
750        Ok(())
751    }
752
753    fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
754        let mut attributes = serde_json::Map::new();
755        let mut attribute_types = BTreeMap::new();
756        for (key, value) in self.attributes {
757            if let Some(type_name) = value.type_name() {
758                attribute_types.insert(key.clone(), type_name.to_string());
759            }
760            attributes.insert(key, value.into_json());
761        }
762        (Value::Object(attributes), attribute_types)
763    }
764
765    fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
766        if self.attributes.is_empty() {
767            return Err(SearchAttributeUpdateError::Empty);
768        }
769        self.validate_size()
770    }
771}
772
773fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
774    let valid = !key.is_empty()
775        && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
776        && key
777            .bytes()
778            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
779    if valid {
780        Ok(())
781    } else {
782        Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
783    }
784}
785
786/// The registered handler family reported by [`Error::HandlerType`].
787#[derive(Clone, Copy, Debug, PartialEq, Eq)]
788pub enum HandlerKind {
789    Workflow,
790    Activity,
791}
792
793impl std::fmt::Display for HandlerKind {
794    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
795        formatter.write_str(match self {
796            Self::Workflow => "workflow",
797            Self::Activity => "activity",
798        })
799    }
800}
801
802/// Whether a typed handler failed to adapt its input or result.
803#[derive(Clone, Copy, Debug, PartialEq, Eq)]
804pub enum HandlerValueKind {
805    Input,
806    Result,
807}
808
809impl std::fmt::Display for HandlerValueKind {
810    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
811        formatter.write_str(match self {
812            Self::Input => "input",
813            Self::Result => "result",
814        })
815    }
816}
817
818/// The lifecycle command sent to a workflow execution.
819#[derive(Clone, Copy, Debug, PartialEq, Eq)]
820pub enum WorkflowCommandKind {
821    Cancel,
822    Terminate,
823}
824
825impl WorkflowCommandKind {
826    fn as_str(self) -> &'static str {
827        match self {
828            Self::Cancel => "cancel",
829            Self::Terminate => "terminate",
830        }
831    }
832}
833
834/// Optional structured fields for a cancellation or termination request.
835#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
836pub struct WorkflowCommandOptions {
837    #[serde(skip_serializing_if = "Option::is_none")]
838    pub reason: Option<String>,
839    #[serde(skip_serializing_if = "Option::is_none")]
840    pub request_id: Option<String>,
841}
842
843/// Server-enforced timeout policy for a workflow start.
844///
845/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
846/// only bounds how long the caller waits. A server deadline produces a terminal
847/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
848/// `run_timeout`.
849#[derive(Clone, Debug, PartialEq, Eq)]
850pub struct WorkflowStartOptions {
851    pub execution_timeout_seconds: u64,
852    pub run_timeout_seconds: u64,
853}
854
855impl Default for WorkflowStartOptions {
856    fn default() -> Self {
857        Self {
858            execution_timeout_seconds: 3600,
859            run_timeout_seconds: 600,
860        }
861    }
862}
863
864impl WorkflowStartOptions {
865    pub fn new() -> Self {
866        Self::default()
867    }
868
869    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
870        self.execution_timeout_seconds = seconds;
871        self
872    }
873
874    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
875        self.run_timeout_seconds = seconds;
876        self
877    }
878
879    fn validate(&self) -> Result<()> {
880        if self.execution_timeout_seconds == 0 {
881            return Err(Error::Codec(
882                "execution_timeout_seconds must be at least 1".to_string(),
883            ));
884        }
885        if self.run_timeout_seconds == 0 {
886            return Err(Error::Codec(
887                "run_timeout_seconds must be at least 1".to_string(),
888            ));
889        }
890        if self.run_timeout_seconds > self.execution_timeout_seconds {
891            return Err(Error::Codec(
892                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
893            ));
894        }
895
896        Ok(())
897    }
898}
899
900/// Optional routing overrides for a continue-as-new transition.
901///
902/// Omitted values retain the current workflow type and task queue. Server-owned
903/// instance metadata is not accepted here and is carried by the server.
904#[derive(Clone, Debug, Default, PartialEq, Eq)]
905pub struct ContinueAsNewOptions {
906    pub workflow_type: Option<String>,
907    pub task_queue: Option<String>,
908}
909
910impl ContinueAsNewOptions {
911    pub fn new() -> Self {
912        Self::default()
913    }
914
915    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
916        self.workflow_type = Some(workflow_type.into());
917        self
918    }
919
920    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
921        self.task_queue = Some(task_queue.into());
922        self
923    }
924
925    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
926        for (field, value) in [
927            ("workflow_type", self.workflow_type.as_deref()),
928            ("task_queue", self.task_queue.as_deref()),
929        ] {
930            if value.is_some_and(|value| value.trim().is_empty()) {
931                return Err(ContinueAsNewOptionsError {
932                    field,
933                    message: format!("{field} must not be empty"),
934                });
935            }
936        }
937        Ok(())
938    }
939}
940
941/// A stable validation error raised before a continue-as-new command is emitted.
942#[derive(Clone, Debug, Error, PartialEq, Eq)]
943#[error("invalid continue-as-new option {field}: {message}")]
944pub struct ContinueAsNewOptionsError {
945    pub field: &'static str,
946    pub message: String,
947}
948
949/// Public history-budget information attached to the current workflow task.
950#[derive(Clone, Debug, Default, PartialEq, Eq)]
951pub struct WorkflowHistoryBudget {
952    pub event_count: u64,
953    pub size_bytes: Option<u64>,
954    pub continue_as_new_recommended: bool,
955    pub pressure: Option<String>,
956}
957
958#[doc(hidden)]
959#[derive(Clone, Debug)]
960pub struct ContinueAsNewRequest {
961    arguments: AvroValue,
962    options: ContinueAsNewOptions,
963}
964
965impl WorkflowCommandOptions {
966    pub fn new() -> Self {
967        Self::default()
968    }
969
970    pub fn reason(mut self, reason: impl Into<String>) -> Self {
971        self.reason = Some(reason.into());
972        self
973    }
974
975    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
976        self.request_id = Some(request_id.into());
977        self
978    }
979}
980
981/// The accepted, machine-readable result of a lifecycle command.
982#[derive(Clone, Debug, PartialEq)]
983pub struct WorkflowCommandResult {
984    pub command: WorkflowCommandKind,
985    pub workflow_id: String,
986    pub run_id: Option<String>,
987    pub outcome: Option<String>,
988    pub reason: Option<String>,
989    pub command_status: Option<String>,
990    pub raw: Value,
991}
992
993/// A new run continuing a failed run from its recorded activity boundary.
994#[derive(Clone, Debug, PartialEq)]
995pub struct WorkflowRedriveResult {
996    pub workflow_id: String,
997    pub source_run_id: String,
998    pub run_id: String,
999    pub resume_step_sequence: u64,
1000    pub raw: Value,
1001}
1002
1003/// A stable rejection returned by instance- or selected-run lifecycle commands.
1004#[derive(Clone, Debug, Error)]
1005#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
1006pub struct WorkflowCommandRejection {
1007    pub command: WorkflowCommandKind,
1008    pub status: u16,
1009    pub reason: String,
1010    pub message: String,
1011    pub workflow_id: String,
1012    pub run_id: Option<String>,
1013    pub target_scope: Option<String>,
1014    pub body: Value,
1015}
1016
1017/// Stable terminal categories returned by [`WorkflowHandle::result`].
1018#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1019pub enum WorkflowTerminalKind {
1020    Failed,
1021    Cancelled,
1022    Terminated,
1023    TimedOut,
1024}
1025
1026/// A typed terminal workflow outcome with durable identity and failure metadata.
1027///
1028/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
1029/// of parsing its display representation. Fields remain `None` when an older
1030/// server did not publish that metadata.
1031#[derive(Clone, Debug, Error)]
1032#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
1033pub struct WorkflowTerminalOutcome {
1034    pub kind: WorkflowTerminalKind,
1035    pub workflow_id: String,
1036    pub run_id: Option<String>,
1037    pub reason: String,
1038    pub failure_category: Option<String>,
1039    pub failure_id: Option<String>,
1040    pub exception_type: Option<String>,
1041    pub exception_class: Option<String>,
1042    pub non_retryable: Option<bool>,
1043    pub message: Option<String>,
1044    pub exception: Option<Value>,
1045    pub raw: Value,
1046}
1047
1048/// A worker-side activity settlement or heartbeat rejected by durable state.
1049#[derive(Clone, Debug, Error)]
1050#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
1051pub struct ActivityTaskRejection {
1052    pub operation: String,
1053    pub status: u16,
1054    pub reason: String,
1055    pub task_id: String,
1056    pub activity_attempt_id: String,
1057    pub cancel_requested: bool,
1058    pub can_continue: Option<bool>,
1059    pub run_closed_reason: Option<String>,
1060    pub body: Value,
1061}
1062
1063/// Stable validation categories for [`ActivityOptions`].
1064#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1065pub enum ActivityOptionsErrorKind {
1066    EmptyTaskQueue,
1067    EmptyRetryPolicy,
1068    InvalidMaxAttempts,
1069    BackoffWithoutRetryBudget,
1070    TooManyBackoffIntervals,
1071    InvalidBackoffCoefficient,
1072    BackoffGenerationTooLarge,
1073    BackoffOverflow,
1074    EmptyNonRetryableErrorType,
1075    TimeoutNotPositive,
1076    MissingTotalTimeout,
1077    TimeoutOverflow,
1078    TimeoutOrder,
1079}
1080
1081/// A machine-readable activity-options validation failure.
1082#[derive(Clone, Debug, Error, PartialEq, Eq)]
1083#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
1084pub struct ActivityOptionsError {
1085    pub kind: ActivityOptionsErrorKind,
1086    pub field: Option<&'static str>,
1087    pub message: String,
1088}
1089
1090impl ActivityOptionsError {
1091    fn new(
1092        kind: ActivityOptionsErrorKind,
1093        field: Option<&'static str>,
1094        message: impl Into<String>,
1095    ) -> Self {
1096        Self {
1097            kind,
1098            field,
1099            message: message.into(),
1100        }
1101    }
1102}
1103
1104/// Stable terminal categories returned when an awaited activity does not succeed.
1105#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1106pub enum ActivityFailureKind {
1107    Failed,
1108    Cancelled,
1109    TimedOut,
1110}
1111
1112/// A stable, machine-readable terminal activity failure.
1113///
1114/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
1115/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
1116#[derive(Clone, Debug, Error, PartialEq)]
1117#[error("activity failed ({reason}): {message}")]
1118pub struct ActivityFailure {
1119    pub kind: ActivityFailureKind,
1120    pub reason: String,
1121    pub message: String,
1122    pub activity_execution_id: Option<String>,
1123    pub activity_attempt_id: Option<String>,
1124    pub activity_type: Option<String>,
1125    pub activity_class: Option<String>,
1126    pub attempt_number: Option<u64>,
1127    pub failure_id: Option<String>,
1128    pub failure_category: Option<String>,
1129    pub timeout_kind: Option<String>,
1130    pub non_retryable: bool,
1131    pub exception_type: Option<String>,
1132    pub exception_class: Option<String>,
1133    pub code: Option<Value>,
1134    pub exception: Option<Value>,
1135}
1136
1137/// Stable terminal categories returned when an awaited child does not succeed.
1138#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1139pub enum ChildWorkflowFailureKind {
1140    Failed,
1141    Cancelled,
1142    Terminated,
1143}
1144
1145/// A stable, machine-readable child workflow failure delivered to its parent.
1146///
1147/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
1148/// of parsing the display message. Child and parent identifiers retain the
1149/// relationship recorded in durable history across worker restarts.
1150#[derive(Clone, Debug, Error)]
1151#[error("child workflow failed ({reason}): {message}")]
1152pub struct ChildWorkflowFailure {
1153    pub kind: ChildWorkflowFailureKind,
1154    pub reason: String,
1155    pub message: String,
1156    pub parent_workflow_id: Option<String>,
1157    pub parent_workflow_run_id: Option<String>,
1158    pub child_workflow_id: Option<String>,
1159    pub child_workflow_run_id: Option<String>,
1160    pub child_workflow_type: Option<String>,
1161    pub failure_id: Option<String>,
1162    pub failure_category: Option<String>,
1163    pub exception_type: Option<String>,
1164    pub exception_class: Option<String>,
1165    pub non_retryable: bool,
1166    pub code: Option<Value>,
1167    pub exception: Option<Value>,
1168}
1169
1170/// The identity of one durable workflow execution.
1171#[derive(Clone, Debug, PartialEq, Eq)]
1172pub struct WorkflowIdentity {
1173    pub workflow_id: Option<String>,
1174    pub run_id: Option<String>,
1175}
1176
1177/// A successful child result together with its durable parent-child identity.
1178#[derive(Clone, Debug, PartialEq)]
1179pub struct ChildWorkflowResult {
1180    pub parent: WorkflowIdentity,
1181    pub child: WorkflowIdentity,
1182    pub child_workflow_type: Option<String>,
1183    pub result: Value,
1184}
1185
1186/// Lossless successful child result for fixed Avro Value workflows.
1187#[derive(Clone, Debug, PartialEq)]
1188pub struct ChildWorkflowAvroResult {
1189    pub parent: WorkflowIdentity,
1190    pub child: WorkflowIdentity,
1191    pub child_workflow_type: Option<String>,
1192    pub result: AvroValue,
1193}
1194
1195/// Stable user-facing identity for one member of a durable selection group.
1196#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1197#[serde(untagged)]
1198pub enum SelectionKey {
1199    Index(usize),
1200    Name(String),
1201}
1202
1203impl From<usize> for SelectionKey {
1204    fn from(value: usize) -> Self {
1205        Self::Index(value)
1206    }
1207}
1208
1209impl From<String> for SelectionKey {
1210    fn from(value: String) -> Self {
1211        Self::Name(value)
1212    }
1213}
1214
1215impl From<&str> for SelectionKey {
1216    fn from(value: &str) -> Self {
1217        Self::Name(value.to_string())
1218    }
1219}
1220
1221/// Typed result of explicitly awaiting a cancelled non-winning operation.
1222#[derive(Clone, Debug, Error, PartialEq, Eq)]
1223#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1224pub struct DurableOperationCancelled {
1225    pub selection_group_id: String,
1226    pub member_key: SelectionKey,
1227    pub member_index: usize,
1228    pub operation_kind: String,
1229    pub operation_identity: String,
1230}
1231
1232/// Stable identity for one enclosing deterministic parallel group.
1233///
1234/// The same fields are attached to every ordinary activity, timer, or child
1235/// workflow command in the group. Nested leaves carry an outer-to-inner path;
1236/// no Rust-specific wire command is introduced.
1237#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1238pub struct ParallelGroupMetadata {
1239    pub parallel_group_id: String,
1240    pub parallel_group_kind: String,
1241    pub parallel_group_base_sequence: u64,
1242    pub parallel_group_size: usize,
1243    pub parallel_group_index: usize,
1244    #[serde(default, skip_serializing_if = "Option::is_none")]
1245    pub parallel_group_mode: Option<String>,
1246    #[serde(default, skip_serializing_if = "Option::is_none")]
1247    pub selection_member_key: Option<SelectionKey>,
1248    #[serde(default, skip_serializing_if = "Option::is_none")]
1249    pub selection_member_index: Option<usize>,
1250    #[serde(default, skip_serializing_if = "Option::is_none")]
1251    pub selection_member_base_sequence: Option<u64>,
1252    #[serde(default, skip_serializing_if = "Option::is_none")]
1253    pub selection_member_size: Option<usize>,
1254    #[serde(default, skip_serializing_if = "Option::is_none")]
1255    pub selection_member_kind: Option<String>,
1256}
1257
1258/// One input-ordered result returned by [`WorkflowContext::parallel`].
1259#[derive(Clone, Debug, PartialEq)]
1260pub enum ParallelResult {
1261    Activity(Value),
1262    ChildWorkflow(ChildWorkflowResult),
1263    Timer,
1264    Signal(Vec<Value>),
1265    Condition(ConditionWaitResult),
1266    Group(Vec<ParallelResult>),
1267}
1268
1269/// Lossless fixed-Avro counterpart to [`ParallelResult`].
1270#[derive(Clone, Debug, PartialEq)]
1271pub enum ParallelAvroResult {
1272    Activity(AvroValue),
1273    ChildWorkflow(ChildWorkflowAvroResult),
1274    Timer,
1275    Signal(Vec<AvroValue>),
1276    Condition(ConditionWaitResult),
1277    Group(Vec<ParallelAvroResult>),
1278}
1279
1280impl ParallelAvroResult {
1281    fn into_json_result(self) -> Result<ParallelResult> {
1282        match self {
1283            Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1284            Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1285                parent: result.parent,
1286                child: result.child,
1287                child_workflow_type: result.child_workflow_type,
1288                result: result.result.into_json()?,
1289            })),
1290            Self::Timer => Ok(ParallelResult::Timer),
1291            Self::Signal(values) => Ok(ParallelResult::Signal(
1292                values
1293                    .into_iter()
1294                    .map(AvroValue::into_json)
1295                    .collect::<Result<Vec<_>>>()?,
1296            )),
1297            Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1298            Self::Group(results) => Ok(ParallelResult::Group(
1299                results
1300                    .into_iter()
1301                    .map(Self::into_json_result)
1302                    .collect::<Result<Vec<_>>>()?,
1303            )),
1304        }
1305    }
1306}
1307
1308/// One successful leaf retained when another parallel member failed.
1309#[derive(Clone, Debug, PartialEq)]
1310pub struct ParallelCompletion {
1311    pub member_path: Vec<usize>,
1312    pub result: ParallelResult,
1313}
1314
1315/// A deterministic join failed after some siblings had already completed.
1316///
1317/// `cause` retains the typed activity, child-workflow, cancellation, or codec
1318/// error. `completed` is declaration ordered and contains only durable
1319/// successes observed in the same replay. Late sibling completions can add
1320/// entries on a later replay without changing `member_path` or the selected
1321/// positional failure.
1322#[derive(Debug, Error)]
1323#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1324pub struct ParallelFailure {
1325    pub group_id: String,
1326    pub member_path: Vec<usize>,
1327    pub group_path: Vec<ParallelGroupMetadata>,
1328    pub completed: Vec<ParallelCompletion>,
1329    #[source]
1330    pub cause: Box<Error>,
1331}
1332
1333/// Stable validation error returned before an invalid group emits commands.
1334#[derive(Clone, Debug, Error, PartialEq, Eq)]
1335#[error("invalid deterministic parallel group ({reason}): {message}")]
1336pub struct ParallelGroupError {
1337    pub reason: &'static str,
1338    pub member_path: Vec<usize>,
1339    pub message: String,
1340}
1341
1342/// Cooperative workflow cancellation observed at an author-controlled point.
1343#[derive(Clone, Debug, Error, PartialEq, Eq)]
1344#[error("workflow cancellation was requested")]
1345pub struct WorkflowCancellationRequested;
1346
1347/// A forward saga failure followed by a terminal compensation failure.
1348#[derive(Debug, Error)]
1349#[error(
1350    "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1351)]
1352pub struct SagaCompensationFailure {
1353    pub initiating_failure: Box<Error>,
1354    pub compensation_failure: Box<Error>,
1355    pub compensation_activity_type: String,
1356    pub compensation_registration_order: usize,
1357}
1358
1359/// Server behavior when a parent closes while its child is still open.
1360#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1361pub enum ParentClosePolicy {
1362    #[default]
1363    Abandon,
1364    /// Legacy terminal cancellation without cooperative cleanup.
1365    RequestCancel,
1366    /// Request cooperative cleanup with the original lineage and deadline.
1367    RequestCancellation,
1368    Terminate,
1369}
1370
1371impl ParentClosePolicy {
1372    fn as_str(self) -> &'static str {
1373        match self {
1374            Self::Abandon => "abandon",
1375            Self::RequestCancel => "request_cancel",
1376            Self::RequestCancellation => "request_cancellation",
1377            Self::Terminate => "terminate",
1378        }
1379    }
1380}
1381
1382/// Cancellation at an awaiting operation. Activities default to Try, children to Abandon.
1383#[non_exhaustive]
1384#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1385pub enum CancellationPolicy {
1386    /// Request cancellation and continue without waiting. Historical Activity default.
1387    TryCancel,
1388    /// Wait for recorded child termination or the original Activity attempt's stop receipt.
1389    WaitCancellationCompleted,
1390    /// Leave work independent. Historical child default. Remote Activities require a total timeout.
1391    #[default]
1392    Abandon,
1393}
1394
1395impl CancellationPolicy {
1396    fn as_str(self) -> &'static str {
1397        match self {
1398            Self::TryCancel => "try_cancel",
1399            Self::WaitCancellationCompleted => "wait_cancellation_completed",
1400            Self::Abandon => "abandon",
1401        }
1402    }
1403}
1404
1405/// Durable retry policy for one child workflow invocation.
1406#[derive(Clone, Debug, Default, PartialEq, Eq)]
1407pub struct ChildWorkflowRetryPolicy {
1408    pub max_attempts: Option<u32>,
1409    pub backoff_seconds: Vec<u64>,
1410    pub non_retryable_error_types: Vec<String>,
1411}
1412
1413/// Options recorded with a child-workflow command.
1414///
1415/// The task queue is mandatory so routing is explicit and replay-stable.
1416#[derive(Clone, Debug, PartialEq, Eq)]
1417pub struct ChildWorkflowOptions {
1418    pub task_queue: String,
1419    pub parent_close_policy: ParentClosePolicy,
1420    pub cancellation_policy: CancellationPolicy,
1421    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1422    pub execution_timeout_seconds: Option<u64>,
1423    pub run_timeout_seconds: Option<u64>,
1424}
1425
1426impl ChildWorkflowOptions {
1427    pub fn new(task_queue: impl Into<String>) -> Self {
1428        Self {
1429            task_queue: task_queue.into(),
1430            parent_close_policy: ParentClosePolicy::Abandon,
1431            cancellation_policy: CancellationPolicy::Abandon,
1432            retry_policy: None,
1433            execution_timeout_seconds: None,
1434            run_timeout_seconds: None,
1435        }
1436    }
1437
1438    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1439        self.parent_close_policy = policy;
1440        self
1441    }
1442
1443    pub fn cancellation_policy(mut self, policy: CancellationPolicy) -> Self {
1444        self.cancellation_policy = policy;
1445        self
1446    }
1447
1448    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1449        self.retry_policy = Some(policy);
1450        self
1451    }
1452
1453    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1454        self.execution_timeout_seconds = Some(seconds);
1455        self
1456    }
1457
1458    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1459        self.run_timeout_seconds = Some(seconds);
1460        self
1461    }
1462}
1463
1464/// Backoff intervals for one durable activity retry policy.
1465#[derive(Clone, Debug, PartialEq, Eq)]
1466pub enum ActivityBackoff {
1467    /// Use these intervals between attempts. The server repeats the final
1468    /// interval if the retry budget contains more attempts than entries.
1469    Explicit(Vec<Duration>),
1470    /// Generate one interval for every retry using integer exponential growth.
1471    Exponential {
1472        initial_interval: Duration,
1473        coefficient: u32,
1474        maximum_interval: Option<Duration>,
1475    },
1476}
1477
1478/// Durable server-side retry policy for one activity execution.
1479#[derive(Clone, Debug, Default, PartialEq, Eq)]
1480pub struct ActivityRetryPolicy {
1481    pub max_attempts: Option<u32>,
1482    pub backoff: Option<ActivityBackoff>,
1483    pub non_retryable_error_types: Vec<String>,
1484}
1485
1486impl ActivityRetryPolicy {
1487    /// Start a policy with a finite attempt budget, including the first attempt.
1488    pub fn new(max_attempts: u32) -> Self {
1489        Self {
1490            max_attempts: Some(max_attempts),
1491            ..Self::default()
1492        }
1493    }
1494
1495    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1496        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1497        self
1498    }
1499
1500    pub fn exponential_backoff(
1501        mut self,
1502        initial_interval: Duration,
1503        coefficient: u32,
1504        maximum_interval: Option<Duration>,
1505    ) -> Self {
1506        self.backoff = Some(ActivityBackoff::Exponential {
1507            initial_interval,
1508            coefficient,
1509            maximum_interval,
1510        });
1511        self
1512    }
1513
1514    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1515        self.non_retryable_error_types.push(error_type.into());
1516        self
1517    }
1518
1519    pub fn non_retryable_error_types(
1520        mut self,
1521        error_types: impl IntoIterator<Item = impl Into<String>>,
1522    ) -> Self {
1523        self.non_retryable_error_types
1524            .extend(error_types.into_iter().map(Into::into));
1525        self
1526    }
1527}
1528
1529/// Options recorded atomically on one deterministic `schedule_activity` command.
1530///
1531/// Durations are rounded up to whole seconds when encoded, so the server never
1532/// receives a shorter timeout or backoff than the caller requested.
1533#[derive(Clone, Debug, Default, PartialEq, Eq)]
1534pub struct ActivityOptions {
1535    pub task_queue: Option<String>,
1536    pub retry_policy: Option<ActivityRetryPolicy>,
1537    pub start_to_close_timeout: Option<Duration>,
1538    pub schedule_to_start_timeout: Option<Duration>,
1539    pub schedule_to_close_timeout: Option<Duration>,
1540    pub heartbeat_timeout: Option<Duration>,
1541    pub cancellation_policy: Option<CancellationPolicy>,
1542}
1543
1544impl ActivityOptions {
1545    pub fn new() -> Self {
1546        Self::default()
1547    }
1548
1549    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1550        self.task_queue = Some(task_queue.into());
1551        self
1552    }
1553
1554    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1555        self.retry_policy = Some(policy);
1556        self
1557    }
1558
1559    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1560        self.start_to_close_timeout = Some(timeout);
1561        self
1562    }
1563
1564    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1565        self.schedule_to_start_timeout = Some(timeout);
1566        self
1567    }
1568
1569    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1570        self.schedule_to_close_timeout = Some(timeout);
1571        self
1572    }
1573
1574    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1575        self.heartbeat_timeout = Some(timeout);
1576        self
1577    }
1578
1579    pub fn cancellation_policy(mut self, policy: CancellationPolicy) -> Self {
1580        self.cancellation_policy = Some(policy);
1581        self
1582    }
1583
1584    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1585        if self.cancellation_policy == Some(CancellationPolicy::Abandon)
1586            && self.schedule_to_close_timeout.is_none()
1587        {
1588            return Err(ActivityOptionsError::new(
1589                ActivityOptionsErrorKind::MissingTotalTimeout,
1590                Some("schedule_to_close_timeout"),
1591                "remote Activity Abandon requires a finite positive total timeout",
1592            ));
1593        }
1594        if self
1595            .task_queue
1596            .as_deref()
1597            .is_some_and(|queue| queue.trim().is_empty())
1598        {
1599            return Err(ActivityOptionsError::new(
1600                ActivityOptionsErrorKind::EmptyTaskQueue,
1601                Some("task_queue"),
1602                "task_queue must not be empty",
1603            ));
1604        }
1605
1606        for (field, value) in [
1607            ("start_to_close_timeout", self.start_to_close_timeout),
1608            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1609            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1610            ("heartbeat_timeout", self.heartbeat_timeout),
1611        ] {
1612            if value.is_some_and(|value| value.is_zero()) {
1613                return Err(ActivityOptionsError::new(
1614                    ActivityOptionsErrorKind::TimeoutNotPositive,
1615                    Some(field),
1616                    format!("{field} must be positive"),
1617                ));
1618            }
1619        }
1620
1621        validate_timeout_order(
1622            "heartbeat_timeout",
1623            self.heartbeat_timeout,
1624            "start_to_close_timeout",
1625            self.start_to_close_timeout,
1626        )?;
1627        validate_timeout_order(
1628            "start_to_close_timeout",
1629            self.start_to_close_timeout,
1630            "schedule_to_close_timeout",
1631            self.schedule_to_close_timeout,
1632        )?;
1633        validate_timeout_order(
1634            "schedule_to_start_timeout",
1635            self.schedule_to_start_timeout,
1636            "schedule_to_close_timeout",
1637            self.schedule_to_close_timeout,
1638        )?;
1639
1640        Ok(ValidatedActivityOptions {
1641            task_queue: self.task_queue.clone(),
1642            retry_policy: self
1643                .retry_policy
1644                .as_ref()
1645                .map(validate_activity_retry_policy)
1646                .transpose()?,
1647            start_to_close_timeout: timeout_seconds(
1648                "start_to_close_timeout",
1649                self.start_to_close_timeout,
1650            )?,
1651            schedule_to_start_timeout: timeout_seconds(
1652                "schedule_to_start_timeout",
1653                self.schedule_to_start_timeout,
1654            )?,
1655            schedule_to_close_timeout: timeout_seconds(
1656                "schedule_to_close_timeout",
1657                self.schedule_to_close_timeout,
1658            )?,
1659            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1660            cancellation_policy: self.cancellation_policy,
1661        })
1662    }
1663}
1664
1665/// A deferred durable leaf or nested group for [`WorkflowContext::parallel`].
1666///
1667/// Constructors capture arguments but perform no I/O. The join validates the
1668/// complete tree, attaches the existing parallel-group metadata to every
1669/// ordinary command, schedules all leaves, and then suspends.
1670pub enum ParallelOperation {
1671    Activity {
1672        activity_type: String,
1673        options: ActivityOptions,
1674        arguments: Result<AvroValue>,
1675    },
1676    ChildWorkflow {
1677        workflow_type: String,
1678        options: ChildWorkflowOptions,
1679        arguments: Result<AvroValue>,
1680    },
1681    Timer(Duration),
1682    Signal(String),
1683    Condition {
1684        options: ConditionWaitOptions,
1685        predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1686    },
1687    Group(Vec<ParallelOperation>),
1688}
1689
1690impl ParallelOperation {
1691    pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1692        Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1693    }
1694
1695    pub fn activity_with_options<T: Serialize>(
1696        activity_type: impl Into<String>,
1697        options: ActivityOptions,
1698        args: T,
1699    ) -> Self {
1700        Self::Activity {
1701            activity_type: activity_type.into(),
1702            options,
1703            arguments: AvroValue::from_serialize(&args),
1704        }
1705    }
1706
1707    pub fn child_workflow<T: Serialize>(
1708        workflow_type: impl Into<String>,
1709        options: ChildWorkflowOptions,
1710        args: T,
1711    ) -> Self {
1712        Self::ChildWorkflow {
1713            workflow_type: workflow_type.into(),
1714            options,
1715            arguments: AvroValue::from_serialize(&args),
1716        }
1717    }
1718
1719    pub fn timer(duration: Duration) -> Self {
1720        Self::Timer(duration)
1721    }
1722
1723    pub fn signal(signal_name: impl Into<String>) -> Self {
1724        Self::Signal(signal_name.into())
1725    }
1726
1727    pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1728    where
1729        F: Fn() -> Result<bool> + Send + 'static,
1730    {
1731        Self::Condition {
1732            options,
1733            predicate: Box::new(predicate),
1734        }
1735    }
1736
1737    pub fn group(operations: Vec<ParallelOperation>) -> Self {
1738        Self::Group(operations)
1739    }
1740}
1741
1742#[derive(Clone, Debug)]
1743struct ValidatedActivityOptions {
1744    task_queue: Option<String>,
1745    retry_policy: Option<Value>,
1746    start_to_close_timeout: Option<u64>,
1747    schedule_to_start_timeout: Option<u64>,
1748    schedule_to_close_timeout: Option<u64>,
1749    heartbeat_timeout: Option<u64>,
1750    cancellation_policy: Option<CancellationPolicy>,
1751}
1752
1753fn validate_timeout_order(
1754    smaller_name: &'static str,
1755    smaller: Option<Duration>,
1756    larger_name: &'static str,
1757    larger: Option<Duration>,
1758) -> std::result::Result<(), ActivityOptionsError> {
1759    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1760        return Err(ActivityOptionsError::new(
1761            ActivityOptionsErrorKind::TimeoutOrder,
1762            Some(smaller_name),
1763            format!("{smaller_name} must be <= {larger_name}"),
1764        ));
1765    }
1766    Ok(())
1767}
1768
1769fn timeout_seconds(
1770    field: &'static str,
1771    value: Option<Duration>,
1772) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1773    value
1774        .map(|value| {
1775            activity_protocol_seconds(value).ok_or_else(|| {
1776                ActivityOptionsError::new(
1777                    ActivityOptionsErrorKind::TimeoutOverflow,
1778                    Some(field),
1779                    format!("{field} is too large for the worker protocol"),
1780                )
1781            })
1782        })
1783        .transpose()
1784}
1785
1786fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1787    value
1788        .as_secs()
1789        .checked_add(u64::from(value.subsec_nanos() > 0))
1790}
1791
1792fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1793    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1794}
1795
1796fn validate_activity_retry_policy(
1797    policy: &ActivityRetryPolicy,
1798) -> std::result::Result<Value, ActivityOptionsError> {
1799    if policy.max_attempts.is_none()
1800        && policy.backoff.is_none()
1801        && policy.non_retryable_error_types.is_empty()
1802    {
1803        return Err(ActivityOptionsError::new(
1804            ActivityOptionsErrorKind::EmptyRetryPolicy,
1805            Some("retry_policy"),
1806            "retry_policy must configure at least one field",
1807        ));
1808    }
1809    if policy.max_attempts == Some(0) {
1810        return Err(ActivityOptionsError::new(
1811            ActivityOptionsErrorKind::InvalidMaxAttempts,
1812            Some("retry_policy.max_attempts"),
1813            "max_attempts must be >= 1",
1814        ));
1815    }
1816    if policy
1817        .non_retryable_error_types
1818        .iter()
1819        .any(|error_type| error_type.trim().is_empty())
1820    {
1821        return Err(ActivityOptionsError::new(
1822            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1823            Some("retry_policy.non_retryable_error_types"),
1824            "non_retryable_error_types must not contain empty values",
1825        ));
1826    }
1827
1828    let backoff_seconds = match &policy.backoff {
1829        None => None,
1830        Some(backoff) => {
1831            let max_attempts = policy.max_attempts.ok_or_else(|| {
1832                ActivityOptionsError::new(
1833                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1834                    Some("retry_policy.backoff"),
1835                    "backoff requires max_attempts",
1836                )
1837            })?;
1838            let retry_count = max_attempts.saturating_sub(1) as usize;
1839            let intervals = match backoff {
1840                ActivityBackoff::Explicit(intervals) => {
1841                    if intervals.len() > retry_count {
1842                        return Err(ActivityOptionsError::new(
1843                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
1844                            Some("retry_policy.backoff"),
1845                            "backoff interval count must not exceed max_attempts - 1",
1846                        ));
1847                    }
1848                    intervals.clone()
1849                }
1850                ActivityBackoff::Exponential {
1851                    initial_interval,
1852                    coefficient,
1853                    maximum_interval,
1854                } => {
1855                    if *coefficient < 1 {
1856                        return Err(ActivityOptionsError::new(
1857                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1858                            Some("retry_policy.backoff.coefficient"),
1859                            "backoff coefficient must be >= 1",
1860                        ));
1861                    }
1862                    if retry_count > 10_000 {
1863                        return Err(ActivityOptionsError::new(
1864                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1865                            Some("retry_policy.max_attempts"),
1866                            "generated backoff supports at most 10000 retry intervals",
1867                        ));
1868                    }
1869                    let mut current = *initial_interval;
1870                    let mut intervals = Vec::with_capacity(retry_count);
1871                    for _ in 0..retry_count {
1872                        let interval = maximum_interval
1873                            .map(|maximum| current.min(maximum))
1874                            .unwrap_or(current);
1875                        intervals.push(interval);
1876                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
1877                            break;
1878                        }
1879                        current = current.checked_mul(*coefficient).ok_or_else(|| {
1880                            ActivityOptionsError::new(
1881                                ActivityOptionsErrorKind::BackoffOverflow,
1882                                Some("retry_policy.backoff"),
1883                                "generated backoff interval overflowed",
1884                            )
1885                        })?;
1886                    }
1887                    intervals
1888                }
1889            };
1890            Some(
1891                intervals
1892                    .into_iter()
1893                    .map(|interval| {
1894                        activity_protocol_seconds(interval).ok_or_else(|| {
1895                            ActivityOptionsError::new(
1896                                ActivityOptionsErrorKind::BackoffOverflow,
1897                                Some("retry_policy.backoff"),
1898                                "backoff interval is too large for the worker protocol",
1899                            )
1900                        })
1901                    })
1902                    .collect::<std::result::Result<Vec<_>, _>>()?,
1903            )
1904        }
1905    };
1906
1907    let mut encoded = serde_json::Map::new();
1908    if let Some(max_attempts) = policy.max_attempts {
1909        encoded.insert("max_attempts".to_string(), json!(max_attempts));
1910    }
1911    if let Some(backoff_seconds) = backoff_seconds {
1912        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1913    }
1914    if !policy.non_retryable_error_types.is_empty() {
1915        let mut canonical_error_types = Vec::new();
1916        for error_type in policy
1917            .non_retryable_error_types
1918            .iter()
1919            .map(|error_type| error_type.trim())
1920        {
1921            if !canonical_error_types.contains(&error_type) {
1922                canonical_error_types.push(error_type);
1923            }
1924        }
1925        encoded.insert(
1926            "non_retryable_error_types".to_string(),
1927            json!(canonical_error_types),
1928        );
1929    }
1930    Ok(Value::Object(encoded))
1931}
1932
1933/// A stable, machine-readable failure raised when workflow code no longer
1934/// reconstructs the durable command stream recorded in history.
1935#[derive(Clone, Debug, Error)]
1936#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1937pub struct ReplayFailure {
1938    pub reason: String,
1939    pub sequence: Option<u64>,
1940    pub expected: Option<String>,
1941    pub actual: Option<String>,
1942    pub message: String,
1943}
1944
1945impl ReplayFailure {
1946    fn new(
1947        reason: impl Into<String>,
1948        sequence: Option<u64>,
1949        expected: Option<String>,
1950        actual: Option<String>,
1951        message: impl Into<String>,
1952    ) -> Self {
1953        Self {
1954            reason: reason.into(),
1955            sequence,
1956            expected,
1957            actual,
1958            message: message.into(),
1959        }
1960    }
1961}
1962
1963/// A stable, machine-readable workflow query or query-task settlement failure.
1964#[derive(Clone, Debug, Error)]
1965#[error("query failed ({reason}, HTTP {status}): {message}")]
1966pub struct QueryFailure {
1967    pub status: u16,
1968    pub reason: String,
1969    pub message: String,
1970    pub body: Value,
1971}
1972
1973/// A stable failure returned when a server rejects an SDK protocol version.
1974#[derive(Clone, Debug, Error)]
1975#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1976pub struct ProtocolFailure {
1977    pub status: u16,
1978    pub reason: String,
1979    pub message: String,
1980    pub supported_version: Option<String>,
1981    pub requested_version: Option<String>,
1982    pub body: Value,
1983}
1984
1985#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1986pub struct PayloadEnvelope {
1987    pub codec: String,
1988    pub blob: String,
1989}
1990
1991impl PayloadEnvelope {
1992    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1993        encode_payload(value, DEFAULT_CODEC)
1994    }
1995
1996    /// Encode an explicit typed value, including the bytes branch that JSON
1997    /// serialization cannot represent.
1998    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1999        encode_avro_value(value)
2000    }
2001}
2002
2003/// Native adapter for the fixed language-neutral Avro Value schema.
2004#[derive(Clone, Debug)]
2005pub enum AvroValue {
2006    Null,
2007    Boolean(bool),
2008    Long(i64),
2009    Double(f64),
2010    Bytes(Vec<u8>),
2011    String(String),
2012    Array(Vec<AvroValue>),
2013    Map(BTreeMap<String, AvroValue>),
2014}
2015
2016impl PartialEq for AvroValue {
2017    fn eq(&self, other: &Self) -> bool {
2018        match (self, other) {
2019            (Self::Null, Self::Null) => true,
2020            (Self::Boolean(left), Self::Boolean(right)) => left == right,
2021            (Self::Long(left), Self::Long(right)) => left == right,
2022            (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
2023            (Self::Bytes(left), Self::Bytes(right)) => left == right,
2024            (Self::String(left), Self::String(right)) => left == right,
2025            (Self::Array(left), Self::Array(right)) => left == right,
2026            (Self::Map(left), Self::Map(right)) => left == right,
2027            _ => false,
2028        }
2029    }
2030}
2031
2032impl AvroValue {
2033    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
2034        Self::from_serde_value(
2035            serde_value::to_value(value).map_err(|error| {
2036                Error::Codec(format!("could not adapt value for Avro: {error}"))
2037            })?,
2038        )
2039    }
2040
2041    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
2042        use serde_value::Value as SerdeValue;
2043
2044        match value {
2045            SerdeValue::Unit => Ok(Self::Null),
2046            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
2047            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
2048            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
2049            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
2050            SerdeValue::I64(value) => Ok(Self::Long(value)),
2051            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
2052            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
2053            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
2054            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
2055                Error::Codec(
2056                    "integer_overflow: Avro Value long must be within signed 64-bit range"
2057                        .to_string(),
2058                )
2059            }),
2060            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
2061            SerdeValue::F64(value) => Self::finite_double(value),
2062            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
2063            SerdeValue::String(value) => Ok(Self::String(value)),
2064            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
2065            SerdeValue::Option(None) => Ok(Self::Null),
2066            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
2067                Self::from_serde_value(*value)
2068            }
2069            SerdeValue::Seq(values) => values
2070                .into_iter()
2071                .map(Self::from_serde_value)
2072                .collect::<Result<Vec<_>>>()
2073                .map(Self::Array),
2074            SerdeValue::Map(values) => values
2075                .into_iter()
2076                .map(|(key, value)| {
2077                    let SerdeValue::String(key) = key else {
2078                        return Err(Error::Codec(
2079                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
2080                        ));
2081                    };
2082
2083                    Ok((key, Self::from_serde_value(value)?))
2084                })
2085                .collect::<Result<BTreeMap<_, _>>>()
2086                .map(Self::Map),
2087        }
2088    }
2089
2090    fn finite_double(value: f64) -> Result<Self> {
2091        if !value.is_finite() {
2092            return Err(Error::Codec(
2093                "non_finite_float: Avro Value doubles must be finite".to_string(),
2094            ));
2095        }
2096
2097        Ok(Self::Double(value))
2098    }
2099
2100    fn into_json(self) -> Result<Value> {
2101        match self {
2102            Self::Null => Ok(Value::Null),
2103            Self::Boolean(value) => Ok(Value::Bool(value)),
2104            Self::Long(value) => Ok(Value::Number(value.into())),
2105            Self::Double(value) => serde_json::Number::from_f64(value)
2106                .map(Value::Number)
2107                .ok_or_else(|| {
2108                    Error::Codec(
2109                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
2110                    )
2111                }),
2112            Self::Bytes(value) => Ok(json!({
2113                "$type": "bytes",
2114                "base64": BASE64.encode(value),
2115            })),
2116            Self::String(value) => Ok(Value::String(value)),
2117            Self::Array(values) => values
2118                .into_iter()
2119                .map(Self::into_json)
2120                .collect::<Result<Vec<_>>>()
2121                .map(Value::Array),
2122            Self::Map(values) => values
2123                .into_iter()
2124                .map(|(key, value)| Ok((key, value.into_json()?)))
2125                .collect::<Result<serde_json::Map<_, _>>>()
2126                .map(Value::Object),
2127        }
2128    }
2129
2130    fn into_serde_value(self) -> serde_value::Value {
2131        use serde_value::Value as SerdeValue;
2132
2133        match self {
2134            Self::Null => SerdeValue::Unit,
2135            Self::Boolean(value) => SerdeValue::Bool(value),
2136            Self::Long(value) => SerdeValue::I64(value),
2137            Self::Double(value) => SerdeValue::F64(value),
2138            Self::Bytes(value) => SerdeValue::Bytes(value),
2139            Self::String(value) => SerdeValue::String(value),
2140            Self::Array(values) => {
2141                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
2142            }
2143            Self::Map(values) => SerdeValue::Map(
2144                values
2145                    .into_iter()
2146                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
2147                    .collect(),
2148            ),
2149        }
2150    }
2151
2152    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
2153        self.into_serde_value().deserialize_into().map_err(|error| {
2154            Error::Codec(format!(
2155                "avro_value_type_mismatch: could not adapt decoded value: {error}"
2156            ))
2157        })
2158    }
2159}
2160
2161impl Serialize for AvroValue {
2162    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
2163    where
2164        S: Serializer,
2165    {
2166        match self {
2167            Self::Null => serializer.serialize_unit(),
2168            Self::Boolean(value) => serializer.serialize_bool(*value),
2169            Self::Long(value) => serializer.serialize_i64(*value),
2170            Self::Double(value) => serializer.serialize_f64(*value),
2171            Self::Bytes(value) => serializer.serialize_bytes(value),
2172            Self::String(value) => serializer.serialize_str(value),
2173            Self::Array(values) => {
2174                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2175                for value in values {
2176                    sequence.serialize_element(value)?;
2177                }
2178                sequence.end()
2179            }
2180            Self::Map(values) => {
2181                let mut map = serializer.serialize_map(Some(values.len()))?;
2182                for (key, value) in values {
2183                    map.serialize_entry(key, value)?;
2184                }
2185                map.end()
2186            }
2187        }
2188    }
2189}
2190
2191pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2192    let datum = avro_value_to_datum(value)?;
2193    let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2194        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2195    let mut bytes = Vec::with_capacity(datum.len() + 10);
2196    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2197    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2198    bytes.extend_from_slice(&datum);
2199    Ok(PayloadEnvelope {
2200        codec: DEFAULT_CODEC.to_string(),
2201        blob: BASE64.encode(bytes),
2202    })
2203}
2204
2205pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2206    if envelope.codec != DEFAULT_CODEC {
2207        return Err(unsupported_payload_codec(&envelope.codec));
2208    }
2209    decode_avro_value_blob(&envelope.blob)
2210}
2211
2212pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2213    let blob = match codec {
2214        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2215        other => return Err(unsupported_payload_codec(other)),
2216    };
2217
2218    Ok(PayloadEnvelope {
2219        codec: codec.to_string(),
2220        blob,
2221    })
2222}
2223
2224pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2225    match envelope.codec.as_str() {
2226        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2227        other => Err(unsupported_payload_codec(other)),
2228    }
2229}
2230
2231fn handler_type_error<T>(
2232    handler_kind: HandlerKind,
2233    handler_name: &str,
2234    value_kind: HandlerValueKind,
2235    message: impl Into<String>,
2236) -> Error {
2237    Error::HandlerType {
2238        handler_kind,
2239        handler_name: handler_name.to_string(),
2240        value_kind,
2241        rust_type: type_name::<T>(),
2242        message: message.into(),
2243    }
2244}
2245
2246fn decode_handler_input<T: DeserializeOwned>(
2247    arguments: AvroValue,
2248    handler_kind: HandlerKind,
2249    handler_name: &str,
2250) -> Result<T> {
2251    let argument = match arguments {
2252        AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2253            arguments.pop().expect("one typed handler argument")
2254        }
2255        AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2256        AvroValue::Array(arguments) => {
2257            return Err(handler_type_error::<T>(
2258                handler_kind,
2259                handler_name,
2260                HandlerValueKind::Input,
2261                format!(
2262                    "typed handlers accept one request value, but the task carried {} arguments",
2263                    arguments.len()
2264                ),
2265            ));
2266        }
2267        argument => argument,
2268    };
2269
2270    argument.deserialize().map_err(|error| {
2271        handler_type_error::<T>(
2272            handler_kind,
2273            handler_name,
2274            HandlerValueKind::Input,
2275            error.to_string(),
2276        )
2277    })
2278}
2279
2280fn encode_handler_result<T: Serialize>(
2281    result: &T,
2282    handler_kind: HandlerKind,
2283    handler_name: &str,
2284) -> Result<AvroValue> {
2285    AvroValue::from_serialize(result).map_err(|error| {
2286        handler_type_error::<T>(
2287            handler_kind,
2288            handler_name,
2289            HandlerValueKind::Result,
2290            error.to_string(),
2291        )
2292    })
2293}
2294
2295fn decode_handler_result<T: DeserializeOwned>(
2296    result: AvroValue,
2297    handler_kind: HandlerKind,
2298    handler_name: &str,
2299) -> Result<T> {
2300    result.deserialize().map_err(|error| {
2301        handler_type_error::<T>(
2302            handler_kind,
2303            handler_name,
2304            HandlerValueKind::Result,
2305            error.to_string(),
2306        )
2307    })
2308}
2309
2310#[cfg(test)]
2311fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2312    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2313}
2314
2315fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2316    validate_payload_codec(fallback_codec)?;
2317
2318    if value.is_null() {
2319        return Ok(Value::Null);
2320    }
2321
2322    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2323        return decode_blob(blob, codec);
2324    }
2325
2326    if let Some(blob) = value.as_str() {
2327        return decode_blob(blob, fallback_codec);
2328    }
2329
2330    Err(untagged_payload_value())
2331}
2332
2333fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2334    let envelope = match codec {
2335        DEFAULT_CODEC => encode_avro_value(value)?,
2336        other => return Err(unsupported_payload_codec(other)),
2337    };
2338    Ok(serde_json::to_value(envelope)?)
2339}
2340
2341fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2342    validate_payload_codec(fallback_codec)?;
2343
2344    if value.is_null() {
2345        return Ok(AvroValue::Null);
2346    }
2347
2348    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2349        validate_payload_codec(codec)?;
2350        return decode_avro_value_blob(blob);
2351    }
2352
2353    if let Some(blob) = value.as_str() {
2354        return match fallback_codec {
2355            DEFAULT_CODEC => decode_avro_value_blob(blob),
2356            other => Err(unsupported_payload_codec(other)),
2357        };
2358    }
2359
2360    Err(untagged_payload_value())
2361}
2362
2363fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2364    match value {
2365        AvroValue::Null => AvroValue::Array(Vec::new()),
2366        AvroValue::Array(_) => value,
2367        other => AvroValue::Array(vec![other]),
2368    }
2369}
2370
2371fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2372    match codec {
2373        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2374        other => Err(unsupported_payload_codec(other)),
2375    }
2376}
2377
2378fn validate_payload_codec(codec: &str) -> Result<()> {
2379    match codec {
2380        DEFAULT_CODEC => Ok(()),
2381        MISSING_TASK_PAYLOAD_CODEC => {
2382            Err(invalid_task_payload_codec("task payload_codec is missing"))
2383        }
2384        NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2385        NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2386            "task payload_codec must be a string",
2387        )),
2388        other => Err(unsupported_payload_codec(other)),
2389    }
2390}
2391
2392fn invalid_task_payload_codec(reason: &str) -> Error {
2393    Error::Codec(format!(
2394        "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2395    ))
2396}
2397
2398fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2399    let Some(object) = value.as_object() else {
2400        return Ok(None);
2401    };
2402    if !object.contains_key("codec") && !object.contains_key("blob") {
2403        return Ok(None);
2404    }
2405
2406    let codec = object
2407        .get("codec")
2408        .and_then(Value::as_str)
2409        .ok_or_else(invalid_payload_envelope)?;
2410    validate_payload_codec(codec)?;
2411    let blob = object
2412        .get("blob")
2413        .and_then(Value::as_str)
2414        .ok_or_else(invalid_payload_envelope)?;
2415    Ok(Some((codec, blob)))
2416}
2417
2418fn invalid_payload_envelope() -> Error {
2419    Error::Codec(
2420        "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2421            .to_string(),
2422    )
2423}
2424
2425fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2426    for command in commands {
2427        let Some(command) = command.as_object() else {
2428            continue;
2429        };
2430        let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2431            continue;
2432        };
2433        let Some(payload_field) = workflow_command_payload_field(command_type) else {
2434            continue;
2435        };
2436
2437        if let Some(codec) = command.get("payload_codec") {
2438            let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2439            validate_payload_codec(codec)?;
2440        }
2441
2442        let payload = command
2443            .get(payload_field)
2444            .ok_or_else(invalid_payload_envelope)?;
2445        if runtime_payloads::Reference::parse(payload)?.is_none() {
2446            validate_outbound_payload_envelope(payload)?;
2447        }
2448    }
2449    Ok(())
2450}
2451
2452fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2453    if commands.iter().any(|command| {
2454        command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2455            && command
2456                .get("condition_wait_occurrence_id")
2457                .and_then(Value::as_str)
2458                .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2459    }) {
2460        CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2461    } else if commands.iter().any(|command| {
2462        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2463            && command.get("attribute_types").is_some()
2464    }) {
2465        TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2466    } else if commands
2467        .iter()
2468        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2469    {
2470        MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2471    } else if commands
2472        .iter()
2473        .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2474    {
2475        CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2476    } else if commands.iter().any(|command| {
2477        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2478    }) {
2479        SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2480    } else {
2481        WORKER_PROTOCOL_VERSION
2482    }
2483}
2484
2485fn workflow_completion_protocol_version_with_message_streams(
2486    commands: &[Value],
2487    has_message_stream_metadata: bool,
2488) -> &'static str {
2489    let command_protocol = workflow_completion_protocol_version(commands);
2490    if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2491        MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2492    } else {
2493        command_protocol
2494    }
2495}
2496
2497fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2498    match command_type {
2499        "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2500        "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2501        "start_service_operation" => Some("request_payload"),
2502        "upsert_memo" => Some("entries"),
2503        _ => None,
2504    }
2505}
2506
2507fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2508    let Some((codec, blob)) = payload_envelope_parts(value)? else {
2509        return Err(untagged_payload_value());
2510    };
2511    validate_payload_codec(codec)?;
2512    decode_avro_value_blob(blob)?;
2513    Ok(())
2514}
2515
2516fn unsupported_payload_codec(codec: &str) -> Error {
2517    Error::Codec(format!(
2518        "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2519    ))
2520}
2521
2522fn untagged_payload_value() -> Error {
2523    Error::Codec(
2524        "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2525            .to_string(),
2526    )
2527}
2528
2529fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2530    let bytes = BASE64.decode(blob).map_err(|err| {
2531        Error::Codec(format!(
2532            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2533        ))
2534    })?;
2535
2536    if serde_json::from_slice::<Value>(&bytes).is_ok() {
2537        return Err(unsupported_payload_codec("json"));
2538    }
2539
2540    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2541        return Err(Error::Codec(
2542            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2543        ));
2544    }
2545
2546    let fingerprint: [u8; 8] = bytes[2..10]
2547        .try_into()
2548        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2549    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2550        return Err(Error::Codec(format!(
2551            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2552            fingerprint
2553                .iter()
2554                .map(|byte| format!("{byte:02x}"))
2555                .collect::<String>()
2556        )));
2557    }
2558
2559    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2560    // The current fingerprint selects the current immutable schema, so reader
2561    // resolution would only re-walk the same recursive union. Future retained
2562    // writer fingerprints supply a distinct reader schema in this branch.
2563    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2564    if datum_reader.truncated {
2565        return Err(Error::Codec(
2566            "invalid_payload_framing: truncated Avro Value datum".to_string(),
2567        ));
2568    }
2569    let datum = datum.map_err(|err| {
2570        Error::Codec(format!(
2571            "invalid_payload_framing: malformed Avro Value datum: {err}"
2572        ))
2573    })?;
2574    if datum_reader.remaining() != 0 {
2575        return Err(Error::Codec(format!(
2576            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2577            datum_reader.remaining()
2578        )));
2579    }
2580    avro_value_from_datum(datum)
2581}
2582
2583struct StrictAvroDatumReader<'a> {
2584    bytes: &'a [u8],
2585    offset: usize,
2586    truncated: bool,
2587}
2588
2589impl<'a> StrictAvroDatumReader<'a> {
2590    fn new(bytes: &'a [u8]) -> Self {
2591        Self {
2592            bytes,
2593            offset: 0,
2594            truncated: false,
2595        }
2596    }
2597
2598    fn remaining(&self) -> usize {
2599        self.bytes.len() - self.offset
2600    }
2601}
2602
2603impl Read for StrictAvroDatumReader<'_> {
2604    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2605        let count = buffer.len().min(self.remaining());
2606        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2607        self.offset += count;
2608        if count < buffer.len() {
2609            self.truncated = true;
2610        }
2611
2612        Ok(count)
2613    }
2614}
2615
2616fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2617    let branch = match value {
2618        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2619        AvroValue::Boolean(value) => AvroDatum::Union(
2620            1,
2621            Box::new(AvroDatum::Record(vec![(
2622                "boolean".to_string(),
2623                AvroDatum::Boolean(*value),
2624            )])),
2625        ),
2626        AvroValue::Long(value) => AvroDatum::Union(
2627            2,
2628            Box::new(AvroDatum::Record(vec![(
2629                "long".to_string(),
2630                AvroDatum::Long(*value),
2631            )])),
2632        ),
2633        AvroValue::Double(value) => {
2634            if !value.is_finite() {
2635                return Err(Error::Codec(
2636                    "non_finite_float: Avro Value doubles must be finite".to_string(),
2637                ));
2638            }
2639            AvroDatum::Union(
2640                3,
2641                Box::new(AvroDatum::Record(vec![(
2642                    "double".to_string(),
2643                    AvroDatum::Double(*value),
2644                )])),
2645            )
2646        }
2647        AvroValue::Bytes(value) => AvroDatum::Union(
2648            4,
2649            Box::new(AvroDatum::Record(vec![(
2650                "bytes".to_string(),
2651                AvroDatum::Bytes(value.clone()),
2652            )])),
2653        ),
2654        AvroValue::String(value) => AvroDatum::Union(
2655            5,
2656            Box::new(AvroDatum::Record(vec![(
2657                "string".to_string(),
2658                AvroDatum::String(value.clone()),
2659            )])),
2660        ),
2661        AvroValue::Array(values) => AvroDatum::Union(
2662            6,
2663            Box::new(AvroDatum::Record(vec![(
2664                "items".to_string(),
2665                AvroDatum::Array(
2666                    values
2667                        .iter()
2668                        .map(avro_value_to_datum)
2669                        .collect::<Result<Vec<_>>>()?,
2670                ),
2671            )])),
2672        ),
2673        AvroValue::Map(values) => AvroDatum::Union(
2674            7,
2675            Box::new(AvroDatum::Record(vec![(
2676                "entries".to_string(),
2677                AvroDatum::Array(
2678                    values
2679                        .iter()
2680                        .map(|(key, value)| {
2681                            Ok(AvroDatum::Record(vec![
2682                                ("key".to_string(), AvroDatum::String(key.clone())),
2683                                ("value".to_string(), avro_value_to_datum(value)?),
2684                            ]))
2685                        })
2686                        .collect::<Result<Vec<_>>>()?,
2687                ),
2688            )])),
2689        ),
2690    };
2691    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2692}
2693
2694fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2695    let AvroDatum::Record(mut outer) = datum else {
2696        return Err(Error::Codec(
2697            "invalid_payload_framing: datum is not a Value record".to_string(),
2698        ));
2699    };
2700    let (_, branch) = outer
2701        .pop()
2702        .filter(|(name, _)| name == "value")
2703        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2704    let AvroDatum::Union(_, branch) = branch else {
2705        return Err(Error::Codec(
2706            "invalid_payload_framing: invalid Value union".to_string(),
2707        ));
2708    };
2709    match *branch {
2710        AvroDatum::Null => Ok(AvroValue::Null),
2711        AvroDatum::Record(mut fields) => {
2712            let (name, value) = fields.pop().ok_or_else(|| {
2713                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2714            })?;
2715            match (name.as_str(), value) {
2716                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2717                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2718                ("double", AvroDatum::Double(value)) if value.is_finite() => {
2719                    Ok(AvroValue::Double(value))
2720                }
2721                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2722                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2723                ("items", AvroDatum::Array(values)) => values
2724                    .into_iter()
2725                    .map(avro_value_from_datum)
2726                    .collect::<Result<Vec<_>>>()
2727                    .map(AvroValue::Array),
2728                ("entries", AvroDatum::Map(values)) => values
2729                    .into_iter()
2730                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2731                    .collect::<Result<BTreeMap<_, _>>>()
2732                    .map(AvroValue::Map),
2733                _ => Err(Error::Codec(
2734                    "invalid_payload_framing: unknown Value branch".to_string(),
2735                )),
2736            }
2737        }
2738        _ => Err(Error::Codec(
2739            "invalid_payload_framing: invalid Value branch".to_string(),
2740        )),
2741    }
2742}
2743
2744fn avro_value_schema() -> Result<&'static Schema> {
2745    match AVRO_VALUE_SCHEMA.get_or_init(|| {
2746        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2747            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2748    }) {
2749        Ok(schema) => Ok(schema),
2750        Err(message) => Err(Error::Codec(message.clone())),
2751    }
2752}
2753
2754fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2755    match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2756        // Apache Avro's Value::Map uses a randomized HashMap. Arrays and maps
2757        // have the same block representation when each ordered array record
2758        // contains the map key followed by its value, so this encoding-only
2759        // adaptation lets the official encoder retain BTreeMap wire order.
2760        let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2761            .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2762        let entries_schema = schema
2763            .pointer_mut("/fields/0/type/7/fields/0/type")
2764            .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2765        if *entries_schema != json!({"type": "map", "values": "Value"}) {
2766            return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2767        }
2768        *entries_schema = json!({
2769            "type": "array",
2770            "items": {
2771                "type": "record",
2772                "name": "MapEntry",
2773                "fields": [
2774                    {"name": "key", "type": "string"},
2775                    {"name": "value", "type": "Value"}
2776                ]
2777            }
2778        });
2779        Schema::parse_str(&schema.to_string())
2780            .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2781    }) {
2782        Ok(schema) => Ok(schema),
2783        Err(message) => Err(Error::Codec(message.clone())),
2784    }
2785}
2786
2787#[derive(Clone, Debug)]
2788pub struct Client {
2789    http: reqwest::Client,
2790    base_url: String,
2791    token: Option<String>,
2792    control_token: Option<String>,
2793    worker_token: Option<String>,
2794    namespace: String,
2795    max_external_payload_bytes: usize,
2796    worker_storage_admission: Option<WorkerStorageAdmission>,
2797    cooperative_worker_protocol: bool,
2798    local_activities_enabled: bool,
2799    worker_sessions_enabled: bool,
2800    max_concurrent_worker_sessions: usize,
2801    worker_build_id: Option<String>,
2802    sticky_cache: Option<Arc<Mutex<sticky_workflow_cache::StickyWorkflowCache>>>,
2803    runtime_upload_policy: Arc<Mutex<runtime_uploads::PolicyCache>>,
2804}
2805
2806impl Client {
2807    pub fn new(base_url: impl Into<String>) -> Result<Self> {
2808        Self::builder(base_url).build()
2809    }
2810
2811    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2812        ClientBuilder {
2813            base_url: base_url.into(),
2814            token: None,
2815            control_token: None,
2816            worker_token: None,
2817            namespace: "default".to_string(),
2818            timeout: Duration::from_secs(60),
2819            max_external_payload_bytes: 64 * 1024 * 1024,
2820        }
2821    }
2822
2823    pub async fn health(&self) -> Result<Value> {
2824        self.request_json(
2825            reqwest::Method::GET,
2826            "/health",
2827            RequestProtocol::ControlPlane,
2828            Option::<&Value>::None,
2829        )
2830        .await
2831    }
2832
2833    pub async fn cluster_info(&self) -> Result<Value> {
2834        self.request_json(
2835            reqwest::Method::GET,
2836            "/cluster/info",
2837            RequestProtocol::ControlPlane,
2838            Option::<&Value>::None,
2839        )
2840        .await
2841    }
2842
2843    pub async fn start_workflow<T: Serialize>(
2844        &self,
2845        workflow_type: &str,
2846        task_queue: &str,
2847        workflow_id: &str,
2848        input: T,
2849    ) -> Result<WorkflowHandle> {
2850        self.start_workflow_with_options(
2851            workflow_type,
2852            task_queue,
2853            workflow_id,
2854            WorkflowStartOptions::default(),
2855            input,
2856        )
2857        .await
2858    }
2859
2860    /// Start a workflow with explicit server-enforced execution and run
2861    /// deadlines.
2862    pub async fn start_workflow_with_options<T: Serialize>(
2863        &self,
2864        workflow_type: &str,
2865        task_queue: &str,
2866        workflow_id: &str,
2867        options: WorkflowStartOptions,
2868        input: T,
2869    ) -> Result<WorkflowHandle> {
2870        options.validate()?;
2871        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2872        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2873        let body = json!({
2874            "workflow_id": workflow_id,
2875            "workflow_type": workflow_type,
2876            "task_queue": task_queue,
2877            "input": input_envelope,
2878            "execution_timeout_seconds": options.execution_timeout_seconds,
2879            "run_timeout_seconds": options.run_timeout_seconds
2880        });
2881
2882        let data: Value = self
2883            .request_json(
2884                reqwest::Method::POST,
2885                "/workflows",
2886                RequestProtocol::ControlPlane,
2887                Some(&body),
2888            )
2889            .await?;
2890
2891        Ok(WorkflowHandle {
2892            client: self.clone(),
2893            workflow_id: data
2894                .get("workflow_id")
2895                .and_then(Value::as_str)
2896                .unwrap_or(workflow_id)
2897                .to_string(),
2898            run_id: data
2899                .get("run_id")
2900                .and_then(Value::as_str)
2901                .map(str::to_string),
2902            workflow_type: data
2903                .get("workflow_type")
2904                .and_then(Value::as_str)
2905                .unwrap_or(workflow_type)
2906                .to_string(),
2907        })
2908    }
2909
2910    pub async fn signal_workflow<T: Serialize>(
2911        &self,
2912        workflow_id: &str,
2913        signal_name: &str,
2914        input: T,
2915    ) -> Result<Value> {
2916        self.signal_workflow_target(workflow_id, None, signal_name, input)
2917            .await
2918    }
2919
2920    /// Append one idempotently identified item to an instance-scoped input stream.
2921    pub async fn append_message_stream<T: Serialize>(
2922        &self,
2923        workflow_id: &str,
2924        stream_name: &str,
2925        message_id: &str,
2926        input: T,
2927    ) -> Result<Value> {
2928        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2929        let body = json!({
2930            "message_id": message_id,
2931            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2932        });
2933        self.request_json(
2934            reqwest::Method::POST,
2935            &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2936            RequestProtocol::ControlPlane,
2937            Some(&body),
2938        )
2939        .await
2940    }
2941
2942    /// Signal only if `run_id` is still the current run for this instance.
2943    pub async fn signal_workflow_run<T: Serialize>(
2944        &self,
2945        workflow_id: &str,
2946        run_id: &str,
2947        signal_name: &str,
2948        input: T,
2949    ) -> Result<Value> {
2950        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2951            .await
2952    }
2953
2954    async fn signal_workflow_target<T: Serialize>(
2955        &self,
2956        workflow_id: &str,
2957        run_id: Option<&str>,
2958        signal_name: &str,
2959        input: T,
2960    ) -> Result<Value> {
2961        validate_user_signal_name(signal_name)?;
2962        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2963        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2964        let body = json!({
2965            "input": input_envelope
2966        });
2967        let path = match run_id {
2968            Some(run_id) => {
2969                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2970            }
2971            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2972        };
2973        self.request_json(
2974            reqwest::Method::POST,
2975            &path,
2976            RequestProtocol::ControlPlane,
2977            Some(&body),
2978        )
2979        .await
2980    }
2981
2982    /// Close the current run as cancelled immediately.
2983    ///
2984    /// Server revokes open tasks and timers without resuming workflow code for
2985    /// cleanup. Use [`Client::request_workflow_cancellation`] for a separate cooperative
2986    /// request on a supporting runtime and opted-in worker.
2987    pub async fn cancel_workflow(
2988        &self,
2989        workflow_id: &str,
2990        options: WorkflowCommandOptions,
2991    ) -> Result<WorkflowCommandResult> {
2992        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2993            .await
2994    }
2995
2996    /// Close the selected run as cancelled, only if `run_id` is still current.
2997    pub async fn cancel_workflow_run(
2998        &self,
2999        workflow_id: &str,
3000        run_id: &str,
3001        options: WorkflowCommandOptions,
3002    ) -> Result<WorkflowCommandResult> {
3003        self.workflow_command(
3004            workflow_id,
3005            Some(run_id),
3006            WorkflowCommandKind::Cancel,
3007            options,
3008        )
3009        .await
3010    }
3011
3012    /// Forcefully terminate the current run for an instance.
3013    pub async fn terminate_workflow(
3014        &self,
3015        workflow_id: &str,
3016        options: WorkflowCommandOptions,
3017    ) -> Result<WorkflowCommandResult> {
3018        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
3019            .await
3020    }
3021
3022    /// Forcefully terminate only if `run_id` is still current.
3023    pub async fn terminate_workflow_run(
3024        &self,
3025        workflow_id: &str,
3026        run_id: &str,
3027        options: WorkflowCommandOptions,
3028    ) -> Result<WorkflowCommandResult> {
3029        self.workflow_command(
3030            workflow_id,
3031            Some(run_id),
3032            WorkflowCommandKind::Terminate,
3033            options,
3034        )
3035        .await
3036    }
3037
3038    /// Continue a failed run from its recorded activity failure boundary.
3039    pub async fn redrive_workflow_run(
3040        &self,
3041        workflow_id: &str,
3042        failed_run_id: &str,
3043        request_id: Option<&str>,
3044    ) -> Result<WorkflowRedriveResult> {
3045        let body = request_id
3046            .map(|id| json!({"request_id": id}))
3047            .unwrap_or_else(|| json!({}));
3048        let data: Value = self
3049            .request_json(
3050                reqwest::Method::POST,
3051                &format!("/workflows/{workflow_id}/runs/{failed_run_id}/redrive"),
3052                RequestProtocol::ControlPlane,
3053                Some(&body),
3054            )
3055            .await?;
3056        if data.get("command_status").and_then(Value::as_str) != Some("accepted")
3057            || data.get("outcome").and_then(Value::as_str) != Some("redriven")
3058        {
3059            return Err(Error::Codec(
3060                "redrive response was not accepted".to_string(),
3061            ));
3062        }
3063        if data.get("workflow_id").and_then(Value::as_str) != Some(workflow_id)
3064            || data.get("continued_from_run_id").and_then(Value::as_str) != Some(failed_run_id)
3065        {
3066            return Err(Error::Codec(
3067                "redrive response does not match the requested source run".to_string(),
3068            ));
3069        }
3070        let run_id = data
3071            .get("run_id")
3072            .and_then(Value::as_str)
3073            .filter(|id| !id.is_empty() && *id != failed_run_id)
3074            .ok_or_else(|| {
3075                Error::Codec("redrive response is missing a distinct successor run_id".to_string())
3076            })?;
3077        let resume_step_sequence = data
3078            .get("resume_step_sequence")
3079            .and_then(Value::as_u64)
3080            .filter(|sequence| *sequence > 0)
3081            .ok_or_else(|| {
3082                Error::Codec("redrive response is missing resume_step_sequence".to_string())
3083            })?;
3084        Ok(WorkflowRedriveResult {
3085            workflow_id: workflow_id.to_string(),
3086            source_run_id: failed_run_id.to_string(),
3087            run_id: run_id.to_string(),
3088            resume_step_sequence,
3089            raw: data,
3090        })
3091    }
3092
3093    async fn workflow_command(
3094        &self,
3095        workflow_id: &str,
3096        run_id: Option<&str>,
3097        command: WorkflowCommandKind,
3098        options: WorkflowCommandOptions,
3099    ) -> Result<WorkflowCommandResult> {
3100        let path = match run_id {
3101            Some(run_id) => format!(
3102                "/workflows/{workflow_id}/runs/{run_id}/{}",
3103                command.as_str()
3104            ),
3105            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
3106        };
3107        let data = match self
3108            .request_json(
3109                reqwest::Method::POST,
3110                &path,
3111                RequestProtocol::ControlPlane,
3112                Some(&options),
3113            )
3114            .await
3115        {
3116            Ok(data) => data,
3117            Err(Error::Http { status, body }) => {
3118                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
3119                    command,
3120                    status,
3121                    body,
3122                    workflow_id,
3123                    run_id,
3124                )));
3125            }
3126            Err(error) => return Err(error),
3127        };
3128
3129        Ok(workflow_command_result(command, data, workflow_id, run_id))
3130    }
3131
3132    /// Execute a named, read-only query against a running or completed workflow.
3133    ///
3134    /// Arguments and results use the platform payload envelope. Server and
3135    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
3136    /// reason, HTTP status, and original response body.
3137    pub async fn query_workflow<T: Serialize>(
3138        &self,
3139        workflow_id: &str,
3140        query_name: &str,
3141        input: T,
3142    ) -> Result<Value> {
3143        self.query_workflow_target(workflow_id, None, query_name, input)
3144            .await
3145    }
3146
3147    /// Query only if `run_id` is still current, preventing accidental retargeting.
3148    pub async fn query_workflow_run<T: Serialize>(
3149        &self,
3150        workflow_id: &str,
3151        run_id: &str,
3152        query_name: &str,
3153        input: T,
3154    ) -> Result<Value> {
3155        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
3156            .await
3157    }
3158
3159    /// Query a workflow and return the lossless fixed Avro Value result.
3160    pub async fn query_workflow_avro_value<T: Serialize>(
3161        &self,
3162        workflow_id: &str,
3163        query_name: &str,
3164        input: T,
3165    ) -> Result<AvroValue> {
3166        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
3167            .await
3168    }
3169
3170    /// Query a selected run and return the lossless fixed Avro Value result.
3171    pub async fn query_workflow_run_avro_value<T: Serialize>(
3172        &self,
3173        workflow_id: &str,
3174        run_id: &str,
3175        query_name: &str,
3176        input: T,
3177    ) -> Result<AvroValue> {
3178        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
3179            .await
3180    }
3181
3182    async fn query_workflow_avro_value_target<T: Serialize>(
3183        &self,
3184        workflow_id: &str,
3185        run_id: Option<&str>,
3186        query_name: &str,
3187        input: T,
3188    ) -> Result<AvroValue> {
3189        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3190        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
3191        let path = match run_id {
3192            Some(run_id) => {
3193                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3194            }
3195            None => format!("/workflows/{workflow_id}/query/{query_name}"),
3196        };
3197        let response: Value = match self
3198            .request_json(
3199                reqwest::Method::POST,
3200                &path,
3201                RequestProtocol::ControlPlane,
3202                Some(&body),
3203            )
3204            .await
3205        {
3206            Ok(response) => response,
3207            Err(Error::Http { status, body }) => {
3208                return Err(Error::QueryFailed(query_failure(status, body)));
3209            }
3210            Err(error) => return Err(error),
3211        };
3212
3213        let envelope = response
3214            .get("result_envelope")
3215            .filter(|envelope| !envelope.is_null())
3216            .ok_or_else(|| {
3217                Error::Codec(
3218                    "missing_payload_envelope: typed query result requires result_envelope"
3219                        .to_string(),
3220                )
3221            })?;
3222        decode_wire_avro_value(envelope, DEFAULT_CODEC)
3223    }
3224
3225    async fn query_workflow_target<T: Serialize>(
3226        &self,
3227        workflow_id: &str,
3228        run_id: Option<&str>,
3229        query_name: &str,
3230        input: T,
3231    ) -> Result<Value> {
3232        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3233        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
3234        let body = json!({
3235            "input": input_envelope
3236        });
3237        let path = match run_id {
3238            Some(run_id) => {
3239                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3240            }
3241            None => format!("/workflows/{workflow_id}/query/{query_name}"),
3242        };
3243        let response: Value = match self
3244            .request_json(
3245                reqwest::Method::POST,
3246                &path,
3247                RequestProtocol::ControlPlane,
3248                Some(&body),
3249            )
3250            .await
3251        {
3252            Ok(response) => response,
3253            Err(Error::Http { status, body }) => {
3254                return Err(Error::QueryFailed(query_failure(status, body)));
3255            }
3256            Err(error) => return Err(error),
3257        };
3258
3259        if let Some(envelope) = response
3260            .get("result_envelope")
3261            .filter(|envelope| !envelope.is_null())
3262        {
3263            return decode_wire_value(envelope, DEFAULT_CODEC);
3264        }
3265
3266        Ok(response.get("result").cloned().unwrap_or(Value::Null))
3267    }
3268
3269    /// Send a synchronous update using fixed Avro Value arguments.
3270    pub async fn update_workflow<T: Serialize>(
3271        &self,
3272        workflow_id: &str,
3273        update_name: &str,
3274        input: T,
3275        request_id: Option<&str>,
3276    ) -> Result<Value> {
3277        let response = self
3278            .update_workflow_response(workflow_id, update_name, input, request_id)
3279            .await?;
3280        if let Some(envelope) = response
3281            .get("result_envelope")
3282            .filter(|envelope| !envelope.is_null())
3283        {
3284            return decode_wire_value(envelope, DEFAULT_CODEC);
3285        }
3286        Ok(response.get("result").cloned().unwrap_or(response))
3287    }
3288
3289    /// Send a synchronous update and retain a bytes-capable Avro result.
3290    pub async fn update_workflow_avro_value<T: Serialize>(
3291        &self,
3292        workflow_id: &str,
3293        update_name: &str,
3294        input: T,
3295        request_id: Option<&str>,
3296    ) -> Result<AvroValue> {
3297        let response = self
3298            .update_workflow_response(workflow_id, update_name, input, request_id)
3299            .await?;
3300        let envelope = response
3301            .get("result_envelope")
3302            .filter(|envelope| !envelope.is_null())
3303            .ok_or_else(|| {
3304                Error::Codec(
3305                    "missing_payload_envelope: typed update result requires result_envelope"
3306                        .to_string(),
3307                )
3308            })?;
3309        decode_wire_avro_value(envelope, DEFAULT_CODEC)
3310    }
3311
3312    async fn update_workflow_response<T: Serialize>(
3313        &self,
3314        workflow_id: &str,
3315        update_name: &str,
3316        input: T,
3317        request_id: Option<&str>,
3318    ) -> Result<Value> {
3319        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3320        let mut body = json!({
3321            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3322            "wait_for": "completed",
3323        });
3324        if let Some(request_id) = request_id {
3325            body["request_id"] = json!(request_id);
3326        }
3327        self.request_json(
3328            reqwest::Method::POST,
3329            &format!("/workflows/{workflow_id}/update/{update_name}"),
3330            RequestProtocol::ControlPlane,
3331            Some(&body),
3332        )
3333        .await
3334    }
3335
3336    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3337        let path = format!("/workflows/{workflow_id}");
3338        let mut data: WorkflowDescription = self
3339            .request_json(
3340                reqwest::Method::GET,
3341                &path,
3342                RequestProtocol::ControlPlane,
3343                Option::<&Value>::None,
3344            )
3345            .await?;
3346        data.decode_payloads()?;
3347        Ok(data)
3348    }
3349
3350    /// Describe one selected run, including historical terminal runs.
3351    pub async fn describe_workflow_run(
3352        &self,
3353        workflow_id: &str,
3354        run_id: &str,
3355    ) -> Result<WorkflowDescription> {
3356        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3357        let mut data: WorkflowDescription = self
3358            .request_json(
3359                reqwest::Method::GET,
3360                &path,
3361                RequestProtocol::ControlPlane,
3362                Option::<&Value>::None,
3363            )
3364            .await?;
3365        data.decode_payloads()?;
3366        Ok(data)
3367    }
3368
3369    fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3370        let mut path = format!(
3371            "/workflows/{}/runs/{}/streams",
3372            percent_encode_path_segment(workflow_id),
3373            percent_encode_path_segment(run_id),
3374        );
3375        if let Some(stream_name) = stream_name {
3376            path.push('/');
3377            path.push_str(&percent_encode_path_segment(stream_name));
3378        }
3379        path
3380    }
3381
3382    /// List the run-scoped output streams already opened by a workflow.
3383    pub async fn list_workflow_streams(
3384        &self,
3385        workflow_id: &str,
3386        run_id: &str,
3387    ) -> Result<Vec<WorkflowStreamDescription>> {
3388        let response: WorkflowStreamListResponse = self
3389            .request_json(
3390                reqwest::Method::GET,
3391                &Self::workflow_stream_path(workflow_id, run_id, None),
3392                RequestProtocol::ControlPlane,
3393                Option::<&Value>::None,
3394            )
3395            .await?;
3396        Ok(response.streams)
3397    }
3398
3399    /// Describe stream lifecycle, offsets, pending count, and terminal error.
3400    pub async fn describe_workflow_stream(
3401        &self,
3402        workflow_id: &str,
3403        run_id: &str,
3404        stream_name: &str,
3405    ) -> Result<WorkflowStreamDescription> {
3406        let response: WorkflowStreamDescriptionResponse = self
3407            .request_json(
3408                reqwest::Method::GET,
3409                &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3410                RequestProtocol::ControlPlane,
3411                Option::<&Value>::None,
3412            )
3413            .await?;
3414        Ok(response.stream)
3415    }
3416
3417    /// Read one bounded page beginning at a zero-based offset.
3418    ///
3419    /// Delivery is at least once: persist `next_offset` only after processing
3420    /// the page. The future is cancellation-safe; dropping it cancels the
3421    /// in-flight request. Long polling is capped at 60 seconds by the SDK and
3422    /// service contract.
3423    pub async fn subscribe_workflow_stream(
3424        &self,
3425        workflow_id: &str,
3426        run_id: &str,
3427        stream_name: &str,
3428        from_offset: u64,
3429        max_items: usize,
3430        wait: Duration,
3431    ) -> Result<WorkflowStreamPage> {
3432        let max_items = max_items.clamp(1, 500);
3433        let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3434        let path = format!(
3435            "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3436            Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3437        );
3438        let response: WorkflowStreamPageResponse = self
3439            .request_json_with_timeout(
3440                reqwest::Method::GET,
3441                &path,
3442                RequestProtocol::ControlPlane,
3443                Option::<&Value>::None,
3444                Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3445            )
3446            .await?;
3447
3448        let items = response
3449            .items
3450            .into_iter()
3451            .map(|raw| {
3452                let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3453                let envelope = raw.get("payload").cloned();
3454                let payload = envelope
3455                    .as_ref()
3456                    .filter(|value| value.get("blob").is_some())
3457                    .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3458                    .transpose()?
3459                    .map(AvroValue::into_json)
3460                    .transpose()?;
3461                Ok(WorkflowStreamItem {
3462                    offset,
3463                    payload,
3464                    payload_envelope: envelope,
3465                    payload_reference: raw
3466                        .get("payload_reference")
3467                        .and_then(Value::as_str)
3468                        .map(str::to_string),
3469                    payload_codec: raw
3470                        .get("payload_codec")
3471                        .and_then(Value::as_str)
3472                        .map(str::to_string),
3473                    idempotency_key: raw
3474                        .get("idempotency_key")
3475                        .and_then(Value::as_str)
3476                        .map(str::to_string),
3477                    item_type: raw
3478                        .get("item_type")
3479                        .and_then(Value::as_str)
3480                        .map(str::to_string),
3481                    content_type: raw
3482                        .get("content_type")
3483                        .and_then(Value::as_str)
3484                        .map(str::to_string),
3485                    origin: raw
3486                        .get("origin")
3487                        .and_then(Value::as_str)
3488                        .map(str::to_string),
3489                    origin_reference: raw
3490                        .get("origin_reference")
3491                        .and_then(Value::as_str)
3492                        .map(str::to_string),
3493                    emitted_at: raw
3494                        .get("emitted_at")
3495                        .and_then(Value::as_str)
3496                        .map(str::to_string),
3497                    raw,
3498                })
3499            })
3500            .collect::<Result<Vec<_>>>()?;
3501        Ok(WorkflowStreamPage {
3502            stream: response.stream,
3503            items,
3504            next_offset: response.next_offset,
3505            terminal: response.terminal,
3506        })
3507    }
3508
3509    /// Append inline Avro envelopes or opaque external payload references.
3510    pub async fn append_workflow_stream(
3511        &self,
3512        workflow_id: &str,
3513        run_id: &str,
3514        stream_name: &str,
3515        items: &[WorkflowStreamAppendItem],
3516        max_pending_items: Option<u64>,
3517    ) -> Result<WorkflowStreamAppendResult> {
3518        if items.is_empty() {
3519            return Err(Error::Codec(
3520                "workflow_stream_items_empty: append requires at least one item".to_string(),
3521            ));
3522        }
3523        let mut body = json!({
3524            "items": items
3525                .iter()
3526                .map(|item| item.wire_value(None))
3527                .collect::<Vec<_>>(),
3528        });
3529        if let Some(max_pending_items) = max_pending_items {
3530            if max_pending_items == 0 {
3531                return Err(Error::Codec(
3532                    "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3533                        .to_string(),
3534                ));
3535            }
3536            body["max_pending_items"] = json!(max_pending_items);
3537        }
3538        let response: WorkflowStreamAppendResponse = self
3539            .request_json(
3540                reqwest::Method::POST,
3541                &format!(
3542                    "{}/items",
3543                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3544                ),
3545                RequestProtocol::ControlPlane,
3546                Some(&body),
3547            )
3548            .await?;
3549        Ok(WorkflowStreamAppendResult {
3550            stream: response.stream,
3551            accepted_offsets: response.accepted_offsets,
3552            accepted: response.accepted,
3553            deduped: response.deduped,
3554        })
3555    }
3556
3557    /// Close a stream, or mark it errored when `error_reason` is supplied.
3558    pub async fn close_workflow_stream(
3559        &self,
3560        workflow_id: &str,
3561        run_id: &str,
3562        stream_name: &str,
3563        error_reason: Option<&str>,
3564        retention_seconds: Option<u64>,
3565    ) -> Result<WorkflowStreamDescription> {
3566        let mut body = json!({});
3567        if let Some(error_reason) = error_reason {
3568            body["error_reason"] = json!(error_reason);
3569        }
3570        if let Some(retention_seconds) = retention_seconds {
3571            if retention_seconds == 0 {
3572                return Err(Error::Codec(
3573                    "workflow_stream_retention_invalid: retention_seconds must be positive"
3574                        .to_string(),
3575                ));
3576            }
3577            body["retention_seconds"] = json!(retention_seconds);
3578        }
3579        let response: WorkflowStreamDescriptionResponse = self
3580            .request_json(
3581                reqwest::Method::POST,
3582                &format!(
3583                    "{}/close",
3584                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3585                ),
3586                RequestProtocol::ControlPlane,
3587                Some(&body),
3588            )
3589            .await?;
3590        Ok(response.stream)
3591    }
3592
3593    pub async fn register_worker(
3594        &self,
3595        worker_id: &str,
3596        task_queue: &str,
3597        supported_workflow_types: Vec<String>,
3598        supported_activity_types: Vec<String>,
3599        max_concurrent_workflow_tasks: usize,
3600        max_concurrent_activity_tasks: usize,
3601    ) -> Result<RegisterWorkerResponse> {
3602        self.register_worker_with_capabilities(
3603            worker_id,
3604            task_queue,
3605            supported_workflow_types,
3606            supported_activity_types,
3607            max_concurrent_workflow_tasks,
3608            max_concurrent_activity_tasks,
3609            Vec::new(),
3610        )
3611        .await
3612    }
3613
3614    /// Register a worker and explicitly advertise additive worker capabilities.
3615    pub async fn register_worker_with_capabilities(
3616        &self,
3617        worker_id: &str,
3618        task_queue: &str,
3619        supported_workflow_types: Vec<String>,
3620        supported_activity_types: Vec<String>,
3621        max_concurrent_workflow_tasks: usize,
3622        max_concurrent_activity_tasks: usize,
3623        capabilities: Vec<String>,
3624    ) -> Result<RegisterWorkerResponse> {
3625        self.register_worker_with_command_contracts(
3626            worker_id,
3627            task_queue,
3628            supported_workflow_types,
3629            supported_activity_types,
3630            max_concurrent_workflow_tasks,
3631            max_concurrent_activity_tasks,
3632            capabilities,
3633            Value::Object(serde_json::Map::new()),
3634        )
3635        .await
3636    }
3637
3638    /// Register a worker and advertise its named query and update handlers.
3639    ///
3640    /// This Rust SDK cannot execute synchronous pre-accept update validation,
3641    /// so a workflow contract with a non-empty or malformed
3642    /// `update_validators` declaration returns
3643    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
3644    #[allow(clippy::too_many_arguments)]
3645    pub async fn register_worker_with_command_contracts(
3646        &self,
3647        worker_id: &str,
3648        task_queue: &str,
3649        supported_workflow_types: Vec<String>,
3650        supported_activity_types: Vec<String>,
3651        max_concurrent_workflow_tasks: usize,
3652        max_concurrent_activity_tasks: usize,
3653        capabilities: Vec<String>,
3654        workflow_command_contracts: Value,
3655    ) -> Result<RegisterWorkerResponse> {
3656        self.register_worker_with_definition_fingerprints(
3657            worker_id,
3658            task_queue,
3659            supported_workflow_types,
3660            supported_activity_types,
3661            max_concurrent_workflow_tasks,
3662            max_concurrent_activity_tasks,
3663            capabilities,
3664            workflow_command_contracts,
3665            None,
3666        )
3667        .await
3668    }
3669
3670    #[allow(clippy::too_many_arguments)]
3671    async fn register_worker_with_definition_fingerprints(
3672        &self,
3673        worker_id: &str,
3674        task_queue: &str,
3675        supported_workflow_types: Vec<String>,
3676        supported_activity_types: Vec<String>,
3677        max_concurrent_workflow_tasks: usize,
3678        max_concurrent_activity_tasks: usize,
3679        capabilities: Vec<String>,
3680        workflow_command_contracts: Value,
3681        workflow_definition_fingerprints: Option<BTreeMap<String, String>>,
3682    ) -> Result<RegisterWorkerResponse> {
3683        if let Some(contracts) = workflow_command_contracts.as_object() {
3684            for (workflow_type, contract) in contracts {
3685                let Some(update_validators) = contract.get("update_validators") else {
3686                    continue;
3687                };
3688                if !update_validators
3689                    .as_array()
3690                    .is_some_and(|validators| validators.is_empty())
3691                {
3692                    return Err(Error::UnsupportedUpdateValidators {
3693                        workflow_type: workflow_type.clone(),
3694                    });
3695                }
3696            }
3697        }
3698
3699        let mut body = json!({
3700            "worker_id": worker_id,
3701            "task_queue": task_queue,
3702            "runtime": "rust",
3703            "sdk_version": SDK_VERSION,
3704            "supported_workflow_types": supported_workflow_types,
3705            "supported_activity_types": supported_activity_types,
3706            "capabilities": capabilities,
3707            "capability_manifest": portable_worker_affinity_capability_manifest(),
3708            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3709            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3710        });
3711        if let Some(build_id) = &self.worker_build_id {
3712            if build_id.is_empty() || build_id.len() > 255 {
3713                return Err(Error::WorkerLoop(
3714                    "worker build ID must contain 1 to 255 bytes".into(),
3715                ));
3716            }
3717            body["build_id"] = json!(build_id);
3718        }
3719        if self.sticky_cache.is_some() {
3720            body["capability_manifest"]["sticky_execution"] = json!({"supported":true,
3721                "minimum_protocol_version":PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
3722                "implementation":"bounded_durable_history_cache"});
3723        }
3724        if self.local_activities_enabled {
3725            body["capability_manifest"]["local_activities"] = json!({"supported":true,
3726                "minimum_protocol_version":PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION});
3727        }
3728        if self.worker_sessions_enabled {
3729            body["capability_manifest"]["worker_sessions"] = json!({"supported":true,
3730                "minimum_protocol_version":PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION});
3731            body["max_concurrent_worker_sessions"] = json!(self.max_concurrent_worker_sessions);
3732        }
3733        if workflow_command_contracts
3734            .as_object()
3735            .is_some_and(|contracts| !contracts.is_empty())
3736        {
3737            body["workflow_command_contracts"] = workflow_command_contracts;
3738        }
3739        if let Some(fingerprints) = workflow_definition_fingerprints {
3740            body["workflow_definition_fingerprints"] = json!(fingerprints);
3741        }
3742
3743        let response: Value = self
3744            .request_json(
3745                reqwest::Method::POST,
3746                "/worker/register",
3747                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3748                Some(&body),
3749            )
3750            .await?;
3751        if self.sticky_cache.is_some() {
3752            self.confirm_sticky_registration(&response, worker_id, task_queue)
3753                .await?;
3754        }
3755        if self.local_activities_enabled || self.worker_sessions_enabled {
3756            let compatible = response["registered"].as_bool() == Some(true)
3757                && response["worker_id"].as_str() == Some(worker_id)
3758                && response["namespace"].as_str() == Some(self.namespace.as_str())
3759                && response["task_queue"].as_str() == Some(task_queue)
3760                && response["protocol_version"]
3761                    .as_str()
3762                    .and_then(|version| version.strip_prefix("1."))
3763                    .and_then(|minor| minor.parse::<u64>().ok())
3764                    .is_some_and(|minor| minor >= 18)
3765                && (!self.local_activities_enabled
3766                    || response["capability_manifest"]["local_activities"]["supported"].as_bool()
3767                        == Some(true))
3768                && (!self.worker_sessions_enabled
3769                    || (response["capability_manifest"]["worker_sessions"]["supported"].as_bool()
3770                        == Some(true)
3771                        && response["capabilities"].as_array().is_some_and(|accepted| {
3772                            capabilities.iter().all(|capability| {
3773                                accepted
3774                                    .iter()
3775                                    .any(|value| value.as_str() == Some(capability.as_str()))
3776                            })
3777                        })));
3778            if !compatible {
3779                let error = Error::WorkerLoop(if self.worker_sessions_enabled {
3780                    "worker_session_registration_unconfirmed: Server must acknowledge this worker, namespace, queue and session capabilities".into()
3781                } else {
3782                    "local_activity_registration_unconfirmed: Server must acknowledge this worker, namespace, queue and local capability".into()
3783                });
3784                if response["registered"] == true
3785                    && response["worker_id"].as_str() == Some(worker_id)
3786                {
3787                    if let Err(deregistration) =
3788                        self.deregister_worker_registration(worker_id).await
3789                    {
3790                        return Err(Error::WorkerShutdown {
3791                            primary: Box::new(error),
3792                            deregistration: Box::new(deregistration),
3793                        });
3794                    }
3795                }
3796                return Err(error);
3797            }
3798        }
3799        if self.cooperative_worker_protocol {
3800            let compatible = response["registered"].as_bool() == Some(true)
3801                && response["worker_id"].as_str() == Some(worker_id)
3802                && response["namespace"].as_str() == Some(self.namespace.as_str())
3803                && response["task_queue"].as_str() == Some(task_queue)
3804                && response["protocol_version"]
3805                    .as_str()
3806                    .is_some_and(cooperative_cancellation::supports_protocol)
3807                && response["server_capabilities"]["cooperative_cancellation"].as_bool()
3808                    == Some(true)
3809                && response["capabilities"].as_array().is_some_and(|accepted| {
3810                    capabilities.iter().all(|capability| {
3811                        accepted
3812                            .iter()
3813                            .any(|value| value.as_str() == Some(capability.as_str()))
3814                    })
3815                });
3816            if !compatible {
3817                let error = Error::CooperativeCancellationUnavailable(
3818                    "registration must acknowledge this namespace, queue, worker, capabilities and compatible runtime protocol 1.20".into(),
3819                );
3820                if response["registered"].as_bool() == Some(true)
3821                    && response["worker_id"].as_str() == Some(worker_id)
3822                {
3823                    if let Err(deregistration) =
3824                        self.deregister_worker_registration(worker_id).await
3825                    {
3826                        return Err(Error::WorkerShutdown {
3827                            primary: Box::new(error),
3828                            deregistration: Box::new(deregistration),
3829                        });
3830                    }
3831                }
3832                return Err(error);
3833            }
3834        }
3835        Ok(serde_json::from_value(response)?)
3836    }
3837
3838    /// Gracefully remove one worker's registration through the worker plane.
3839    ///
3840    /// This operation is separate from operator-facing worker management. It
3841    /// uses worker-protocol authentication and returns the server's lease
3842    /// recovery result.
3843    pub async fn deregister_worker_registration(
3844        &self,
3845        worker_id: &str,
3846    ) -> Result<WorkerDeregistrationEnvelope> {
3847        let path = format!(
3848            "/worker/registrations/{}",
3849            percent_encode_path_segment(worker_id)
3850        );
3851        self.request_json(
3852            reqwest::Method::DELETE,
3853            &path,
3854            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3855            Option::<&Value>::None,
3856        )
3857        .await
3858    }
3859
3860    /// Long-poll for an ephemeral, read-only workflow query task.
3861    pub async fn poll_query_task(
3862        &self,
3863        worker_id: &str,
3864        task_queue: &str,
3865        timeout: Duration,
3866    ) -> Result<Option<QueryTask>> {
3867        Ok(self
3868            .poll_query_task_response(worker_id, task_queue, timeout)
3869            .await?
3870            .task)
3871    }
3872
3873    /// Poll a query task while preserving server stop and drain metadata.
3874    pub async fn poll_query_task_response(
3875        &self,
3876        worker_id: &str,
3877        task_queue: &str,
3878        timeout: Duration,
3879    ) -> Result<PollQueryTaskResponse> {
3880        let poll_request_id = unique_request_id("rust-query-poll");
3881        self.poll_query_task_response_with_request_id(
3882            worker_id,
3883            task_queue,
3884            timeout,
3885            &poll_request_id,
3886            1,
3887        )
3888        .await
3889    }
3890
3891    async fn poll_query_task_response_with_request_id(
3892        &self,
3893        worker_id: &str,
3894        task_queue: &str,
3895        timeout: Duration,
3896        poll_request_id: &str,
3897        transport_retries: usize,
3898    ) -> Result<PollQueryTaskResponse> {
3899        let timeout_seconds = long_poll_timeout_seconds(timeout);
3900        let body = json!({
3901            "worker_id": worker_id,
3902            "task_queue": task_queue,
3903            "poll_request_id": poll_request_id,
3904            "timeout_seconds": timeout_seconds,
3905        });
3906        self.poll_request_json(
3907            "/worker/query-tasks/poll",
3908            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3909            &body,
3910            timeout + Duration::from_secs(5),
3911            transport_retries,
3912        )
3913        .await
3914    }
3915
3916    /// Complete a query task without appending workflow history.
3917    pub async fn complete_query_task<T: Serialize>(
3918        &self,
3919        query_task_id: &str,
3920        lease_owner: &str,
3921        query_task_attempt: u64,
3922        result: T,
3923        codec: &str,
3924    ) -> Result<Value> {
3925        let typed_result = AvroValue::from_serialize(&result)?;
3926        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3927        self.complete_query_task_with_envelope(
3928            query_task_id,
3929            lease_owner,
3930            query_task_attempt,
3931            typed_result.into_json()?,
3932            result_envelope,
3933        )
3934        .await
3935    }
3936
3937    async fn complete_query_task_with_envelope(
3938        &self,
3939        query_task_id: &str,
3940        lease_owner: &str,
3941        query_task_attempt: u64,
3942        result: Value,
3943        result_envelope: Value,
3944    ) -> Result<Value> {
3945        let body = json!({
3946            "lease_owner": lease_owner,
3947            "query_task_attempt": query_task_attempt,
3948            "result": result,
3949            "result_envelope": result_envelope,
3950        });
3951        let path = format!("/worker/query-tasks/{query_task_id}/complete");
3952        let response = self
3953            .request_json(
3954                reqwest::Method::POST,
3955                &path,
3956                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3957                Some(&body),
3958            )
3959            .await;
3960        query_task_response(response)
3961    }
3962
3963    /// Report a stable machine-readable query-task failure.
3964    pub async fn fail_query_task(
3965        &self,
3966        query_task_id: &str,
3967        lease_owner: &str,
3968        query_task_attempt: u64,
3969        message: impl Into<String>,
3970        reason: impl Into<String>,
3971        failure_type: impl Into<String>,
3972    ) -> Result<Value> {
3973        let body = json!({
3974            "lease_owner": lease_owner,
3975            "query_task_attempt": query_task_attempt,
3976            "failure": {
3977                "message": message.into(),
3978                "reason": reason.into(),
3979                "type": failure_type.into(),
3980            }
3981        });
3982        let path = format!("/worker/query-tasks/{query_task_id}/fail");
3983        let response = self
3984            .request_json(
3985                reqwest::Method::POST,
3986                &path,
3987                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3988                Some(&body),
3989            )
3990            .await;
3991        query_task_response(response)
3992    }
3993
3994    pub async fn heartbeat_worker(
3995        &self,
3996        worker_id: &str,
3997        workflow_available: usize,
3998        activity_available: usize,
3999    ) -> Result<Value> {
4000        self.heartbeat_worker_with_sessions(worker_id, workflow_available, activity_available, None)
4001            .await
4002    }
4003
4004    async fn heartbeat_worker_with_sessions(
4005        &self,
4006        worker_id: &str,
4007        workflow_available: usize,
4008        activity_available: usize,
4009        session_available: Option<usize>,
4010    ) -> Result<Value> {
4011        let mut body = json!({
4012            "worker_id": worker_id,
4013            "task_slots": {
4014                "workflow_available": workflow_available,
4015                "activity_available": activity_available
4016            },
4017            "process_metrics": {
4018                "process_id": std::process::id(),
4019                "process_uptime_seconds": 0
4020            }
4021        });
4022
4023        if let Some(available) = session_available {
4024            body["task_slots"]["session_available"] = json!(available);
4025        }
4026
4027        self.request_json(
4028            reqwest::Method::POST,
4029            "/worker/heartbeat",
4030            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4031            Some(&body),
4032        )
4033        .await
4034    }
4035
4036    pub async fn poll_workflow_task(
4037        &self,
4038        worker_id: &str,
4039        task_queue: &str,
4040        timeout: Duration,
4041    ) -> Result<Option<WorkflowTask>> {
4042        Ok(self
4043            .poll_workflow_task_response(worker_id, task_queue, timeout)
4044            .await?
4045            .task)
4046    }
4047
4048    pub async fn poll_workflow_task_response(
4049        &self,
4050        worker_id: &str,
4051        task_queue: &str,
4052        timeout: Duration,
4053    ) -> Result<PollWorkflowTaskResponse> {
4054        let poll_request_id = unique_request_id("rust-workflow-poll");
4055        self.poll_workflow_task_response_with_request_id(
4056            worker_id,
4057            task_queue,
4058            timeout,
4059            &poll_request_id,
4060            1,
4061        )
4062        .await
4063    }
4064
4065    async fn poll_workflow_task_response_with_request_id(
4066        &self,
4067        worker_id: &str,
4068        task_queue: &str,
4069        timeout: Duration,
4070        poll_request_id: &str,
4071        transport_retries: usize,
4072    ) -> Result<PollWorkflowTaskResponse> {
4073        let body = json!({
4074            "worker_id": worker_id,
4075            "task_queue": task_queue,
4076            "poll_request_id": poll_request_id,
4077            "timeout_seconds": long_poll_timeout_seconds(timeout),
4078            "history_page_size": WORKFLOW_HISTORY_PAGE_SIZE,
4079        });
4080        let mut data: PollWorkflowTaskResponse = self
4081            .poll_request_json(
4082                "/worker/workflow-tasks/poll",
4083                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4084                &body,
4085                timeout + Duration::from_secs(5),
4086                transport_retries,
4087            )
4088            .await?;
4089
4090        if let Some(task) = data.task.as_mut() {
4091            self.fetch_remaining_workflow_history(worker_id, task)
4092                .await?;
4093        }
4094
4095        Ok(data)
4096    }
4097
4098    async fn fetch_remaining_workflow_history(
4099        &self,
4100        worker_id: &str,
4101        task: &mut WorkflowTask,
4102    ) -> Result<()> {
4103        let mut next_token = task.next_history_page_token.clone();
4104
4105        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
4106            let lease_owner = task
4107                .lease_owner
4108                .clone()
4109                .unwrap_or_else(|| worker_id.to_string());
4110            let page = self
4111                .workflow_task_history_page(
4112                    &task.task_id,
4113                    &lease_owner,
4114                    task.workflow_task_attempt,
4115                    &token,
4116                )
4117                .await?;
4118
4119            task.append_history_page(page);
4120
4121            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
4122                return Err(Error::Codec(
4123                    "workflow history pagination returned the same page token".to_string(),
4124                ));
4125            }
4126
4127            next_token = task.next_history_page_token.clone();
4128        }
4129
4130        Ok(())
4131    }
4132
4133    async fn workflow_task_history_page(
4134        &self,
4135        task_id: &str,
4136        lease_owner: &str,
4137        workflow_task_attempt: u64,
4138        next_history_page_token: &str,
4139    ) -> Result<WorkflowTaskHistoryPage> {
4140        let body = json!({
4141            "lease_owner": lease_owner,
4142            "workflow_task_attempt": workflow_task_attempt,
4143            "next_history_page_token": next_history_page_token
4144        });
4145        let path = format!("/worker/workflow-tasks/{task_id}/history");
4146
4147        self.request_json(
4148            reqwest::Method::POST,
4149            &path,
4150            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4151            Some(&body),
4152        )
4153        .await
4154    }
4155
4156    pub async fn complete_workflow_task(
4157        &self,
4158        task_id: &str,
4159        lease_owner: &str,
4160        workflow_task_attempt: u64,
4161        commands: Vec<Value>,
4162    ) -> Result<Value> {
4163        self.complete_workflow_task_with_message_streams(
4164            task_id,
4165            lease_owner,
4166            workflow_task_attempt,
4167            commands,
4168            Vec::new(),
4169            Vec::new(),
4170            None,
4171        )
4172        .await
4173    }
4174
4175    #[allow(clippy::too_many_arguments)]
4176    async fn complete_workflow_task_with_message_streams(
4177        &self,
4178        task_id: &str,
4179        lease_owner: &str,
4180        workflow_task_attempt: u64,
4181        commands: Vec<Value>,
4182        message_stream_cursors: Vec<Value>,
4183        message_stream_waits: Vec<Value>,
4184        sticky_cache: Option<Value>,
4185    ) -> Result<Value> {
4186        validate_workflow_task_commands(&commands)?;
4187        let has_message_stream_metadata =
4188            !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
4189        if has_message_stream_metadata
4190            && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
4191        {
4192            return Err(Error::Codec(
4193                "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
4194                    .to_string(),
4195            ));
4196        }
4197        let protocol_version = workflow_completion_protocol_version_with_message_streams(
4198            &commands,
4199            has_message_stream_metadata,
4200        );
4201        let protocol_version = if sticky_cache.is_some()
4202            && protocol_version
4203                .strip_prefix("1.")
4204                .and_then(|minor| minor.parse::<u64>().ok())
4205                .is_some_and(|minor| minor < 18)
4206        {
4207            PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION
4208        } else {
4209            protocol_version
4210        };
4211        let mut body = json!({
4212            "lease_owner": lease_owner,
4213            "workflow_task_attempt": workflow_task_attempt,
4214            "commands": commands
4215        });
4216        if !message_stream_cursors.is_empty() {
4217            body["message_stream_cursors"] = Value::Array(message_stream_cursors);
4218        }
4219        if !message_stream_waits.is_empty() {
4220            body["message_stream_waits"] = Value::Array(message_stream_waits);
4221        }
4222        if let Some(claim) = sticky_cache {
4223            body["sticky_cache"] = claim;
4224        }
4225        let path = format!("/worker/workflow-tasks/{task_id}/complete");
4226        self.request_json(
4227            reqwest::Method::POST,
4228            &path,
4229            RequestProtocol::Worker(protocol_version),
4230            Some(&body),
4231        )
4232        .await
4233    }
4234
4235    pub async fn fail_workflow_task(
4236        &self,
4237        task_id: &str,
4238        lease_owner: &str,
4239        workflow_task_attempt: u64,
4240        message: impl Into<String>,
4241    ) -> Result<Value> {
4242        self.fail_workflow_task_with_type(
4243            task_id,
4244            lease_owner,
4245            workflow_task_attempt,
4246            message,
4247            "RustWorkflowTaskFailure",
4248        )
4249        .await
4250    }
4251
4252    async fn fail_workflow_task_with_type(
4253        &self,
4254        task_id: &str,
4255        lease_owner: &str,
4256        workflow_task_attempt: u64,
4257        message: impl Into<String>,
4258        failure_type: &str,
4259    ) -> Result<Value> {
4260        let body = json!({
4261            "lease_owner": lease_owner,
4262            "workflow_task_attempt": workflow_task_attempt,
4263            "failure": {
4264                "message": message.into(),
4265                "type": failure_type
4266            }
4267        });
4268        let path = format!("/worker/workflow-tasks/{task_id}/fail");
4269        self.request_json(
4270            reqwest::Method::POST,
4271            &path,
4272            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4273            Some(&body),
4274        )
4275        .await
4276    }
4277
4278    pub async fn poll_activity_task(
4279        &self,
4280        worker_id: &str,
4281        task_queue: &str,
4282        timeout: Duration,
4283    ) -> Result<Option<ActivityTask>> {
4284        Ok(self
4285            .poll_activity_task_response(worker_id, task_queue, timeout)
4286            .await?
4287            .task)
4288    }
4289
4290    /// Poll an activity task while preserving server stop and drain metadata.
4291    pub async fn poll_activity_task_response(
4292        &self,
4293        worker_id: &str,
4294        task_queue: &str,
4295        timeout: Duration,
4296    ) -> Result<PollActivityTaskResponse> {
4297        let poll_request_id = unique_request_id("rust-activity-poll");
4298        self.poll_activity_task_response_with_request_id(
4299            worker_id,
4300            task_queue,
4301            timeout,
4302            &poll_request_id,
4303            1,
4304        )
4305        .await
4306        .map(worker_session::SessionPollResponse::ordinary)
4307    }
4308
4309    async fn poll_activity_task_response_with_request_id(
4310        &self,
4311        worker_id: &str,
4312        task_queue: &str,
4313        timeout: Duration,
4314        poll_request_id: &str,
4315        transport_retries: usize,
4316    ) -> Result<worker_session::SessionPollResponse> {
4317        let body = json!({
4318            "worker_id": worker_id,
4319            "task_queue": task_queue,
4320            "poll_request_id": poll_request_id,
4321            "timeout_seconds": long_poll_timeout_seconds(timeout),
4322        });
4323        let data: worker_session::SessionPollResponse = self
4324            .poll_request_json(
4325                "/worker/activity-tasks/poll",
4326                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4327                &body,
4328                timeout + Duration::from_secs(5),
4329                transport_retries,
4330            )
4331            .await?;
4332        Ok(data)
4333    }
4334
4335    pub async fn complete_activity_task<T: Serialize>(
4336        &self,
4337        task_id: &str,
4338        activity_attempt_id: &str,
4339        lease_owner: &str,
4340        result: T,
4341        codec: &str,
4342    ) -> Result<Value> {
4343        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
4344        let body = json!({
4345            "activity_attempt_id": activity_attempt_id,
4346            "lease_owner": lease_owner,
4347            "result": result
4348        });
4349        let path = format!("/worker/activity-tasks/{task_id}/complete");
4350        activity_task_response(
4351            self.request_json(
4352                reqwest::Method::POST,
4353                &path,
4354                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4355                Some(&body),
4356            )
4357            .await,
4358            "complete",
4359            task_id,
4360            activity_attempt_id,
4361        )
4362    }
4363
4364    pub async fn fail_activity_task(
4365        &self,
4366        task_id: &str,
4367        activity_attempt_id: &str,
4368        lease_owner: &str,
4369        message: impl Into<String>,
4370        non_retryable: bool,
4371    ) -> Result<Value> {
4372        let body = json!({
4373            "activity_attempt_id": activity_attempt_id,
4374            "lease_owner": lease_owner,
4375            "failure": {
4376                "message": message.into(),
4377                "type": "RustActivityFailure",
4378                "non_retryable": non_retryable
4379            }
4380        });
4381        let path = format!("/worker/activity-tasks/{task_id}/fail");
4382        activity_task_response(
4383            self.request_json(
4384                reqwest::Method::POST,
4385                &path,
4386                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4387                Some(&body),
4388            )
4389            .await,
4390            "fail",
4391            task_id,
4392            activity_attempt_id,
4393        )
4394    }
4395
4396    /// Record a flat map of scalar/null activity progress metadata.
4397    ///
4398    /// Heartbeat details use the Server's bounded JSON progress contract.
4399    /// They are not workflow payload envelopes.
4400    pub async fn heartbeat_activity_task<T: Serialize>(
4401        &self,
4402        task_id: &str,
4403        activity_attempt_id: &str,
4404        lease_owner: &str,
4405        details: T,
4406    ) -> Result<ActivityHeartbeatResponse> {
4407        serde_json::from_value(
4408            self.heartbeat_activity_task_value(task_id, activity_attempt_id, lease_owner, details)
4409                .await?,
4410        )
4411        .map_err(Error::from)
4412    }
4413
4414    async fn heartbeat_activity_task_value<T: Serialize>(
4415        &self,
4416        task_id: &str,
4417        activity_attempt_id: &str,
4418        lease_owner: &str,
4419        details: T,
4420    ) -> Result<Value> {
4421        let details = AvroValue::from_serialize(&details)?.into_json()?;
4422        let body = json!({
4423            "activity_attempt_id": activity_attempt_id,
4424            "lease_owner": lease_owner,
4425            "details": details
4426        });
4427        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
4428        activity_task_response(
4429            self.request_json(
4430                reqwest::Method::POST,
4431                &path,
4432                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4433                Some(&body),
4434            )
4435            .await,
4436            "heartbeat",
4437            task_id,
4438            activity_attempt_id,
4439        )
4440    }
4441
4442    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4443        &self,
4444        method: reqwest::Method,
4445        path: &str,
4446        protocol: RequestProtocol,
4447        body: Option<&B>,
4448    ) -> Result<T> {
4449        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4450            .await
4451    }
4452
4453    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4454        &self,
4455        method: reqwest::Method,
4456        path: &str,
4457        protocol: RequestProtocol,
4458        body: Option<&B>,
4459        timeout: Duration,
4460    ) -> Result<T> {
4461        let protocol = match protocol {
4462            RequestProtocol::Worker(version)
4463                if self.cooperative_worker_protocol
4464                    && version
4465                        .strip_prefix("1.")
4466                        .and_then(|minor| minor.parse::<u64>().ok())
4467                        .is_some_and(|minor| minor < 20) =>
4468            {
4469                RequestProtocol::Worker("1.20")
4470            }
4471            protocol => protocol,
4472        };
4473        let auth_token = self.auth_token(protocol)?;
4474        let mut request = self
4475            .http
4476            .request(method.clone(), format!("{}/api{}", self.base_url, path))
4477            .timeout(timeout)
4478            .header(reqwest::header::ACCEPT, "application/json")
4479            .header(reqwest::header::CONTENT_TYPE, "application/json")
4480            .header("X-Namespace", &self.namespace);
4481
4482        match protocol {
4483            RequestProtocol::Worker(version) => {
4484                request = request.header("X-Durable-Workflow-Protocol-Version", version);
4485            }
4486            RequestProtocol::ControlPlane => {
4487                request = request.header(
4488                    "X-Durable-Workflow-Control-Plane-Version",
4489                    CONTROL_PLANE_VERSION,
4490                );
4491            }
4492        }
4493
4494        if let Some(token) = auth_token {
4495            request = request.bearer_auth(token);
4496        }
4497
4498        if let Some(body) = body {
4499            let mut body = serde_json::to_value(body)?;
4500            if matches!(
4501                method,
4502                reqwest::Method::POST | reqwest::Method::PUT | reqwest::Method::PATCH
4503            ) {
4504                self.externalize_runtime_payloads(&mut body, path, protocol)
4505                    .await?;
4506            }
4507            request = request.json(&body);
4508        }
4509
4510        let request = request.build()?;
4511        let request_body = request
4512            .body()
4513            .and_then(reqwest::Body::as_bytes)
4514            .and_then(|body| serde_json::from_slice::<Value>(body).ok());
4515        let poll_request_id = path.ends_with("/poll").then(|| {
4516            request_body
4517                .as_ref()
4518                .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4519                .unwrap_or_default()
4520        });
4521        let mut storage_retries = 0_usize;
4522
4523        loop {
4524            // Keep the serialized body and lease/poll identity across admission pauses.
4525            let response = self
4526                .http
4527                .execute(request.try_clone().ok_or_else(|| {
4528                    Error::WorkerLoop("worker request body cannot be retried".to_string())
4529                })?)
4530                .await?;
4531            let status = response.status();
4532            let bytes = response.bytes().await?;
4533
4534            if !status.is_success() {
4535                let body = String::from_utf8_lossy(&bytes).to_string();
4536                if let Some(protocol) = protocol_failure(status, &body) {
4537                    return Err(Error::Protocol(protocol));
4538                }
4539                let error = Error::Http { status, body };
4540                if self
4541                    .wait_for_storage_admission(
4542                        &error,
4543                        protocol,
4544                        Some(path),
4545                        poll_request_id.as_deref(),
4546                        request_body.as_ref(),
4547                        &mut storage_retries,
4548                    )
4549                    .await
4550                {
4551                    continue;
4552                }
4553                return Err(error);
4554            }
4555
4556            if bytes.is_empty() {
4557                return Ok(serde_json::from_value(Value::Null)?);
4558            }
4559
4560            let mut value: Value = serde_json::from_slice(&bytes)?;
4561            self.resolve_runtime_payloads(&mut value, path, protocol)
4562                .await?;
4563            return Ok(serde_json::from_value(value)?);
4564        }
4565    }
4566
4567    async fn wait_for_storage_admission(
4568        &self,
4569        error: &Error,
4570        protocol: RequestProtocol,
4571        path: Option<&str>,
4572        poll_request_id: Option<&str>,
4573        request_body: Option<&Value>,
4574        retries: &mut usize,
4575    ) -> bool {
4576        let Some(admission) = self
4577            .worker_storage_admission
4578            .as_ref()
4579            .filter(|_| matches!(protocol, RequestProtocol::Worker(_)))
4580        else {
4581            return false;
4582        };
4583        let Some(advertised_delay) = worker_storage_admission_retry_after(error, poll_request_id)
4584            .or_else(|| {
4585                path.and_then(|path| {
4586                    worker_backend_unavailable_retry_after(error, path, poll_request_id)
4587                        .or_else(|| worker_fenced_completion_retry_after(error, path, request_body))
4588                })
4589            })
4590        else {
4591            return false;
4592        };
4593        *retries = retries.saturating_add(1);
4594        let delay = worker_retry_delay(admission.policy, *retries)
4595            .max(advertised_delay)
4596            .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4597        let deadline = tokio::time::Instant::now() + delay;
4598        loop {
4599            if admission.stop.load(Ordering::SeqCst) {
4600                return false;
4601            }
4602            let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
4603            if remaining.is_zero() {
4604                return true;
4605            }
4606            tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4607        }
4608    }
4609
4610    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4611        &self,
4612        path: &str,
4613        protocol: RequestProtocol,
4614        body: &B,
4615        timeout: Duration,
4616        max_retries: usize,
4617    ) -> Result<T> {
4618        let mut retries = 0;
4619
4620        loop {
4621            let response = self
4622                .request_json_with_timeout(
4623                    reqwest::Method::POST,
4624                    path,
4625                    protocol,
4626                    Some(body),
4627                    timeout,
4628                )
4629                .await;
4630
4631            match response {
4632                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4633                response => return worker_poll_response(response),
4634            }
4635        }
4636    }
4637
4638    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4639        match protocol {
4640            RequestProtocol::Worker(_) => {
4641                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4642                    return Ok(Some(token));
4643                }
4644                if self.control_token.is_some() {
4645                    return Err(Error::MissingRoleCredentials {
4646                        role: "worker",
4647                        opposite_role: "control",
4648                    });
4649                }
4650                Ok(None)
4651            }
4652            RequestProtocol::ControlPlane => {
4653                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4654                    return Ok(Some(token));
4655                }
4656                if self.worker_token.is_some() {
4657                    return Err(Error::MissingRoleCredentials {
4658                        role: "control",
4659                        opposite_role: "worker",
4660                    });
4661                }
4662                Ok(None)
4663            }
4664        }
4665    }
4666}
4667
4668fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4669    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4670    let reason = body
4671        .get("reason")
4672        .and_then(Value::as_str)
4673        .unwrap_or("query_rejected")
4674        .to_string();
4675    let message = body
4676        .get("message")
4677        .or_else(|| body.get("error"))
4678        .and_then(Value::as_str)
4679        .unwrap_or("workflow query was rejected")
4680        .to_string();
4681
4682    QueryFailure {
4683        status: status.as_u16(),
4684        reason,
4685        message,
4686        body,
4687    }
4688}
4689
4690fn workflow_command_result(
4691    command: WorkflowCommandKind,
4692    data: Value,
4693    workflow_id: &str,
4694    run_id: Option<&str>,
4695) -> WorkflowCommandResult {
4696    WorkflowCommandResult {
4697        command,
4698        workflow_id: data
4699            .get("workflow_id")
4700            .and_then(Value::as_str)
4701            .unwrap_or(workflow_id)
4702            .to_string(),
4703        run_id: data
4704            .get("run_id")
4705            .and_then(Value::as_str)
4706            .or(run_id)
4707            .map(str::to_string),
4708        outcome: data
4709            .get("outcome")
4710            .and_then(Value::as_str)
4711            .map(str::to_string),
4712        reason: data
4713            .get("reason")
4714            .and_then(Value::as_str)
4715            .map(str::to_string),
4716        command_status: data
4717            .get("command_status")
4718            .and_then(Value::as_str)
4719            .map(str::to_string),
4720        raw: data,
4721    }
4722}
4723
4724fn workflow_command_rejection(
4725    command: WorkflowCommandKind,
4726    status: reqwest::StatusCode,
4727    raw_body: String,
4728    workflow_id: &str,
4729    run_id: Option<&str>,
4730) -> WorkflowCommandRejection {
4731    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4732    WorkflowCommandRejection {
4733        command,
4734        status: status.as_u16(),
4735        reason: body
4736            .get("reason")
4737            .and_then(Value::as_str)
4738            .unwrap_or("workflow_command_rejected")
4739            .to_string(),
4740        message: body
4741            .get("message")
4742            .or_else(|| body.get("error"))
4743            .and_then(Value::as_str)
4744            .unwrap_or("workflow lifecycle command was rejected")
4745            .to_string(),
4746        workflow_id: body
4747            .get("workflow_id")
4748            .and_then(Value::as_str)
4749            .unwrap_or(workflow_id)
4750            .to_string(),
4751        run_id: body
4752            .get("run_id")
4753            .and_then(Value::as_str)
4754            .or(run_id)
4755            .map(str::to_string),
4756        target_scope: body
4757            .get("target_scope")
4758            .and_then(Value::as_str)
4759            .map(str::to_string),
4760        body,
4761    }
4762}
4763
4764fn query_task_response(response: Result<Value>) -> Result<Value> {
4765    match response {
4766        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4767        response => response,
4768    }
4769}
4770
4771fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4772    match response {
4773        Err(Error::Http { status, body })
4774            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4775        {
4776            Ok(serde_json::from_str(&body)?)
4777        }
4778        response => response,
4779    }
4780}
4781
4782fn worker_poll_body_is_stop(body: &str) -> bool {
4783    serde_json::from_str::<Value>(body)
4784        .ok()
4785        .is_some_and(|body| {
4786            worker_poll_is_stop(
4787                body.get("poll_status").and_then(Value::as_str),
4788                body.get("reason").and_then(Value::as_str),
4789            )
4790        })
4791}
4792
4793fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4794    matches!(poll_status, Some("draining" | "stopped"))
4795        || matches!(reason, Some("worker_draining" | "worker_stopped"))
4796}
4797
4798fn query_task_rejection_is_final(error: &Error) -> bool {
4799    matches!(
4800        error,
4801        Error::QueryFailed(failure)
4802            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4803    )
4804}
4805
4806fn activity_task_response<T>(
4807    response: Result<T>,
4808    operation: &str,
4809    task_id: &str,
4810    activity_attempt_id: &str,
4811) -> Result<T> {
4812    match response {
4813        Err(Error::Http { status, body }) => {
4814            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4815            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4816                operation: operation.to_string(),
4817                status: status.as_u16(),
4818                reason: body
4819                    .get("reason")
4820                    .and_then(Value::as_str)
4821                    .unwrap_or("activity_task_rejected")
4822                    .to_string(),
4823                task_id: body
4824                    .get("task_id")
4825                    .and_then(Value::as_str)
4826                    .unwrap_or(task_id)
4827                    .to_string(),
4828                activity_attempt_id: body
4829                    .get("activity_attempt_id")
4830                    .and_then(Value::as_str)
4831                    .unwrap_or(activity_attempt_id)
4832                    .to_string(),
4833                cancel_requested: body
4834                    .get("cancel_requested")
4835                    .and_then(Value::as_bool)
4836                    .unwrap_or(false),
4837                can_continue: body.get("can_continue").and_then(Value::as_bool),
4838                run_closed_reason: body
4839                    .get("run_closed_reason")
4840                    .and_then(Value::as_str)
4841                    .map(str::to_string),
4842                body,
4843            }))
4844        }
4845        response => response,
4846    }
4847}
4848
4849fn activity_task_rejection_is_final(error: &Error) -> bool {
4850    matches!(
4851        error,
4852        Error::ActivityTaskRejected(rejection)
4853            if matches!(
4854                rejection.reason.as_str(),
4855                "run_cancelled"
4856                    | "run_terminated"
4857                    | "attempt_closed"
4858                    | "stale_attempt"
4859                    | "activity_cancelled"
4860                    | "task_cancelled"
4861                    | "run_closed"
4862                    | "activity_not_running"
4863                    | "attempt_not_found"
4864            )
4865    )
4866}
4867
4868fn workflow_task_completion_is_terminal_timeout(
4869    error: &Error,
4870    task_id: &str,
4871    workflow_task_attempt: u64,
4872    run_id: Option<&str>,
4873) -> bool {
4874    let Error::Http { status, body } = error else {
4875        return false;
4876    };
4877    if *status != reqwest::StatusCode::CONFLICT {
4878        return false;
4879    }
4880
4881    let Some(run_id) = run_id else {
4882        return false;
4883    };
4884    let Ok(body) = serde_json::from_str::<Value>(body) else {
4885        return false;
4886    };
4887
4888    body.get("recorded").and_then(Value::as_bool) == Some(false)
4889        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4890        && body.get("run_status").and_then(Value::as_str) == Some("failed")
4891        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4892        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4893        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4894}
4895
4896fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4897    let body: Value = serde_json::from_str(raw_body).ok()?;
4898    let reason = body.get("reason")?.as_str()?;
4899    if !matches!(
4900        reason,
4901        "missing_protocol_version"
4902            | "unsupported_protocol_version"
4903            | "missing_control_plane_version"
4904            | "unsupported_control_plane_version"
4905    ) {
4906        return None;
4907    }
4908
4909    Some(ProtocolFailure {
4910        status: status.as_u16(),
4911        reason: reason.to_string(),
4912        message: body
4913            .get("message")
4914            .or_else(|| body.get("error"))
4915            .and_then(Value::as_str)
4916            .unwrap_or("protocol version rejected")
4917            .to_string(),
4918        supported_version: body
4919            .get("supported_version")
4920            .and_then(Value::as_str)
4921            .map(str::to_string),
4922        requested_version: body
4923            .get("requested_version")
4924            .and_then(Value::as_str)
4925            .map(str::to_string),
4926        body,
4927    })
4928}
4929
4930fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4931    timeout
4932        .as_secs()
4933        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4934        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4935}
4936
4937fn worker_operation_is_retryable(error: &Error) -> bool {
4938    if worker_poll_capacity_retry_after(error).is_some()
4939        || worker_storage_admission_body(error).is_some()
4940        || worker_backend_unavailable_body(error).is_some()
4941        || worker_operation_is_explicitly_non_retryable(error)
4942    {
4943        return false;
4944    }
4945
4946    match error {
4947        Error::Transport(error) => {
4948            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4949        }
4950        Error::Http { status, .. } => {
4951            matches!(
4952                *status,
4953                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4954            ) || status.is_server_error()
4955        }
4956        _ => false,
4957    }
4958}
4959
4960fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4961    let body: Value = match error {
4962        Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4963        Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4964        _ => return None,
4965    };
4966    matches!(
4967        body.get("reason").and_then(Value::as_str),
4968        Some("storage_pressure" | "storage_admission_unavailable")
4969    )
4970    .then_some(body)
4971}
4972
4973fn worker_storage_admission_retry_after(
4974    error: &Error,
4975    poll_request_id: Option<&str>,
4976) -> Option<Duration> {
4977    let Error::Http { status, .. } = error else {
4978        return None;
4979    };
4980    let body = worker_storage_admission_body(error)?;
4981    let delay = body.get("retry_after_seconds")?.as_u64()?;
4982    if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4983        || delay == 0
4984        || body.get("retryable") != Some(&Value::Bool(true))
4985        || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4986        || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4987        || body
4988            .get("request_admitted")
4989            .is_some_and(|admitted| admitted != &Value::Bool(false))
4990    {
4991        return None;
4992    }
4993    match poll_request_id {
4994        Some(id) => {
4995            if id.is_empty()
4996                || body.get("task") != Some(&Value::Null)
4997                || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4998                || body.get("poll_status") != body.get("reason")
4999                || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
5000                || body.get("claim_admitted") != Some(&Value::Bool(false))
5001            {
5002                return None;
5003            }
5004        }
5005        None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
5006        None => {}
5007    }
5008    Some(Duration::from_secs(delay))
5009}
5010
5011fn worker_backend_unavailable_retry_after(
5012    error: &Error,
5013    path: &str,
5014    poll_request_id: Option<&str>,
5015) -> Option<Duration> {
5016    let operation = match path {
5017        "/worker/register" => "register_worker",
5018        "/worker/heartbeat" => "heartbeat_worker",
5019        "/worker/workflow-tasks/poll" => "poll_workflow_task",
5020        "/worker/activity-tasks/poll" => "poll_activity_task",
5021        "/worker/query-tasks/poll" => "poll_query_task",
5022        "/worker/update-validation-tasks/poll" => "poll_update_validation_task",
5023        _ => return None,
5024    };
5025    let body = worker_backend_unavailable_body(error)?;
5026    let delay = body.get("retry_after_seconds")?.as_u64()?;
5027    if delay == 0
5028        || body.get("operation")?.as_str()? != operation
5029        || body.get("outcome")?.as_str()? != "unknown"
5030        || body.get("retryable") != Some(&Value::Bool(true))
5031        || body.get("worker_id")?.as_str()?.is_empty()
5032    {
5033        return None;
5034    }
5035    if let Some(id) = poll_request_id {
5036        if id.is_empty()
5037            || body.get("task") != Some(&Value::Null)
5038            || body.get("poll_status")?.as_str()? != "backend_unavailable"
5039            || body.get("poll_request_id")?.as_str()? != id
5040            || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
5041        {
5042            return None;
5043        }
5044    } else if path.ends_with("/poll") {
5045        return None;
5046    }
5047    Some(Duration::from_secs(delay))
5048}
5049
5050fn worker_fenced_completion_retry_after(
5051    error: &Error,
5052    path: &str,
5053    request_body: Option<&Value>,
5054) -> Option<Duration> {
5055    let (operation, task_id, attempt_field) = if let Some(task_id) = path
5056        .strip_prefix("/worker/workflow-tasks/")
5057        .and_then(|tail| tail.strip_suffix("/complete"))
5058    {
5059        ("complete_workflow_task", task_id, "workflow_task_attempt")
5060    } else if let Some(task_id) = path
5061        .strip_prefix("/worker/activity-tasks/")
5062        .and_then(|tail| tail.strip_suffix("/complete"))
5063    {
5064        ("complete_activity_task", task_id, "activity_attempt_id")
5065    } else {
5066        return None;
5067    };
5068    if task_id.is_empty() || task_id.contains('/') {
5069        return None;
5070    }
5071
5072    let request = request_body?.as_object()?;
5073    let lease_owner = request.get("lease_owner")?.as_str()?;
5074    if lease_owner.is_empty() {
5075        return None;
5076    }
5077    let attempt = request.get(attempt_field)?;
5078    if (attempt_field == "workflow_task_attempt" && attempt.as_u64().filter(|id| *id > 0).is_none())
5079        || (attempt_field == "activity_attempt_id"
5080            && attempt.as_str().filter(|id| !id.is_empty()).is_none())
5081    {
5082        return None;
5083    }
5084
5085    let body = worker_backend_unavailable_body(error)?;
5086    let delay = body.get("retry_after_seconds")?.as_u64()?;
5087    if delay == 0
5088        || body.get("operation")?.as_str()? != operation
5089        || body.get("outcome")?.as_str()? != "unknown"
5090        || body.get("retryable") != Some(&Value::Bool(true))
5091        || body.get("worker_id")?.as_str()? != lease_owner
5092        || body.get("task_queue") != Some(&Value::Null)
5093        || body.get("task_id")?.as_str()? != task_id
5094        || body.get("lease_owner")?.as_str()? != lease_owner
5095        || body.get(attempt_field) != Some(attempt)
5096    {
5097        return None;
5098    }
5099
5100    Some(Duration::from_secs(delay))
5101}
5102
5103fn worker_backend_unavailable_body(error: &Error) -> Option<Value> {
5104    let Error::Http { status, body } = error else {
5105        return None;
5106    };
5107    if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE {
5108        return None;
5109    }
5110    let body: Value = serde_json::from_str(body).ok()?;
5111    (body.get("reason")?.as_str()? == "backend_unavailable").then_some(body)
5112}
5113
5114fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
5115    let Error::Http { body, .. } = error else {
5116        return false;
5117    };
5118
5119    serde_json::from_str::<Value>(body)
5120        .ok()
5121        .and_then(|body| body.get("retryable").and_then(Value::as_bool))
5122        == Some(false)
5123}
5124
5125fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
5126    let Error::Http { status, body } = error else {
5127        return None;
5128    };
5129    if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
5130        return None;
5131    }
5132
5133    let body = serde_json::from_str::<Value>(body).ok()?;
5134    let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
5135        == Some("long_poll_capacity_exhausted")
5136        || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
5137    if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
5138        return None;
5139    }
5140
5141    Some(Duration::from_secs(
5142        body.get("retry_after_seconds")
5143            .and_then(Value::as_u64)
5144            .unwrap_or_default(),
5145    ))
5146}
5147
5148fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
5149    let exponent = retry.saturating_sub(1).min(31) as u32;
5150    policy
5151        .initial_backoff
5152        .saturating_mul(1_u32 << exponent)
5153        .min(policy.max_backoff)
5154}
5155
5156#[derive(Debug)]
5157pub struct ClientBuilder {
5158    base_url: String,
5159    token: Option<String>,
5160    control_token: Option<String>,
5161    worker_token: Option<String>,
5162    namespace: String,
5163    timeout: Duration,
5164    max_external_payload_bytes: usize,
5165}
5166
5167impl ClientBuilder {
5168    pub fn token(mut self, token: Option<String>) -> Self {
5169        self.token = token;
5170        self
5171    }
5172
5173    pub fn control_token(mut self, token: Option<String>) -> Self {
5174        self.control_token = token;
5175        self
5176    }
5177
5178    pub fn worker_token(mut self, token: Option<String>) -> Self {
5179        self.worker_token = token;
5180        self
5181    }
5182
5183    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
5184        self.namespace = namespace.into();
5185        self
5186    }
5187
5188    pub fn timeout(mut self, timeout: Duration) -> Self {
5189        self.timeout = timeout;
5190        self
5191    }
5192
5193    /// Maximum unique external-payload bytes fetched per response (default 64 MiB).
5194    /// References are fetched only from this client's authenticated runtime.
5195    pub fn max_external_payload_bytes(mut self, bytes: usize) -> Self {
5196        self.max_external_payload_bytes = bytes;
5197        self
5198    }
5199
5200    pub fn build(self) -> Result<Client> {
5201        let base_url = self.base_url.trim_end_matches('/').to_string();
5202        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
5203            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
5204            .unwrap_or_else(|_| base_url.ends_with("/api"));
5205
5206        if has_sdk_api_suffix {
5207            return Err(Error::InvalidBaseUrl);
5208        }
5209
5210        Ok(Client {
5211            http: reqwest::Client::builder()
5212                .timeout(self.timeout)
5213                .redirect(reqwest::redirect::Policy::none())
5214                .build()?,
5215            base_url,
5216            token: self.token,
5217            control_token: self.control_token,
5218            worker_token: self.worker_token,
5219            namespace: self.namespace,
5220            max_external_payload_bytes: self.max_external_payload_bytes,
5221            worker_storage_admission: None,
5222            cooperative_worker_protocol: false,
5223            local_activities_enabled: false,
5224            worker_sessions_enabled: false,
5225            max_concurrent_worker_sessions: 10,
5226            worker_build_id: None,
5227            sticky_cache: None,
5228            runtime_upload_policy: Arc::new(Mutex::new([None, None])),
5229        })
5230    }
5231}
5232
5233#[derive(Clone, Debug)]
5234pub struct WorkflowHandle {
5235    client: Client,
5236    pub workflow_id: String,
5237    pub run_id: Option<String>,
5238    pub workflow_type: String,
5239}
5240
5241impl WorkflowHandle {
5242    /// Describe whichever run is current for this stable workflow instance.
5243    pub async fn describe(&self) -> Result<WorkflowDescription> {
5244        self.client.describe_workflow(&self.workflow_id).await
5245    }
5246
5247    /// Describe the run identity originally selected by this handle.
5248    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
5249        let run_id = self.run_id.as_deref().ok_or_else(|| {
5250            Error::Codec("run_id is required for selected-run description".to_string())
5251        })?;
5252        self.client
5253            .describe_workflow_run(&self.workflow_id, run_id)
5254            .await
5255    }
5256
5257    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
5258        self.client
5259            .signal_workflow(&self.workflow_id, signal_name, input)
5260            .await
5261    }
5262
5263    pub async fn append_message<T: Serialize>(
5264        &self,
5265        stream_name: &str,
5266        message_id: &str,
5267        input: T,
5268    ) -> Result<Value> {
5269        self.client
5270            .append_message_stream(&self.workflow_id, stream_name, message_id, input)
5271            .await
5272    }
5273
5274    /// Signal only if this handle's selected run is still current.
5275    pub async fn signal_selected_run<T: Serialize>(
5276        &self,
5277        signal_name: &str,
5278        input: T,
5279    ) -> Result<Value> {
5280        let run_id = self.run_id.as_deref().ok_or_else(|| {
5281            Error::Codec("run_id is required for selected-run signaling".to_string())
5282        })?;
5283        self.client
5284            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
5285            .await
5286    }
5287
5288    /// Close whichever run is current as cancelled, without workflow cleanup.
5289    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
5290        self.client
5291            .cancel_workflow(&self.workflow_id, options)
5292            .await
5293    }
5294
5295    /// Request cancellation only if this handle's selected run is still current.
5296    pub async fn cancel_selected_run(
5297        &self,
5298        options: WorkflowCommandOptions,
5299    ) -> Result<WorkflowCommandResult> {
5300        let run_id = self.run_id.as_deref().ok_or_else(|| {
5301            Error::Codec("run_id is required for selected-run cancellation".to_string())
5302        })?;
5303        self.client
5304            .cancel_workflow_run(&self.workflow_id, run_id, options)
5305            .await
5306    }
5307
5308    /// Forcefully terminate whichever run is current.
5309    pub async fn terminate(
5310        &self,
5311        options: WorkflowCommandOptions,
5312    ) -> Result<WorkflowCommandResult> {
5313        self.client
5314            .terminate_workflow(&self.workflow_id, options)
5315            .await
5316    }
5317
5318    /// Terminate only if this handle's selected run is still current.
5319    pub async fn terminate_selected_run(
5320        &self,
5321        options: WorkflowCommandOptions,
5322    ) -> Result<WorkflowCommandResult> {
5323        let run_id = self.run_id.as_deref().ok_or_else(|| {
5324            Error::Codec("run_id is required for selected-run termination".to_string())
5325        })?;
5326        self.client
5327            .terminate_workflow_run(&self.workflow_id, run_id, options)
5328            .await
5329    }
5330
5331    /// Continue this handle's failed run without repeating completed activities.
5332    pub async fn redrive(&self, request_id: Option<&str>) -> Result<WorkflowRedriveResult> {
5333        let run_id = self
5334            .run_id
5335            .as_deref()
5336            .ok_or_else(|| Error::Codec("run_id is required for redrive".to_string()))?;
5337        self.client
5338            .redrive_workflow_run(&self.workflow_id, run_id, request_id)
5339            .await
5340    }
5341
5342    /// Execute a named, read-only query against this workflow.
5343    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
5344        self.client
5345            .query_workflow(&self.workflow_id, query_name, input)
5346            .await
5347    }
5348
5349    pub async fn query_avro_value<T: Serialize>(
5350        &self,
5351        query_name: &str,
5352        input: T,
5353    ) -> Result<AvroValue> {
5354        self.client
5355            .query_workflow_avro_value(&self.workflow_id, query_name, input)
5356            .await
5357    }
5358
5359    pub async fn update<T: Serialize>(
5360        &self,
5361        update_name: &str,
5362        input: T,
5363        request_id: Option<&str>,
5364    ) -> Result<Value> {
5365        self.client
5366            .update_workflow(&self.workflow_id, update_name, input, request_id)
5367            .await
5368    }
5369
5370    pub async fn update_avro_value<T: Serialize>(
5371        &self,
5372        update_name: &str,
5373        input: T,
5374        request_id: Option<&str>,
5375    ) -> Result<AvroValue> {
5376        self.client
5377            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
5378            .await
5379    }
5380
5381    /// Query only if this handle's selected run is still current.
5382    pub async fn query_selected_run<T: Serialize>(
5383        &self,
5384        query_name: &str,
5385        input: T,
5386    ) -> Result<Value> {
5387        let run_id = self
5388            .run_id
5389            .as_deref()
5390            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
5391        self.client
5392            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
5393            .await
5394    }
5395
5396    /// Await the final terminal outcome of the current continue-as-new chain.
5397    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
5398        self.result_target(options, None).await
5399    }
5400
5401    /// Await the final result without projecting Avro bytes through JSON.
5402    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
5403        self.result_avro_value_target(options, None).await
5404    }
5405
5406    /// Await the final result and decode it into a Serde application type.
5407    pub async fn result_typed<T: DeserializeOwned>(
5408        &self,
5409        options: WorkflowResultOptions,
5410    ) -> Result<T> {
5411        let result = self.result_avro_value(options).await?;
5412        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
5413    }
5414
5415    /// Await only the run identity originally selected by this handle.
5416    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
5417        let run_id = self.run_id.as_deref().ok_or_else(|| {
5418            Error::Codec("run_id is required for selected-run result".to_string())
5419        })?;
5420        self.result_target(options, Some(run_id)).await
5421    }
5422
5423    /// Await the selected run's result on the lossless Avro Value surface.
5424    pub async fn result_selected_run_avro_value(
5425        &self,
5426        options: WorkflowResultOptions,
5427    ) -> Result<AvroValue> {
5428        let run_id = self.run_id.as_deref().ok_or_else(|| {
5429            Error::Codec("run_id is required for selected-run result".to_string())
5430        })?;
5431        self.result_avro_value_target(options, Some(run_id)).await
5432    }
5433
5434    /// Await the selected run and decode its result into a Serde type.
5435    pub async fn result_selected_run_typed<T: DeserializeOwned>(
5436        &self,
5437        options: WorkflowResultOptions,
5438    ) -> Result<T> {
5439        let result = self.result_selected_run_avro_value(options).await?;
5440        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
5441    }
5442
5443    async fn result_avro_value_target(
5444        &self,
5445        options: WorkflowResultOptions,
5446        selected_run_id: Option<&str>,
5447    ) -> Result<AvroValue> {
5448        let started = Instant::now();
5449
5450        loop {
5451            let description = match selected_run_id {
5452                Some(run_id) => {
5453                    self.client
5454                        .describe_workflow_run(&self.workflow_id, run_id)
5455                        .await?
5456                }
5457                None => self.describe().await?,
5458            };
5459            if description.is_completed() {
5460                return description.output_avro_value.ok_or_else(|| {
5461                    Error::Codec(
5462                        "missing_payload_envelope: typed workflow result requires output_envelope"
5463                            .to_string(),
5464                    )
5465                });
5466            }
5467            if description.is_terminal() {
5468                let outcome =
5469                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
5470                return Err(match outcome.kind {
5471                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
5472                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5473                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5474                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5475                });
5476            }
5477            if started.elapsed() >= options.timeout {
5478                return Err(Error::Timeout);
5479            }
5480            tokio::time::sleep(options.poll_interval).await;
5481        }
5482    }
5483
5484    async fn result_target(
5485        &self,
5486        options: WorkflowResultOptions,
5487        selected_run_id: Option<&str>,
5488    ) -> Result<Value> {
5489        let started = Instant::now();
5490
5491        loop {
5492            let description = match selected_run_id {
5493                Some(run_id) => {
5494                    self.client
5495                        .describe_workflow_run(&self.workflow_id, run_id)
5496                        .await?
5497                }
5498                None => self.describe().await?,
5499            };
5500            if description.is_completed() {
5501                return Ok(description.output.unwrap_or(Value::Null));
5502            }
5503
5504            if description.is_terminal() {
5505                let outcome =
5506                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
5507                return Err(match outcome.kind {
5508                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
5509                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5510                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5511                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5512                });
5513            }
5514
5515            if started.elapsed() >= options.timeout {
5516                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
5517                    kind: WorkflowTerminalKind::TimedOut,
5518                    workflow_id: description
5519                        .workflow_id
5520                        .clone()
5521                        .unwrap_or_else(|| self.workflow_id.clone()),
5522                    run_id: description
5523                        .run_id
5524                        .clone()
5525                        .or_else(|| selected_run_id.map(str::to_string)),
5526                    reason: "result_wait_timeout".to_string(),
5527                    failure_category: Some("client_timeout".to_string()),
5528                    failure_id: None,
5529                    exception_type: None,
5530                    exception_class: None,
5531                    non_retryable: None,
5532                    message: Some(format!(
5533                        "workflow result was not terminal within {:?}",
5534                        options.timeout
5535                    )),
5536                    exception: None,
5537                    raw: description.raw_value(),
5538                }));
5539            }
5540
5541            tokio::time::sleep(options.poll_interval).await;
5542        }
5543    }
5544}
5545
5546#[derive(Clone, Copy, Debug)]
5547pub struct WorkflowResultOptions {
5548    pub poll_interval: Duration,
5549    pub timeout: Duration,
5550}
5551
5552impl Default for WorkflowResultOptions {
5553    fn default() -> Self {
5554        Self {
5555            poll_interval: Duration::from_millis(500),
5556            timeout: Duration::from_secs(30),
5557        }
5558    }
5559}
5560
5561#[derive(Clone, Debug, Deserialize)]
5562pub struct WorkflowDescription {
5563    pub workflow_id: Option<String>,
5564    pub run_id: Option<String>,
5565    pub workflow_type: Option<String>,
5566    pub status: Option<String>,
5567    #[serde(default)]
5568    pub closed_reason: Option<String>,
5569    #[serde(default)]
5570    pub error: Option<String>,
5571    #[serde(default)]
5572    pub failure: Option<Value>,
5573    #[serde(default)]
5574    pub exception: Option<Value>,
5575    #[serde(default)]
5576    pub failures: Vec<Value>,
5577    #[serde(default)]
5578    pub output: Option<Value>,
5579    #[serde(default)]
5580    pub output_envelope: Option<Value>,
5581    #[serde(skip)]
5582    pub output_avro_value: Option<AvroValue>,
5583    #[serde(flatten)]
5584    pub raw: HashMap<String, Value>,
5585}
5586
5587/// Lifecycle and backlog metadata for one run-scoped Workflow Stream.
5588#[derive(Clone, Debug, Deserialize)]
5589pub struct WorkflowStreamDescription {
5590    pub stream_name: String,
5591    pub status: String,
5592    pub last_offset: i64,
5593    pub total_items: u64,
5594    pub pending_items: u64,
5595    #[serde(default)]
5596    pub opened_at: Option<String>,
5597    #[serde(default)]
5598    pub last_appended_at: Option<String>,
5599    #[serde(default)]
5600    pub closed_at: Option<String>,
5601    #[serde(default)]
5602    pub error_reason: Option<String>,
5603    #[serde(default)]
5604    pub retention_seconds: Option<u64>,
5605    #[serde(flatten)]
5606    pub raw: HashMap<String, Value>,
5607}
5608
5609impl WorkflowStreamDescription {
5610    pub fn is_terminal(&self) -> bool {
5611        matches!(self.status.as_str(), "closed" | "errored")
5612    }
5613}
5614
5615/// One item for direct or replay-safe append.
5616#[derive(Clone, Debug, Default)]
5617pub struct WorkflowStreamAppendItem {
5618    pub payload_envelope: Option<Value>,
5619    pub payload_reference: Option<String>,
5620    pub item_type: Option<String>,
5621    pub content_type: Option<String>,
5622    pub idempotency_key: Option<String>,
5623}
5624
5625impl WorkflowStreamAppendItem {
5626    /// Encode an inline payload with the SDK's fixed Avro Value envelope.
5627    pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5628        let value = AvroValue::from_serialize(&payload)?;
5629        Ok(Self {
5630            payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5631            ..Self::default()
5632        })
5633    }
5634
5635    /// Preserve an external payload URI as an opaque service-contract reference.
5636    pub fn from_reference(reference: impl Into<String>) -> Self {
5637        Self {
5638            payload_reference: Some(reference.into()),
5639            ..Self::default()
5640        }
5641    }
5642
5643    pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5644        self.item_type = Some(item_type.into());
5645        self
5646    }
5647
5648    pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5649        self.content_type = Some(content_type.into());
5650        self
5651    }
5652
5653    pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5654        self.idempotency_key = Some(idempotency_key.into());
5655        self
5656    }
5657
5658    fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5659        let mut item = serde_json::Map::new();
5660        if let Some(payload) = &self.payload_envelope {
5661            item.insert("payload".to_string(), payload.clone());
5662            item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5663        }
5664        if let Some(reference) = &self.payload_reference {
5665            item.insert("payload_reference".to_string(), json!(reference));
5666        }
5667        if let Some(item_type) = &self.item_type {
5668            item.insert("item_type".to_string(), json!(item_type));
5669        }
5670        if let Some(content_type) = &self.content_type {
5671            item.insert("content_type".to_string(), json!(content_type));
5672        }
5673        if let Some(key) = derived_idempotency_key
5674            .as_ref()
5675            .or(self.idempotency_key.as_ref())
5676        {
5677            item.insert("idempotency_key".to_string(), json!(key));
5678        }
5679        Value::Object(item)
5680    }
5681}
5682
5683/// One durable item at its stable zero-based offset.
5684#[derive(Clone, Debug)]
5685pub struct WorkflowStreamItem {
5686    pub offset: u64,
5687    pub payload: Option<Value>,
5688    pub payload_envelope: Option<Value>,
5689    pub payload_reference: Option<String>,
5690    pub payload_codec: Option<String>,
5691    pub idempotency_key: Option<String>,
5692    pub item_type: Option<String>,
5693    pub content_type: Option<String>,
5694    pub origin: Option<String>,
5695    pub origin_reference: Option<String>,
5696    pub emitted_at: Option<String>,
5697    pub raw: Value,
5698}
5699
5700/// One bounded at-least-once subscription page.
5701#[derive(Clone, Debug)]
5702pub struct WorkflowStreamPage {
5703    pub stream: WorkflowStreamDescription,
5704    pub items: Vec<WorkflowStreamItem>,
5705    pub next_offset: u64,
5706    pub terminal: bool,
5707}
5708
5709/// Durable acceptance and deduplication outcome for an append request.
5710#[derive(Clone, Debug)]
5711pub struct WorkflowStreamAppendResult {
5712    pub stream: WorkflowStreamDescription,
5713    pub accepted_offsets: Vec<u64>,
5714    pub accepted: u64,
5715    pub deduped: u64,
5716}
5717
5718#[derive(Deserialize)]
5719struct WorkflowStreamListResponse {
5720    #[serde(default)]
5721    streams: Vec<WorkflowStreamDescription>,
5722}
5723
5724#[derive(Deserialize)]
5725struct WorkflowStreamDescriptionResponse {
5726    stream: WorkflowStreamDescription,
5727}
5728
5729#[derive(Deserialize)]
5730struct WorkflowStreamPageResponse {
5731    stream: WorkflowStreamDescription,
5732    #[serde(default)]
5733    items: Vec<Value>,
5734    next_offset: u64,
5735    terminal: bool,
5736}
5737
5738#[derive(Deserialize)]
5739struct WorkflowStreamAppendResponse {
5740    stream: WorkflowStreamDescription,
5741    #[serde(default)]
5742    accepted_offsets: Vec<u64>,
5743    accepted: u64,
5744    deduped: u64,
5745}
5746
5747impl WorkflowDescription {
5748    pub fn is_completed(&self) -> bool {
5749        matches!(self.status.as_deref(), Some("completed" | "Completed"))
5750    }
5751
5752    pub fn is_terminal(&self) -> bool {
5753        matches!(
5754            self.status.as_deref(),
5755            Some(
5756                "completed"
5757                    | "Completed"
5758                    | "failed"
5759                    | "Failed"
5760                    | "cancelled"
5761                    | "Cancelled"
5762                    | "terminated"
5763                    | "Terminated"
5764                    | "timed_out"
5765                    | "TimedOut",
5766            )
5767        )
5768    }
5769
5770    fn decode_payloads(&mut self) -> Result<()> {
5771        if let Some(envelope) = &self.output_envelope {
5772            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5773            self.output = Some(value.clone().into_json()?);
5774            self.output_avro_value = Some(value);
5775        }
5776
5777        Ok(())
5778    }
5779
5780    fn raw_value(&self) -> Value {
5781        let mut data = self.raw.clone();
5782        data.insert(
5783            "workflow_id".to_string(),
5784            self.workflow_id
5785                .clone()
5786                .map(Value::String)
5787                .unwrap_or(Value::Null),
5788        );
5789        data.insert(
5790            "run_id".to_string(),
5791            self.run_id
5792                .clone()
5793                .map(Value::String)
5794                .unwrap_or(Value::Null),
5795        );
5796        data.insert(
5797            "workflow_type".to_string(),
5798            self.workflow_type
5799                .clone()
5800                .map(Value::String)
5801                .unwrap_or(Value::Null),
5802        );
5803        data.insert(
5804            "status".to_string(),
5805            self.status
5806                .clone()
5807                .map(Value::String)
5808                .unwrap_or(Value::Null),
5809        );
5810        data.insert(
5811            "closed_reason".to_string(),
5812            self.closed_reason
5813                .clone()
5814                .map(Value::String)
5815                .unwrap_or(Value::Null),
5816        );
5817        if let Some(failure) = &self.failure {
5818            data.insert("failure".to_string(), failure.clone());
5819        }
5820        if let Some(exception) = &self.exception {
5821            data.insert("exception".to_string(), exception.clone());
5822        }
5823        Value::Object(data.into_iter().collect())
5824    }
5825}
5826
5827fn workflow_terminal_outcome(
5828    description: &WorkflowDescription,
5829    workflow_id: &str,
5830    run_id: Option<&str>,
5831) -> WorkflowTerminalOutcome {
5832    let terminal_kind = description
5833        .closed_reason
5834        .as_deref()
5835        .or(description.status.as_deref())
5836        .unwrap_or("failed")
5837        .to_ascii_lowercase();
5838    let kind = match terminal_kind.as_str() {
5839        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5840        "terminated" => WorkflowTerminalKind::Terminated,
5841        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5842        _ => WorkflowTerminalKind::Failed,
5843    };
5844    let default_reason = match kind {
5845        WorkflowTerminalKind::Failed => "workflow_failed",
5846        WorkflowTerminalKind::Cancelled => "cancelled",
5847        WorkflowTerminalKind::Terminated => "terminated",
5848        WorkflowTerminalKind::TimedOut => "timed_out",
5849    };
5850    let failure = description
5851        .failure
5852        .as_ref()
5853        .filter(|value| value.is_object());
5854    let nested_failure = failure
5855        .and_then(|value| value.get("failures"))
5856        .and_then(Value::as_array)
5857        .and_then(|failures| failures.last())
5858        .or_else(|| description.failures.last());
5859    let exception = description
5860        .exception
5861        .clone()
5862        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5863        .or_else(|| {
5864            nested_failure
5865                .and_then(|value| value.get("exception_payload"))
5866                .cloned()
5867        });
5868    let string_field = |name: &str| {
5869        failure
5870            .and_then(|value| value.get(name))
5871            .and_then(Value::as_str)
5872            .or_else(|| {
5873                nested_failure
5874                    .and_then(|value| value.get(name))
5875                    .and_then(Value::as_str)
5876            })
5877            .map(str::to_string)
5878    };
5879    let exception_field = |name: &str| {
5880        exception
5881            .as_ref()
5882            .and_then(|value| value.get(name))
5883            .and_then(Value::as_str)
5884            .map(str::to_string)
5885    };
5886    let message = description
5887        .error
5888        .clone()
5889        .or_else(|| string_field("message"))
5890        .or_else(|| exception_field("message"));
5891    let reason = description
5892        .raw
5893        .get("reason")
5894        .and_then(Value::as_str)
5895        .map(str::to_string)
5896        .or_else(|| {
5897            failure
5898                .and_then(|value| value.get("reason"))
5899                .and_then(Value::as_str)
5900                .map(str::to_string)
5901        })
5902        .or_else(|| description.closed_reason.clone())
5903        .unwrap_or_else(|| default_reason.to_string());
5904    let failure_id = string_field("failure_id").or_else(|| {
5905        nested_failure
5906            .and_then(|value| value.get("id"))
5907            .and_then(Value::as_str)
5908            .map(str::to_string)
5909    });
5910
5911    WorkflowTerminalOutcome {
5912        kind,
5913        workflow_id: description
5914            .workflow_id
5915            .clone()
5916            .unwrap_or_else(|| workflow_id.to_string()),
5917        run_id: description
5918            .run_id
5919            .clone()
5920            .or_else(|| run_id.map(str::to_string)),
5921        reason,
5922        failure_category: string_field("failure_category")
5923            .or_else(|| Some(default_reason.to_string())),
5924        failure_id,
5925        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5926        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5927        non_retryable: failure
5928            .and_then(|value| value.get("non_retryable"))
5929            .and_then(Value::as_bool)
5930            .or_else(|| {
5931                nested_failure
5932                    .and_then(|value| value.get("non_retryable"))
5933                    .and_then(Value::as_bool)
5934            }),
5935        message,
5936        exception,
5937        raw: description.raw_value(),
5938    }
5939}
5940
5941#[derive(Clone, Debug, Deserialize)]
5942pub struct RegisterWorkerResponse {
5943    pub worker_id: String,
5944    pub registered: bool,
5945    #[serde(default)]
5946    pub heartbeat_interval_seconds: Option<u64>,
5947    #[serde(default)]
5948    pub protocol_version: Option<String>,
5949    #[serde(default)]
5950    pub server_capabilities: Option<Value>,
5951}
5952
5953/// Result of gracefully removing a worker-plane registration.
5954#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5955pub struct WorkerDeregistrationEnvelope {
5956    pub worker_id: String,
5957    pub outcome: String,
5958    pub recovered_workflow_task_count: u64,
5959}
5960
5961#[derive(Clone, Debug, Deserialize)]
5962pub struct PollWorkflowTaskResponse {
5963    #[serde(default)]
5964    pub task: Option<WorkflowTask>,
5965    #[serde(default)]
5966    pub poll_status: Option<String>,
5967    #[serde(default)]
5968    pub reason: Option<String>,
5969    #[serde(default)]
5970    pub protocol_version: Option<String>,
5971    #[serde(default)]
5972    pub server_capabilities: Option<Value>,
5973}
5974
5975impl PollWorkflowTaskResponse {
5976    /// Classify this response without parsing server display text.
5977    pub fn outcome(&self) -> WorkerPollOutcome {
5978        worker_poll_outcome(
5979            self.task.is_some(),
5980            self.poll_status.as_deref(),
5981            self.reason.as_deref(),
5982        )
5983    }
5984}
5985
5986fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5987    let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5988        return false;
5989    };
5990    let supported = capabilities
5991        .get("workflow_memo_updates")
5992        .and_then(Value::as_object)
5993        .and_then(|memo| memo.get("supported"))
5994        .and_then(Value::as_bool)
5995        == Some(true);
5996    let command_advertised = capabilities
5997        .get("supported_workflow_task_commands")
5998        .and_then(Value::as_array)
5999        .is_some_and(|commands| {
6000            commands
6001                .iter()
6002                .any(|command| command.as_str() == Some("upsert_memo"))
6003        });
6004    supported && command_advertised
6005}
6006
6007fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
6008    commands
6009        .iter()
6010        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
6011}
6012
6013#[derive(Clone, Debug, Deserialize)]
6014pub struct PollActivityTaskResponse {
6015    #[serde(default)]
6016    pub task: Option<ActivityTask>,
6017    #[serde(default)]
6018    pub poll_status: Option<String>,
6019    #[serde(default)]
6020    pub reason: Option<String>,
6021}
6022
6023impl PollActivityTaskResponse {
6024    /// Classify this response without parsing server display text.
6025    pub fn outcome(&self) -> WorkerPollOutcome {
6026        worker_poll_outcome(
6027            self.task.is_some(),
6028            self.poll_status.as_deref(),
6029            self.reason.as_deref(),
6030        )
6031    }
6032}
6033
6034#[derive(Clone, Debug, Deserialize)]
6035pub struct PollQueryTaskResponse {
6036    #[serde(default)]
6037    pub task: Option<QueryTask>,
6038    #[serde(default)]
6039    pub poll_status: Option<String>,
6040    #[serde(default)]
6041    pub reason: Option<String>,
6042}
6043
6044impl PollQueryTaskResponse {
6045    /// Classify this response without parsing server display text.
6046    pub fn outcome(&self) -> WorkerPollOutcome {
6047        worker_poll_outcome(
6048            self.task.is_some(),
6049            self.poll_status.as_deref(),
6050            self.reason.as_deref(),
6051        )
6052    }
6053}
6054
6055/// Stable classification for worker poll responses.
6056#[derive(Clone, Debug, PartialEq, Eq)]
6057pub enum WorkerPollOutcome {
6058    /// A task was leased and is available on the response.
6059    Task,
6060    /// No task was leased, but the worker should continue polling.
6061    Idle {
6062        poll_status: Option<String>,
6063        reason: Option<String>,
6064    },
6065    /// The server asked this worker to stop claiming new work.
6066    Stop {
6067        poll_status: Option<String>,
6068        reason: Option<String>,
6069    },
6070}
6071
6072impl WorkerPollOutcome {
6073    pub fn should_stop(&self) -> bool {
6074        matches!(self, Self::Stop { .. })
6075    }
6076}
6077
6078fn worker_poll_outcome(
6079    has_task: bool,
6080    poll_status: Option<&str>,
6081    reason: Option<&str>,
6082) -> WorkerPollOutcome {
6083    if worker_poll_is_stop(poll_status, reason) {
6084        return WorkerPollOutcome::Stop {
6085            poll_status: poll_status.map(str::to_string),
6086            reason: reason.map(str::to_string),
6087        };
6088    }
6089
6090    if has_task {
6091        WorkerPollOutcome::Task
6092    } else {
6093        WorkerPollOutcome::Idle {
6094            poll_status: poll_status.map(str::to_string),
6095            reason: reason.map(str::to_string),
6096        }
6097    }
6098}
6099
6100/// An ephemeral server-routed query task.
6101#[derive(Clone, Debug, Deserialize)]
6102pub struct QueryTask {
6103    pub query_task_id: String,
6104    #[serde(default = "default_workflow_task_attempt")]
6105    pub query_task_attempt: u64,
6106    #[serde(default)]
6107    pub lease_owner: Option<String>,
6108    #[serde(default)]
6109    pub workflow_id: Option<String>,
6110    #[serde(default)]
6111    pub run_id: Option<String>,
6112    pub workflow_type: String,
6113    pub query_name: String,
6114    #[serde(
6115        default = "missing_task_payload_codec",
6116        deserialize_with = "deserialize_task_payload_codec"
6117    )]
6118    pub payload_codec: String,
6119    #[serde(default)]
6120    pub workflow_arguments: Option<Value>,
6121    #[serde(default)]
6122    pub query_arguments: Option<Value>,
6123    #[serde(default)]
6124    pub history_events: Vec<HistoryEvent>,
6125    #[serde(default)]
6126    pub history_export: Option<Value>,
6127    #[serde(default)]
6128    pub run_status: Option<String>,
6129}
6130
6131#[derive(Clone, Debug, Deserialize)]
6132pub struct WorkflowTask {
6133    pub task_id: String,
6134    #[serde(default)]
6135    pub workflow_command_id: Option<String>,
6136    #[serde(default)]
6137    pub workflow_id: Option<String>,
6138    #[serde(default)]
6139    pub run_id: Option<String>,
6140    pub workflow_type: String,
6141    #[serde(default)]
6142    pub cancel_requested: bool,
6143    #[serde(
6144        default = "missing_task_payload_codec",
6145        deserialize_with = "deserialize_task_payload_codec"
6146    )]
6147    pub payload_codec: String,
6148    #[serde(default)]
6149    pub arguments: Option<Value>,
6150    #[serde(default)]
6151    pub history_events: Vec<HistoryEvent>,
6152    #[serde(default)]
6153    pub total_history_events: Option<u64>,
6154    #[serde(default)]
6155    pub history_size_bytes: Option<u64>,
6156    #[serde(default)]
6157    pub continue_as_new_recommended: Option<bool>,
6158    #[serde(default)]
6159    pub history_budget_pressure: Option<String>,
6160    #[serde(default)]
6161    pub next_history_page_token: Option<String>,
6162    #[serde(default = "default_workflow_task_attempt")]
6163    pub workflow_task_attempt: u64,
6164    #[serde(default)]
6165    pub workflow_signal_id: Option<String>,
6166    #[serde(default)]
6167    pub signal_name: Option<String>,
6168    #[serde(default)]
6169    pub signal_arguments: Option<Value>,
6170    #[serde(default)]
6171    pub workflow_update_id: Option<String>,
6172    #[serde(default)]
6173    pub update_name: Option<String>,
6174    #[serde(default)]
6175    pub lease_owner: Option<String>,
6176}
6177
6178impl WorkflowTask {
6179    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
6180        self.history_events.extend(page.history_events);
6181
6182        if page.total_history_events.is_some() {
6183            self.total_history_events = page.total_history_events;
6184        }
6185
6186        self.next_history_page_token = page
6187            .next_history_page_token
6188            .filter(|token| !token.is_empty());
6189    }
6190}
6191
6192#[derive(Clone, Debug, Deserialize)]
6193struct WorkflowTaskHistoryPage {
6194    #[serde(default)]
6195    history_events: Vec<HistoryEvent>,
6196    #[serde(default)]
6197    total_history_events: Option<u64>,
6198    #[serde(default)]
6199    next_history_page_token: Option<String>,
6200}
6201
6202#[derive(Clone, Debug, Deserialize)]
6203pub struct ActivityTask {
6204    pub task_id: String,
6205    #[serde(default)]
6206    pub activity_attempt_id: Option<String>,
6207    #[serde(default)]
6208    pub attempt_id: Option<String>,
6209    pub activity_type: String,
6210    #[serde(
6211        default = "missing_task_payload_codec",
6212        deserialize_with = "deserialize_task_payload_codec"
6213    )]
6214    pub payload_codec: String,
6215    #[serde(default)]
6216    pub arguments: Option<Value>,
6217    #[serde(default = "default_attempt_number")]
6218    pub attempt_number: u64,
6219    #[serde(default)]
6220    pub lease_owner: Option<String>,
6221}
6222
6223#[derive(Clone, Debug, Serialize, Deserialize)]
6224pub struct HistoryEvent {
6225    #[serde(alias = "type")]
6226    pub event_type: String,
6227    #[serde(default)]
6228    pub payload: Value,
6229    #[serde(flatten)]
6230    pub raw: HashMap<String, Value>,
6231}
6232
6233/// One decoded signal in the committed workflow-history snapshot.
6234#[derive(Clone, Debug, PartialEq)]
6235pub struct QuerySignal {
6236    pub id: Option<String>,
6237    pub name: String,
6238    pub arguments: Vec<Value>,
6239    avro_arguments: Vec<AvroValue>,
6240    pub workflow_sequence: Option<u64>,
6241}
6242
6243impl QuerySignal {
6244    /// Lossless fixed Avro Value arguments for this committed signal.
6245    pub fn arguments_avro_value(&self) -> &[AvroValue] {
6246        &self.avro_arguments
6247    }
6248}
6249
6250/// Immutable state supplied to a registered query handler.
6251///
6252/// This context intentionally exposes no activity, signal-wait, or command
6253/// APIs. Query handlers inspect committed history and return a value; query
6254/// completion does not append an event or advance deterministic execution.
6255#[derive(Clone, Debug)]
6256pub struct QueryContext {
6257    pub workflow_id: Option<String>,
6258    pub run_id: Option<String>,
6259    pub workflow_type: String,
6260    pub run_status: Option<String>,
6261    workflow_input: Value,
6262    workflow_input_avro_value: AvroValue,
6263    history_events: Arc<Vec<HistoryEvent>>,
6264    signal_events: Arc<Vec<QuerySignal>>,
6265}
6266
6267impl QueryContext {
6268    /// The normalized argument list used to start the workflow.
6269    pub fn workflow_input(&self) -> &Value {
6270        &self.workflow_input
6271    }
6272
6273    /// The lossless fixed Avro Value argument list used to start the workflow.
6274    pub fn workflow_input_avro_value(&self) -> &AvroValue {
6275        &self.workflow_input_avro_value
6276    }
6277
6278    /// The immutable committed history used for this query snapshot.
6279    pub fn history_events(&self) -> &[HistoryEvent] {
6280        self.history_events.as_slice()
6281    }
6282
6283    /// All decoded signals in committed workflow order.
6284    pub fn signal_events(&self) -> &[QuerySignal] {
6285        self.signal_events.as_slice()
6286    }
6287
6288    /// Decoded argument lists for each committed signal with `signal_name`.
6289    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
6290        self.signal_events
6291            .iter()
6292            .filter(|signal| signal.name == signal_name)
6293            .map(|signal| signal.arguments.clone())
6294            .collect()
6295    }
6296
6297    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
6298    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
6299        self.signal_events
6300            .iter()
6301            .filter(|signal| signal.name == signal_name)
6302            .map(|signal| signal.avro_arguments.clone())
6303            .collect()
6304    }
6305}
6306
6307#[derive(Clone, Debug, Deserialize)]
6308pub struct ActivityHeartbeatResponse {
6309    #[serde(default)]
6310    pub cancel_requested: bool,
6311    #[serde(default)]
6312    pub heartbeat_recorded: bool,
6313    #[serde(default)]
6314    pub can_continue: Option<bool>,
6315    #[serde(default)]
6316    pub reason: Option<String>,
6317    #[serde(default)]
6318    pub run_closed_reason: Option<String>,
6319    #[serde(default)]
6320    pub run_closed_at: Option<String>,
6321    #[serde(default)]
6322    pub lease_expires_at: Option<String>,
6323    #[serde(default)]
6324    pub last_heartbeat_at: Option<String>,
6325}
6326
6327impl ActivityHeartbeatResponse {
6328    /// Whether the activity should stop instead of attempting completion.
6329    pub fn should_stop(&self) -> bool {
6330        self.cancel_requested || self.can_continue == Some(false)
6331    }
6332}
6333
6334fn missing_task_payload_codec() -> String {
6335    MISSING_TASK_PAYLOAD_CODEC.to_string()
6336}
6337
6338fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
6339where
6340    D: Deserializer<'de>,
6341{
6342    Ok(match Value::deserialize(deserializer)? {
6343        Value::String(codec) => codec,
6344        Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
6345        _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
6346    })
6347}
6348
6349fn default_workflow_task_attempt() -> u64 {
6350    1
6351}
6352
6353fn default_attempt_number() -> u64 {
6354    1
6355}
6356
6357type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
6358type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
6359type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
6360type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
6361type ReplayedWorkflowHandler =
6362    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
6363type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
6364type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
6365type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
6366type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
6367type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
6368type ReplayedQueryHandler = Arc<
6369    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
6370        + Send
6371        + Sync,
6372>;
6373type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
6374
6375struct ReplayedWorkflowInvocation {
6376    future: WorkflowFuture,
6377    snapshot: WorkflowStateSnapshot,
6378}
6379
6380#[derive(Clone)]
6381struct RegisteredWorkflow {
6382    execute: WorkflowHandler,
6383    replay: Option<ReplayedWorkflowHandler>,
6384    state_type: Option<TypeId>,
6385    definition_fingerprint: Option<String>,
6386}
6387
6388fn argument_vector_contracts(names: &[String]) -> Vec<Value> {
6389    names
6390        .iter()
6391        .map(|name| {
6392            json!({
6393                "name": name,
6394                "parameters": [{
6395                    "name": "args",
6396                    "position": 0,
6397                    "required": false,
6398                    "variadic": true,
6399                    "type": null,
6400                    "allows_null": true,
6401                    "default_available": false,
6402                    "default": null,
6403                }],
6404            })
6405        })
6406        .collect()
6407}
6408
6409#[derive(Debug)]
6410struct WorkflowTaskDecision {
6411    commands: Vec<Value>,
6412    message_stream_cursors: Vec<Value>,
6413    message_stream_waits: Vec<Value>,
6414    cancellation_delivery: Option<CancellationDelivery>,
6415    cancellation_scope_opening: Option<cancellation_scope::CancellationScopeOpening>,
6416    cancellation_scope_delivery: Option<cancellation_scope_replay::ScopeDeliveryIntent>,
6417}
6418
6419enum PreparedWorkflowTask {
6420    Decision(WorkflowTaskDecision),
6421    Execution {
6422        task: WorkflowTask,
6423        context: WorkflowContext,
6424        future: WorkflowFuture,
6425    },
6426}
6427
6428impl WorkflowTaskDecision {
6429    fn without_message_streams(commands: Vec<Value>) -> Self {
6430        Self {
6431            commands,
6432            message_stream_cursors: Vec::new(),
6433            message_stream_waits: Vec::new(),
6434            cancellation_delivery: None,
6435            cancellation_scope_opening: None,
6436            cancellation_scope_delivery: None,
6437        }
6438    }
6439}
6440
6441#[derive(Clone)]
6442enum RegisteredQuery {
6443    Snapshot(QueryHandler),
6444    Replayed {
6445        state_type: TypeId,
6446        handler: ReplayedQueryHandler,
6447    },
6448}
6449
6450#[derive(Clone, Debug)]
6451pub struct WorkerHeartbeatObservation {
6452    pub worker_id: String,
6453    pub task_queue: String,
6454    pub acknowledged_at_unix_millis: u64,
6455    pub acknowledgement: Value,
6456}
6457
6458/// Bounded retry policy for worker poll acquisition and worker heartbeats.
6459///
6460/// Expected empty long polls and explicit Server capacity backpressure are
6461/// normal worker states. Capacity backpressure honors the advertised retry
6462/// delay without consuming this retry budget. Other transport failures,
6463/// retryable HTTP 408/429 responses, and server errors are retried with capped
6464/// exponential backoff. Authentication, protocol, codec, and handler failures
6465/// are never retried by the worker.
6466/// Explicit storage admission pauses also preserve the already-serialized worker
6467/// request without consuming this budget or rerunning its handler. The delay is
6468/// capped by `max_backoff`, and shutdown interrupts storage waits.
6469/// [`Worker::recover_transient_outages`] can keep a long-running worker retrying
6470/// poll acquisition and heartbeats beyond this budget. `run_once` stays bounded,
6471/// and `max_retries = 0` disables ordinary retries in either mode.
6472#[derive(Clone, Copy, Debug)]
6473pub struct WorkerRetryPolicy {
6474    /// Number of retries after the initial request fails.
6475    pub max_retries: usize,
6476    /// Delay before the first retry.
6477    pub initial_backoff: Duration,
6478    /// Maximum delay between retries.
6479    pub max_backoff: Duration,
6480}
6481
6482impl Default for WorkerRetryPolicy {
6483    fn default() -> Self {
6484        Self {
6485            max_retries: 5,
6486            initial_backoff: Duration::from_millis(100),
6487            max_backoff: Duration::from_secs(5),
6488        }
6489    }
6490}
6491
6492#[derive(Clone, Debug)]
6493struct WorkerStorageAdmission {
6494    policy: WorkerRetryPolicy,
6495    stop: Arc<AtomicBool>,
6496}
6497
6498struct StopWorkerOnDrop(Arc<AtomicBool>);
6499
6500impl Drop for StopWorkerOnDrop {
6501    fn drop(&mut self) {
6502        self.0.store(true, Ordering::SeqCst);
6503    }
6504}
6505
6506async fn wait_for_worker_stop(stop: &AtomicBool) {
6507    while !stop.load(Ordering::SeqCst) {
6508        tokio::time::sleep(Duration::from_millis(100)).await;
6509    }
6510}
6511
6512#[derive(Clone, Copy, Debug, PartialEq, Eq)]
6513enum ManagedPollOutcome {
6514    Idle,
6515    Handled,
6516    Stop,
6517}
6518
6519#[derive(Clone)]
6520pub struct Worker {
6521    client: Client,
6522    worker_id: String,
6523    task_queue: String,
6524    workflows: HashMap<String, RegisteredWorkflow>,
6525    activities: HashMap<String, ActivityHandler>,
6526    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
6527    signals: HashMap<String, Vec<String>>,
6528    updates: HashMap<String, HashMap<String, UpdateHandler>>,
6529    registration_definitions:
6530        HashMap<(RegistrationKind, Option<String>, String), HandlerRegistration>,
6531    registration_error: Option<DuplicateRegistrationError>,
6532    max_concurrent_workflow_tasks: usize,
6533    max_concurrent_activity_tasks: usize,
6534    poll_timeout: Duration,
6535    heartbeat_interval: Duration,
6536    retry_policy: WorkerRetryPolicy,
6537    recover_transient_outages: bool,
6538    run_stop: Option<Arc<AtomicBool>>,
6539    heartbeat_observer: Option<WorkerHeartbeatObserver>,
6540    cooperative_cancellation_enabled: bool,
6541    allow_cancellation_scope_authoring: bool,
6542    allow_cancellation_scope_delivery: bool,
6543    cooperative_registration_confirmed: Arc<AtomicBool>,
6544    local_registration_confirmed: Arc<AtomicBool>,
6545    session_registration_confirmed: Arc<AtomicBool>,
6546    sticky_registration_confirmed: Arc<AtomicBool>,
6547    resource_capabilities: Vec<String>,
6548    sessions: Arc<Mutex<HashMap<String, WorkerSession>>>,
6549}
6550
6551impl Worker {
6552    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
6553        Self {
6554            client,
6555            worker_id: default_worker_id(),
6556            task_queue: task_queue.into(),
6557            workflows: HashMap::new(),
6558            activities: HashMap::new(),
6559            queries: HashMap::new(),
6560            signals: HashMap::new(),
6561            updates: HashMap::new(),
6562            registration_definitions: HashMap::new(),
6563            registration_error: None,
6564            max_concurrent_workflow_tasks: 10,
6565            max_concurrent_activity_tasks: 10,
6566            poll_timeout: Duration::from_secs(30),
6567            heartbeat_interval: Duration::from_secs(60),
6568            retry_policy: WorkerRetryPolicy::default(),
6569            recover_transient_outages: false,
6570            run_stop: None,
6571            heartbeat_observer: None,
6572            cooperative_cancellation_enabled: false,
6573            allow_cancellation_scope_authoring: false,
6574            allow_cancellation_scope_delivery: false,
6575            cooperative_registration_confirmed: Arc::new(AtomicBool::new(false)),
6576            local_registration_confirmed: Arc::new(AtomicBool::new(false)),
6577            session_registration_confirmed: Arc::new(AtomicBool::new(false)),
6578            sticky_registration_confirmed: Arc::new(AtomicBool::new(false)),
6579            resource_capabilities: Vec::new(),
6580            sessions: Arc::new(Mutex::new(HashMap::new())),
6581        }
6582    }
6583
6584    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
6585        self.worker_id = worker_id.into();
6586        self.cooperative_registration_confirmed = Arc::new(AtomicBool::new(false));
6587        self.local_registration_confirmed = Arc::new(AtomicBool::new(false));
6588        self.session_registration_confirmed = Arc::new(AtomicBool::new(false));
6589        self.sticky_registration_confirmed = Arc::new(AtomicBool::new(false));
6590        self.client.reset_sticky_cache();
6591        self.sessions = Arc::new(Mutex::new(HashMap::new()));
6592        self
6593    }
6594
6595    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
6596        self.poll_timeout = timeout;
6597        self
6598    }
6599
6600    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
6601        self.heartbeat_interval = interval;
6602        self
6603    }
6604
6605    /// Explicitly enable inline local activities in this workflow worker.
6606    ///
6607    /// The worker advertises the capability to Server only when enabled.
6608    /// These callbacks must yield to Tokio and have idempotent side effects.
6609    /// Cooperative prepared local supervision is a separate capability.
6610    pub fn local_activities(mut self, enabled: bool) -> Self {
6611        self.client.local_activities_enabled = enabled;
6612        self.local_registration_confirmed = Arc::new(AtomicBool::new(false));
6613        self
6614    }
6615
6616    /// Opt in to separate cooperative workflow cancellation requests.
6617    ///
6618    /// Registration requires the Server to acknowledge this exact worker,
6619    /// namespace, queue and capabilities with compatible protocol 1.20. Task
6620    /// processing uses canonical delivery and activity ownership fences.
6621    /// Existing terminal cancel and terminate operations remain terminal.
6622    /// The default is disabled, preserving ordinary protocol 1.19 workers.
6623    /// Call [`Worker::register`] before [`Worker::run_once`]. [`Worker::run`]
6624    /// and [`Worker::run_until`] register automatically.
6625    pub fn cooperative_cancellation(mut self, enabled: bool) -> Self {
6626        self.cooperative_cancellation_enabled = enabled;
6627        self.client.cooperative_worker_protocol = enabled;
6628        self.cooperative_registration_confirmed = Arc::new(AtomicBool::new(false));
6629        self
6630    }
6631
6632    /// Enable candidate scope authoring without advertising scoped execution.
6633    #[doc(hidden)]
6634    pub fn candidate_cancellation_scope_authoring(mut self, enabled: bool) -> Self {
6635        self.allow_cancellation_scope_authoring = enabled;
6636        self
6637    }
6638
6639    /// Enable the candidate scoped call consumer with canonical authoring.
6640    #[doc(hidden)]
6641    pub fn candidate_cancellation_scope_delivery(mut self, enabled: bool) -> Self {
6642        self.allow_cancellation_scope_delivery = enabled;
6643        self
6644    }
6645
6646    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
6647    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
6648        self.retry_policy = policy;
6649        self
6650    }
6651
6652    /// Keep `run` and `run_until` alive during retryable poll and heartbeat outages.
6653    ///
6654    /// Retries preserve the poll request identity and use capped exponential
6655    /// backoff from [`WorkerRetryPolicy`]. Shutdown interrupts a retry wait.
6656    /// An in-flight poll is allowed to finish so leased work is not discarded.
6657    /// Authentication, protocol, codec, handler and task settlement failures
6658    /// remain errors. Registration and deregistration are not retried here.
6659    ///
6660    /// The default is disabled, preserving the bounded retry contract.
6661    /// `run_once` always remains bounded. `max_retries = 0` disables retries,
6662    /// even when this option is enabled.
6663    pub fn recover_transient_outages(mut self, enabled: bool) -> Self {
6664        self.recover_transient_outages = enabled;
6665        self
6666    }
6667
6668    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
6669    where
6670        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
6671    {
6672        self.heartbeat_observer = Some(Arc::new(observer));
6673        self
6674    }
6675
6676    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
6677        self.max_concurrent_workflow_tasks = count.max(1);
6678        self
6679    }
6680
6681    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
6682        self.max_concurrent_activity_tasks = count.max(1);
6683        self
6684    }
6685
6686    /// Validate local handler admission without making a Server request.
6687    ///
6688    /// Registration methods retain their existing signatures. A duplicate keeps
6689    /// the original handler and makes this Worker invalid, including its clones.
6690    /// `register`, `run`, `run_until` and `run_once` enforce this check before
6691    /// any network request. Build a new Worker to correct an invalid configuration.
6692    pub fn validate_registration(&self) -> std::result::Result<(), DuplicateRegistrationError> {
6693        match &self.registration_error {
6694            Some(error) => Err(error.clone()),
6695            None => Ok(()),
6696        }
6697    }
6698
6699    #[track_caller]
6700    fn admit_registration<F>(
6701        &mut self,
6702        handler_kind: RegistrationKind,
6703        workflow_type: Option<&str>,
6704        handler_name: &str,
6705        method: &'static str,
6706    ) -> bool {
6707        let location = std::panic::Location::caller();
6708        let definition = HandlerRegistration {
6709            method,
6710            handler_type: type_name::<F>(),
6711            file: location.file(),
6712            line: location.line(),
6713            column: location.column(),
6714        };
6715        let key = (
6716            handler_kind,
6717            workflow_type.map(str::to_owned),
6718            handler_name.to_owned(),
6719        );
6720        if let Some(first_definition) = self.registration_definitions.get(&key) {
6721            if self.registration_error.is_none() {
6722                self.registration_error = Some(DuplicateRegistrationError {
6723                    handler_kind,
6724                    workflow_type: key.1.clone(),
6725                    handler_name: key.2.clone(),
6726                    first_definition: first_definition.clone(),
6727                    second_definition: definition,
6728                });
6729            }
6730            return false;
6731        }
6732        self.registration_definitions.insert(key, definition);
6733        true
6734    }
6735
6736    /// Register a workflow handler.
6737    ///
6738    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
6739    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
6740    /// while acquiring or decoding a worker task remain worker-operation
6741    /// failures and do not get converted into workflow outcomes.
6742    #[track_caller]
6743    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
6744    where
6745        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
6746        Fut: Future<Output = Result<Value>> + Send + 'static,
6747    {
6748        let workflow_type = workflow_type.into();
6749        if !self.admit_registration::<F>(
6750            RegistrationKind::Workflow,
6751            None,
6752            &workflow_type,
6753            "register_workflow",
6754        ) {
6755            return;
6756        }
6757        let handler = Arc::new(handler);
6758        self.workflows.insert(
6759            workflow_type,
6760            RegisteredWorkflow {
6761                execute: Arc::new(move |ctx, input| {
6762                    let handler = Arc::clone(&handler);
6763                    Box::pin(async move {
6764                        let result = handler(ctx, input.into_json()?).await?;
6765                        AvroValue::from_serialize(&result)
6766                    })
6767                }),
6768                replay: None,
6769                state_type: None,
6770                definition_fingerprint: None,
6771            },
6772        );
6773    }
6774
6775    /// Register a workflow with one Serde request value and a Serde result.
6776    ///
6777    /// This is an ergonomic adapter over the same fixed Avro Value protocol as
6778    /// [`Worker::register_workflow_avro_value`]. It does not create or publish a
6779    /// workflow-specific schema. A task must contain zero arguments for a unit
6780    /// request or exactly one argument for every other request type.
6781    ///
6782    /// See the runnable
6783    /// [`hello_world` example](https://github.com/durable-workflow/sdk-rust/blob/main/examples/hello_world.rs)
6784    /// for typed workflow and activity contracts with retry and timeout policy.
6785    #[track_caller]
6786    pub fn register_typed_workflow<I, O, F, Fut>(
6787        &mut self,
6788        workflow_type: impl Into<String>,
6789        handler: F,
6790    ) where
6791        I: DeserializeOwned + Send + 'static,
6792        O: Serialize + Send + 'static,
6793        F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
6794        Fut: Future<Output = Result<O>> + Send + 'static,
6795    {
6796        let workflow_type = workflow_type.into();
6797        if !self.admit_registration::<F>(
6798            RegistrationKind::Workflow,
6799            None,
6800            &workflow_type,
6801            "register_typed_workflow",
6802        ) {
6803            return;
6804        }
6805        let handler_name = workflow_type.clone();
6806        let handler = Arc::new(handler);
6807        self.workflows.insert(
6808            workflow_type,
6809            RegisteredWorkflow {
6810                execute: Arc::new(move |ctx, input| {
6811                    let handler = Arc::clone(&handler);
6812                    let handler_name = handler_name.clone();
6813                    Box::pin(async move {
6814                        let input =
6815                            decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6816                        let result = handler(ctx, input).await?;
6817                        encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6818                    })
6819                }),
6820                replay: None,
6821                state_type: None,
6822                definition_fingerprint: None,
6823            },
6824        );
6825    }
6826
6827    /// Register a workflow on the lossless fixed Avro Value surface.
6828    #[track_caller]
6829    pub fn register_workflow_avro_value<F, Fut>(
6830        &mut self,
6831        workflow_type: impl Into<String>,
6832        handler: F,
6833    ) where
6834        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6835        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6836    {
6837        let workflow_type = workflow_type.into();
6838        if !self.admit_registration::<F>(
6839            RegistrationKind::Workflow,
6840            None,
6841            &workflow_type,
6842            "register_workflow_avro_value",
6843        ) {
6844            return;
6845        }
6846        self.workflows.insert(
6847            workflow_type,
6848            RegisteredWorkflow {
6849                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6850                replay: None,
6851                state_type: None,
6852                definition_fingerprint: None,
6853            },
6854        );
6855    }
6856
6857    /// Register a workflow whose typed instance state can be reconstructed for queries.
6858    ///
6859    /// `state_factory` creates a fresh instance for every normal workflow task and
6860    /// query replay. The workflow handler is the single source of truth for state
6861    /// transitions: it updates [`WorkflowInstance`] after activities and signals
6862    /// resolve. Query replay runs this same handler over committed history and
6863    /// discards any commands it would emit.
6864    #[track_caller]
6865    pub fn register_replayed_workflow<S, Factory, F, Fut>(
6866        &mut self,
6867        workflow_type: impl Into<String>,
6868        state_factory: Factory,
6869        handler: F,
6870    ) where
6871        S: Clone + Send + Sync + 'static,
6872        Factory: Fn() -> S + Send + Sync + 'static,
6873        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6874        Fut: Future<Output = Result<Value>> + Send + 'static,
6875    {
6876        let workflow_type = workflow_type.into();
6877        if !self.admit_registration::<F>(
6878            RegistrationKind::Workflow,
6879            None,
6880            &workflow_type,
6881            "register_replayed_workflow",
6882        ) {
6883            return;
6884        }
6885        let state_factory = Arc::new(state_factory);
6886        let handler = Arc::new(handler);
6887
6888        let execute_factory = Arc::clone(&state_factory);
6889        let execute_handler = Arc::clone(&handler);
6890        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6891            let state = WorkflowInstance::new(execute_factory());
6892            let handler = Arc::clone(&execute_handler);
6893            Box::pin(async move {
6894                let result = handler(ctx, input.into_json()?, state).await?;
6895                AvroValue::from_serialize(&result)
6896            }) as WorkflowFuture
6897        });
6898
6899        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6900            let state = WorkflowInstance::new(state_factory());
6901            let snapshot_state = state.clone();
6902            let snapshot: WorkflowStateSnapshot =
6903                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6904            let replay_handler = Arc::clone(&handler);
6905            let future = async move {
6906                let result = replay_handler(ctx, input.into_json()?, state).await?;
6907                AvroValue::from_serialize(&result)
6908            };
6909            ReplayedWorkflowInvocation {
6910                future: Box::pin(future),
6911                snapshot,
6912            }
6913        });
6914
6915        self.workflows.insert(
6916            workflow_type,
6917            RegisteredWorkflow {
6918                execute,
6919                replay: Some(replay),
6920                state_type: Some(TypeId::of::<S>()),
6921                definition_fingerprint: None,
6922            },
6923        );
6924    }
6925
6926    /// Register a replayable workflow with one Serde request value and result.
6927    ///
6928    /// Normal task execution and instance-state query replay both decode and
6929    /// encode through the fixed Avro Value codec. The state factory and handler
6930    /// otherwise follow [`Worker::register_replayed_workflow`].
6931    #[track_caller]
6932    pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6933        &mut self,
6934        workflow_type: impl Into<String>,
6935        state_factory: Factory,
6936        handler: F,
6937    ) where
6938        I: DeserializeOwned + Send + 'static,
6939        O: Serialize + Send + 'static,
6940        S: Clone + Send + Sync + 'static,
6941        Factory: Fn() -> S + Send + Sync + 'static,
6942        F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6943        Fut: Future<Output = Result<O>> + Send + 'static,
6944    {
6945        let workflow_type = workflow_type.into();
6946        if !self.admit_registration::<F>(
6947            RegistrationKind::Workflow,
6948            None,
6949            &workflow_type,
6950            "register_typed_replayed_workflow",
6951        ) {
6952            return;
6953        }
6954        let state_factory = Arc::new(state_factory);
6955        let handler = Arc::new(handler);
6956
6957        let execute_name = workflow_type.clone();
6958        let execute_factory = Arc::clone(&state_factory);
6959        let execute_handler = Arc::clone(&handler);
6960        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6961            let state = WorkflowInstance::new(execute_factory());
6962            let handler = Arc::clone(&execute_handler);
6963            let handler_name = execute_name.clone();
6964            Box::pin(async move {
6965                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6966                let result = handler(ctx, input, state).await?;
6967                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6968            }) as WorkflowFuture
6969        });
6970
6971        let replay_name = workflow_type.clone();
6972        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6973            let state = WorkflowInstance::new(state_factory());
6974            let snapshot_state = state.clone();
6975            let snapshot: WorkflowStateSnapshot =
6976                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6977            let handler = Arc::clone(&handler);
6978            let handler_name = replay_name.clone();
6979            let future = async move {
6980                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6981                let result = handler(ctx, input, state).await?;
6982                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6983            };
6984            ReplayedWorkflowInvocation {
6985                future: Box::pin(future),
6986                snapshot,
6987            }
6988        });
6989
6990        self.workflows.insert(
6991            workflow_type,
6992            RegisteredWorkflow {
6993                execute,
6994                replay: Some(replay),
6995                state_type: Some(TypeId::of::<S>()),
6996                definition_fingerprint: None,
6997            },
6998        );
6999    }
7000
7001    /// Declare the signal names consumed by a registered workflow.
7002    ///
7003    /// Call this after registering the workflow and before starting the worker.
7004    /// Names read through `wait_signal` or `signals` need a declaration for
7005    /// Server command admission. Arguments remain arbitrary positional values,
7006    /// including lossless Avro values. This replaces the previous declaration;
7007    /// an empty slice declares no signals. Names are sorted and deduplicated.
7008    /// Existing runs retain the declarations captured when they started.
7009    pub fn declare_workflow_signals(
7010        &mut self,
7011        workflow_type: &str,
7012        signal_names: &[&str],
7013    ) -> Result<()> {
7014        if !self.workflows.contains_key(workflow_type) {
7015            return Err(Error::Codec(format!(
7016                "workflow type {workflow_type:?} is not registered"
7017            )));
7018        }
7019        if signal_names
7020            .iter()
7021            .any(|name| name.is_empty() || *name != name.trim())
7022        {
7023            return Err(Error::Codec(
7024                "signal names must be non-empty and have no surrounding whitespace".into(),
7025            ));
7026        }
7027        let mut names = signal_names
7028            .iter()
7029            .map(|name| (*name).to_string())
7030            .collect::<Vec<_>>();
7031        names.sort();
7032        names.dedup();
7033        self.signals.insert(workflow_type.to_string(), names);
7034        Ok(())
7035    }
7036
7037    /// Bind a registered workflow to compile-time embedded source for safe redrive.
7038    ///
7039    /// Supply `include_str!` values for the workflow body and every helper whose
7040    /// behavior can affect replay. Register the workflow once, then set its
7041    /// identity before starting the worker. Duplicate registrations are invalid. The server
7042    /// rejects reusing a worker ID when its prior fingerprint disappears.
7043    /// Workflows without source identity remain runnable but cannot be safely redriven.
7044    /// For example, pass `&[include_str!("workflows.rs")]` when the handler and
7045    /// replay-sensitive helpers are in a sibling `workflows.rs` file.
7046    pub fn set_workflow_definition_sources(
7047        &mut self,
7048        workflow_type: &str,
7049        sources: &[&str],
7050    ) -> Result<()> {
7051        if sources.is_empty() || sources.iter().any(|source| source.is_empty()) {
7052            return Err(Error::Codec(
7053                "workflow definition sources must be non-empty".to_string(),
7054            ));
7055        }
7056        let workflow = self.workflows.get_mut(workflow_type).ok_or_else(|| {
7057            Error::Codec(format!("workflow type {workflow_type:?} is not registered"))
7058        })?;
7059        let mut hasher = Sha256::new();
7060        hasher.update(b"durable-workflow-rust.workflow-definition.v1\0");
7061        hasher.update((workflow_type.len() as u64).to_be_bytes());
7062        hasher.update(workflow_type.as_bytes());
7063        for source in sources {
7064            hasher.update((source.len() as u64).to_be_bytes());
7065            hasher.update(source.as_bytes());
7066        }
7067        workflow.definition_fingerprint = Some(format!("sha256:{:x}", hasher.finalize()));
7068        Ok(())
7069    }
7070
7071    /// Register a replayable workflow on the lossless fixed Avro Value surface.
7072    #[track_caller]
7073    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
7074        &mut self,
7075        workflow_type: impl Into<String>,
7076        state_factory: Factory,
7077        handler: F,
7078    ) where
7079        S: Clone + Send + Sync + 'static,
7080        Factory: Fn() -> S + Send + Sync + 'static,
7081        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
7082        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
7083    {
7084        let workflow_type = workflow_type.into();
7085        if !self.admit_registration::<F>(
7086            RegistrationKind::Workflow,
7087            None,
7088            &workflow_type,
7089            "register_replayed_workflow_avro_value",
7090        ) {
7091            return;
7092        }
7093        let state_factory = Arc::new(state_factory);
7094        let handler = Arc::new(handler);
7095
7096        let execute_factory = Arc::clone(&state_factory);
7097        let execute_handler = Arc::clone(&handler);
7098        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
7099            let state = WorkflowInstance::new(execute_factory());
7100            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
7101        });
7102
7103        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
7104            let state = WorkflowInstance::new(state_factory());
7105            let snapshot_state = state.clone();
7106            let snapshot: WorkflowStateSnapshot =
7107                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
7108            ReplayedWorkflowInvocation {
7109                future: Box::pin(handler(ctx, input, state)),
7110                snapshot,
7111            }
7112        });
7113
7114        self.workflows.insert(
7115            workflow_type,
7116            RegisteredWorkflow {
7117                execute,
7118                replay: Some(replay),
7119                state_type: Some(TypeId::of::<S>()),
7120                definition_fingerprint: None,
7121            },
7122        );
7123    }
7124
7125    #[track_caller]
7126    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
7127    where
7128        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
7129        Fut: Future<Output = Result<Value>> + Send + 'static,
7130    {
7131        let activity_type = activity_type.into();
7132        if !self.admit_registration::<F>(
7133            RegistrationKind::Activity,
7134            None,
7135            &activity_type,
7136            "register_activity",
7137        ) {
7138            return;
7139        }
7140        let handler = Arc::new(handler);
7141        self.activities.insert(
7142            activity_type,
7143            Arc::new(move |ctx, args| {
7144                let handler = Arc::clone(&handler);
7145                Box::pin(async move {
7146                    let result = handler(ctx, args.into_json()?).await?;
7147                    AvroValue::from_serialize(&result)
7148                })
7149            }),
7150        );
7151    }
7152
7153    /// Register an activity with one Serde request value and a Serde result.
7154    ///
7155    /// Inputs and results use the platform's fixed Avro Value schema. Shape
7156    /// mismatches and unsupported Serde values return [`Error::HandlerType`]
7157    /// with the activity name and Rust type.
7158    #[track_caller]
7159    pub fn register_typed_activity<I, O, F, Fut>(
7160        &mut self,
7161        activity_type: impl Into<String>,
7162        handler: F,
7163    ) where
7164        I: DeserializeOwned + Send + 'static,
7165        O: Serialize + Send + 'static,
7166        F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
7167        Fut: Future<Output = Result<O>> + Send + 'static,
7168    {
7169        let activity_type = activity_type.into();
7170        if !self.admit_registration::<F>(
7171            RegistrationKind::Activity,
7172            None,
7173            &activity_type,
7174            "register_typed_activity",
7175        ) {
7176            return;
7177        }
7178        let handler_name = activity_type.clone();
7179        let handler = Arc::new(handler);
7180        self.activities.insert(
7181            activity_type,
7182            Arc::new(move |ctx, input| {
7183                let handler = Arc::clone(&handler);
7184                let handler_name = handler_name.clone();
7185                Box::pin(async move {
7186                    let input =
7187                        decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
7188                    let result = handler(ctx, input).await?;
7189                    encode_handler_result(&result, HandlerKind::Activity, &handler_name)
7190                })
7191            }),
7192        );
7193    }
7194
7195    /// Register an activity on the lossless fixed Avro Value surface.
7196    #[track_caller]
7197    pub fn register_activity_avro_value<F, Fut>(
7198        &mut self,
7199        activity_type: impl Into<String>,
7200        handler: F,
7201    ) where
7202        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
7203        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
7204    {
7205        let activity_type = activity_type.into();
7206        if !self.admit_registration::<F>(
7207            RegistrationKind::Activity,
7208            None,
7209            &activity_type,
7210            "register_activity_avro_value",
7211        ) {
7212            return;
7213        }
7214        self.activities.insert(
7215            activity_type,
7216            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
7217        );
7218    }
7219
7220    /// Register a named, read-only query handler for a workflow type.
7221    ///
7222    /// The workflow type must also be registered with [`Worker::register_workflow`]
7223    /// before the worker runs. The handler receives only an immutable committed
7224    /// state snapshot and normalized query arguments.
7225    #[track_caller]
7226    pub fn register_query<F, Fut>(
7227        &mut self,
7228        workflow_type: impl Into<String>,
7229        query_name: impl Into<String>,
7230        handler: F,
7231    ) where
7232        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
7233        Fut: Future<Output = Result<Value>> + Send + 'static,
7234    {
7235        let workflow_type = workflow_type.into();
7236        let query_name = query_name.into();
7237        if !self.admit_registration::<F>(
7238            RegistrationKind::Query,
7239            Some(&workflow_type),
7240            &query_name,
7241            "register_query",
7242        ) {
7243            return;
7244        }
7245        let handler = Arc::new(handler);
7246        self.queries.entry(workflow_type).or_default().insert(
7247            query_name,
7248            RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
7249                let handler = Arc::clone(&handler);
7250                Box::pin(async move {
7251                    let result = handler(ctx, args.into_json()?).await?;
7252                    AvroValue::from_serialize(&result)
7253                })
7254            })),
7255        );
7256    }
7257
7258    /// Register a query handler on the lossless fixed Avro Value surface.
7259    #[track_caller]
7260    pub fn register_query_avro_value<F, Fut>(
7261        &mut self,
7262        workflow_type: impl Into<String>,
7263        query_name: impl Into<String>,
7264        handler: F,
7265    ) where
7266        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
7267        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
7268    {
7269        let workflow_type = workflow_type.into();
7270        let query_name = query_name.into();
7271        if !self.admit_registration::<F>(
7272            RegistrationKind::Query,
7273            Some(&workflow_type),
7274            &query_name,
7275            "register_query_avro_value",
7276        ) {
7277            return;
7278        }
7279        self.queries.entry(workflow_type).or_default().insert(
7280            query_name,
7281            RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
7282        );
7283    }
7284
7285    /// Register a named query against deterministically replayed instance state.
7286    ///
7287    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
7288    /// same state type `S`. The handler receives an immutable, detached state
7289    /// clone, so successful and failed queries cannot affect workflow execution
7290    /// or the state reconstructed by a later query.
7291    #[track_caller]
7292    pub fn register_replayed_query<S, F, Fut>(
7293        &mut self,
7294        workflow_type: impl Into<String>,
7295        query_name: impl Into<String>,
7296        handler: F,
7297    ) where
7298        S: Clone + Send + Sync + 'static,
7299        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
7300        Fut: Future<Output = Result<Value>> + Send + 'static,
7301    {
7302        let workflow_type = workflow_type.into();
7303        let query_name = query_name.into();
7304        if !self.admit_registration::<F>(
7305            RegistrationKind::Query,
7306            Some(&workflow_type),
7307            &query_name,
7308            "register_replayed_query",
7309        ) {
7310            return;
7311        }
7312        let handler = Arc::new(handler);
7313        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
7314            let state = state.downcast::<S>().map_err(|_| {
7315                "registered query state type does not match the replayed workflow state".to_string()
7316            })?;
7317            let handler = Arc::clone(&handler);
7318            Ok(Box::pin(async move {
7319                let result = handler(ctx, state, args.into_json()?).await?;
7320                AvroValue::from_serialize(&result)
7321            }))
7322        });
7323
7324        self.queries.entry(workflow_type).or_default().insert(
7325            query_name,
7326            RegisteredQuery::Replayed {
7327                state_type: TypeId::of::<S>(),
7328                handler: erased_handler,
7329            },
7330        );
7331    }
7332
7333    /// Register a replayed-state query on the lossless fixed Avro Value surface.
7334    #[track_caller]
7335    pub fn register_replayed_query_avro_value<S, F, Fut>(
7336        &mut self,
7337        workflow_type: impl Into<String>,
7338        query_name: impl Into<String>,
7339        handler: F,
7340    ) where
7341        S: Clone + Send + Sync + 'static,
7342        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
7343        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
7344    {
7345        let workflow_type = workflow_type.into();
7346        let query_name = query_name.into();
7347        if !self.admit_registration::<F>(
7348            RegistrationKind::Query,
7349            Some(&workflow_type),
7350            &query_name,
7351            "register_replayed_query_avro_value",
7352        ) {
7353            return;
7354        }
7355        let handler = Arc::new(handler);
7356        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
7357            let state = state.downcast::<S>().map_err(|_| {
7358                "registered query state type does not match the replayed workflow state".to_string()
7359            })?;
7360            Ok(Box::pin(handler(ctx, state, args)))
7361        });
7362
7363        self.queries.entry(workflow_type).or_default().insert(
7364            query_name,
7365            RegisteredQuery::Replayed {
7366                state_type: TypeId::of::<S>(),
7367                handler: erased_handler,
7368            },
7369        );
7370    }
7371
7372    /// Register a synchronous workflow update handler.
7373    #[track_caller]
7374    pub fn register_update<F, Fut>(
7375        &mut self,
7376        workflow_type: impl Into<String>,
7377        update_name: impl Into<String>,
7378        handler: F,
7379    ) where
7380        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
7381        Fut: Future<Output = Result<Value>> + Send + 'static,
7382    {
7383        let workflow_type = workflow_type.into();
7384        let update_name = update_name.into();
7385        if !self.admit_registration::<F>(
7386            RegistrationKind::Update,
7387            Some(&workflow_type),
7388            &update_name,
7389            "register_update",
7390        ) {
7391            return;
7392        }
7393        let handler = Arc::new(handler);
7394        self.updates.entry(workflow_type).or_default().insert(
7395            update_name,
7396            Arc::new(move |ctx, args| {
7397                let handler = Arc::clone(&handler);
7398                Box::pin(async move {
7399                    let result = handler(ctx, args.into_json()?).await?;
7400                    AvroValue::from_serialize(&result)
7401                })
7402            }),
7403        );
7404    }
7405
7406    /// Register an update handler on the lossless fixed Avro Value surface.
7407    #[track_caller]
7408    pub fn register_update_avro_value<F, Fut>(
7409        &mut self,
7410        workflow_type: impl Into<String>,
7411        update_name: impl Into<String>,
7412        handler: F,
7413    ) where
7414        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
7415        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
7416    {
7417        let workflow_type = workflow_type.into();
7418        let update_name = update_name.into();
7419        if !self.admit_registration::<F>(
7420            RegistrationKind::Update,
7421            Some(&workflow_type),
7422            &update_name,
7423            "register_update_avro_value",
7424        ) {
7425            return;
7426        }
7427        self.updates.entry(workflow_type).or_default().insert(
7428            update_name,
7429            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
7430        );
7431    }
7432
7433    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
7434        self.validate_registration()?;
7435        self.sticky_registration_confirmed
7436            .store(false, Ordering::SeqCst);
7437        self.session_registration_confirmed
7438            .store(false, Ordering::SeqCst);
7439        for capability in &self.resource_capabilities {
7440            worker_session::validate_resource_capability(capability)?;
7441        }
7442        self.local_registration_confirmed
7443            .store(false, Ordering::SeqCst);
7444        if self.client.local_activities_enabled && self.cooperative_cancellation_enabled {
7445            return Err(Error::CooperativeCancellationUnavailable(
7446                "inline local activities cannot advertise prepared local supervision; use a separate ordinary worker for inline local callbacks".into(),
7447            ));
7448        }
7449        let mut command_contracts = serde_json::Map::new();
7450        for workflow_type in self.workflows.keys() {
7451            let mut queries = self
7452                .queries
7453                .get(workflow_type)
7454                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
7455                .unwrap_or_default();
7456            queries.sort();
7457            let mut updates = self
7458                .updates
7459                .get(workflow_type)
7460                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
7461                .unwrap_or_default();
7462            updates.sort();
7463            let signals = self.signals.get(workflow_type).cloned().unwrap_or_default();
7464            command_contracts.insert(
7465                workflow_type.clone(),
7466                json!({
7467                    "queries": queries,
7468                    "query_contracts": argument_vector_contracts(&queries),
7469                    "signals": signals,
7470                    "signal_contracts": argument_vector_contracts(&signals),
7471                    "updates": updates,
7472                    "update_contracts": argument_vector_contracts(&updates),
7473                    "update_validators": [],
7474                }),
7475            );
7476        }
7477
7478        self.cooperative_registration_confirmed
7479            .store(false, Ordering::SeqCst);
7480        let response = self
7481            .client
7482            .register_worker_with_definition_fingerprints(
7483                &self.worker_id,
7484                &self.task_queue,
7485                self.workflows.keys().cloned().collect(),
7486                self.activities.keys().cloned().collect(),
7487                self.max_concurrent_workflow_tasks,
7488                self.max_concurrent_activity_tasks,
7489                [
7490                    Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
7491                    Some(DURABLE_SELECTION_CAPABILITY.to_string()),
7492                    Some(MEMO_UPSERTS_CAPABILITY.to_string()),
7493                    Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
7494                    self.client
7495                        .local_activities_enabled
7496                        .then(|| "local_activities".to_string()),
7497                    self.client
7498                        .worker_sessions_enabled
7499                        .then(|| "worker_sessions".to_string()),
7500                    self.client
7501                        .sticky_cache
7502                        .is_some()
7503                        .then(|| "sticky_execution".to_string()),
7504                    self.cooperative_cancellation_enabled
7505                        .then(|| "cooperative_cancellation".to_string()),
7506                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
7507                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
7508                    worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
7509                        .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
7510                ]
7511                .into_iter()
7512                .flatten()
7513                .chain(self.resource_capabilities.iter().cloned())
7514                .collect(),
7515                Value::Object(command_contracts),
7516                Some(
7517                    self.workflows
7518                        .iter()
7519                        .filter_map(|(workflow_type, workflow)| {
7520                            workflow
7521                                .definition_fingerprint
7522                                .as_ref()
7523                                .map(|fingerprint| (workflow_type.clone(), fingerprint.clone()))
7524                        })
7525                        .collect(),
7526                ),
7527            )
7528            .await?;
7529        if self.cooperative_cancellation_enabled && response.registered {
7530            self.cooperative_registration_confirmed
7531                .store(true, Ordering::SeqCst);
7532        }
7533        if self.client.local_activities_enabled && response.registered {
7534            self.local_registration_confirmed
7535                .store(true, Ordering::SeqCst);
7536        }
7537        if self.client.worker_sessions_enabled && response.registered {
7538            self.session_registration_confirmed
7539                .store(true, Ordering::SeqCst);
7540        }
7541        if self.client.sticky_cache.is_some() && response.registered {
7542            self.sticky_registration_confirmed
7543                .store(true, Ordering::SeqCst);
7544        }
7545        Ok(response)
7546    }
7547
7548    /// Run until shutdown or a terminal worker error occurs.
7549    ///
7550    /// Empty long-poll expirations do not stop the worker. Retryable poll and
7551    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
7552    /// authentication, protocol, and other non-retryable failures are returned.
7553    /// Enable [`Worker::recover_transient_outages`] to keep retrying transient
7554    /// poll and heartbeat outages beyond the ordinary retry budget.
7555    pub async fn run(&self) -> Result<()> {
7556        self.run_until(std::future::pending::<()>()).await
7557    }
7558
7559    /// Run until `shutdown` resolves or a terminal worker error occurs.
7560    ///
7561    /// This has the same liveness and terminal-error contract as [`Worker::run`].
7562    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
7563    where
7564        F: Future<Output = ()>,
7565    {
7566        let _clear_cache_on_drop =
7567            sticky_worker::ClearCacheOnDrop(self.client.sticky_cache.clone());
7568        let stop = Arc::new(AtomicBool::new(false));
7569        let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
7570        let mut worker = self.with_storage_admission(Arc::clone(&stop));
7571        worker.run_stop = Some(Arc::clone(&stop));
7572        let run = worker.run_with_storage_admission(Arc::clone(&stop));
7573        tokio::pin!(run);
7574        tokio::pin!(shutdown);
7575        tokio::select! {
7576            result = &mut run => result,
7577            _ = &mut shutdown => {
7578                stop.store(true, Ordering::SeqCst);
7579                run.await
7580            }
7581        }
7582    }
7583
7584    fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
7585        let mut worker = self.clone();
7586        worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
7587            policy: self.retry_policy,
7588            stop,
7589        });
7590        worker
7591    }
7592
7593    async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
7594        let registration = self.register().await?;
7595        if !registration.registered {
7596            return Err(Error::WorkerLoop(format!(
7597                "worker registration for {:?} was not accepted",
7598                self.worker_id
7599            )));
7600        }
7601        let registered_worker_id = registration.worker_id.clone();
7602        let primary = self.run_registered_until(stop, registration).await;
7603        let primary = match (primary, self.close_worker_sessions().await) {
7604            (Ok(()), result) => result,
7605            (Err(error), Ok(())) => Err(error),
7606            (Err(primary), Err(close)) => Err(Error::WorkerShutdown {
7607                primary: Box::new(primary),
7608                deregistration: Box::new(close),
7609            }),
7610        };
7611        self.session_registration_confirmed
7612            .store(false, Ordering::SeqCst);
7613        self.cooperative_registration_confirmed
7614            .store(false, Ordering::SeqCst);
7615        self.local_registration_confirmed
7616            .store(false, Ordering::SeqCst);
7617        let deregistration = self
7618            .client
7619            .deregister_worker_registration(&registered_worker_id)
7620            .await;
7621        self.sticky_registration_confirmed
7622            .store(false, Ordering::SeqCst);
7623        self.client.clear_sticky_cache()?;
7624
7625        match (primary, deregistration) {
7626            (Ok(()), Ok(_)) => Ok(()),
7627            (Ok(()), Err(deregistration)) => Err(deregistration),
7628            (Err(primary), Ok(_)) => Err(primary),
7629            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
7630                primary: Box::new(primary),
7631                deregistration: Box::new(deregistration),
7632            }),
7633        }
7634    }
7635
7636    async fn run_registered_until(
7637        &self,
7638        stop: Arc<AtomicBool>,
7639        registration: RegisterWorkerResponse,
7640    ) -> Result<()> {
7641        let heartbeat_interval = Duration::from_secs(
7642            registration
7643                .heartbeat_interval_seconds
7644                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
7645        );
7646        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
7647        // from the completion of the preceding attempt, including its bounded
7648        // retries. A fixed-epoch interval can leave an already-due tick queued
7649        // while an acknowledgement is slow, producing a catch-up heartbeat as
7650        // soon as that request completes.
7651        let heartbeat = tokio::time::sleep(Duration::ZERO);
7652        tokio::pin!(heartbeat);
7653        // Poll responses may already have leased server-side work by the time
7654        // they become ready, so each poller owns its responses through
7655        // completion or failure instead of racing raw polls in this select.
7656        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
7657            let worker = self.clone();
7658            let stop = Arc::clone(&stop);
7659            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
7660        });
7661        let mut activity_poller = (!self.activities.is_empty()).then(|| {
7662            let worker = self.clone();
7663            let stop = Arc::clone(&stop);
7664            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
7665        });
7666        let mut query_poller = (!self.queries.is_empty()).then(|| {
7667            let worker = self.clone();
7668            let stop = Arc::clone(&stop);
7669            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
7670        });
7671
7672        loop {
7673            tokio::select! {
7674                _ = wait_for_worker_stop(&stop) => {
7675                    stop.store(true, Ordering::SeqCst);
7676                    break;
7677                }
7678                _ = &mut heartbeat => {
7679                    let result = self.retry_worker_operation(|| {
7680                        self.client.heartbeat_worker_with_sessions(
7681                            &self.worker_id,
7682                            self.max_concurrent_workflow_tasks,
7683                            self.max_concurrent_activity_tasks,
7684                            self.client.worker_sessions_enabled.then(|| self.session_available()),
7685                        )
7686                    }).await;
7687                    heartbeat
7688                        .as_mut()
7689                        .reset(tokio::time::Instant::now() + heartbeat_interval);
7690                    match result {
7691                        Ok(Some(acknowledgement)) => {
7692                            if let Some(observer) = &self.heartbeat_observer {
7693                                observer(&WorkerHeartbeatObservation {
7694                                    worker_id: self.worker_id.clone(),
7695                                    task_queue: self.task_queue.clone(),
7696                                    acknowledged_at_unix_millis: SystemTime::now()
7697                                        .duration_since(UNIX_EPOCH)
7698                                        .unwrap_or_default()
7699                                        .as_millis()
7700                                        .min(u64::MAX as u128)
7701                                        as u64,
7702                                    acknowledgement,
7703                                });
7704                            }
7705                        }
7706                        Ok(None) => break,
7707                        Err(error) => {
7708                            stop.store(true, Ordering::SeqCst);
7709                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
7710                            return Err(error);
7711                        }
7712                    }
7713                }
7714                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
7715                    workflow_poller = None;
7716                    let stopped_by_server = stop.load(Ordering::SeqCst);
7717                    stop.store(true, Ordering::SeqCst);
7718                    let poller_result = optional_poller_result("workflow", result);
7719                    let join_result =
7720                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
7721                    poller_result?;
7722                    join_result?;
7723                    if stopped_by_server {
7724                        return Ok(());
7725                    }
7726                    return Err(Error::WorkerLoop(
7727                        "workflow poller stopped unexpectedly".to_string(),
7728                    ));
7729                }
7730                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
7731                    activity_poller = None;
7732                    let stopped_by_server = stop.load(Ordering::SeqCst);
7733                    stop.store(true, Ordering::SeqCst);
7734                    let poller_result = optional_poller_result("activity", result);
7735                    let join_result =
7736                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
7737                    poller_result?;
7738                    join_result?;
7739                    if stopped_by_server {
7740                        return Ok(());
7741                    }
7742                    return Err(Error::WorkerLoop(
7743                        "activity poller stopped unexpectedly".to_string(),
7744                    ));
7745                }
7746                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
7747                    query_poller = None;
7748                    let stopped_by_server = stop.load(Ordering::SeqCst);
7749                    stop.store(true, Ordering::SeqCst);
7750                    let poller_result = optional_poller_result("query", result);
7751                    let join_result =
7752                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
7753                    poller_result?;
7754                    join_result?;
7755                    if stopped_by_server {
7756                        return Ok(());
7757                    }
7758                    return Err(Error::WorkerLoop(
7759                        "query poller stopped unexpectedly".to_string(),
7760                    ));
7761                }
7762            }
7763        }
7764
7765        join_pollers(
7766            workflow_poller.take(),
7767            activity_poller.take(),
7768            query_poller.take(),
7769        )
7770        .await
7771    }
7772
7773    /// Poll and settle at most one task from each enabled task family.
7774    ///
7775    /// A workflow may reach its server-enforced run deadline while this worker
7776    /// holds a task. When the completion endpoint authoritatively rejects that
7777    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
7778    /// terminal `run_status=failed`, the workflow tick is considered settled:
7779    /// the late command was not recorded and cannot replace the terminal run.
7780    /// Every other completion rejection remains an error. This worker-level
7781    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
7782    /// only bounds how long a client waits for a result.
7783    ///
7784    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
7785    /// the original [`Error::Http`] status and response body.
7786    pub async fn run_once(&self) -> Result<usize> {
7787        self.validate_registration()?;
7788        if self.client.worker_sessions_enabled {
7789            self.require_session_registration()?;
7790        }
7791        if self.cooperative_cancellation_enabled
7792            && !self
7793                .cooperative_registration_confirmed
7794                .load(Ordering::SeqCst)
7795        {
7796            return Err(Error::CooperativeCancellationUnavailable(
7797                "register this cooperative worker before polling tasks".into(),
7798            ));
7799        }
7800        let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
7801        let mut handled = 0;
7802        match worker.poll_workflow_once().await? {
7803            ManagedPollOutcome::Handled => handled += 1,
7804            ManagedPollOutcome::Stop => return Ok(handled),
7805            ManagedPollOutcome::Idle => {}
7806        }
7807        match worker.poll_activity_once().await? {
7808            ManagedPollOutcome::Handled => handled += 1,
7809            ManagedPollOutcome::Stop => return Ok(handled),
7810            ManagedPollOutcome::Idle => {}
7811        }
7812        if !self.queries.is_empty() {
7813            match worker.poll_query_once().await? {
7814                ManagedPollOutcome::Handled => handled += 1,
7815                ManagedPollOutcome::Stop => return Ok(handled),
7816                ManagedPollOutcome::Idle => {}
7817            }
7818        }
7819        Ok(handled)
7820    }
7821
7822    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
7823        if self.client.sticky_cache.is_some()
7824            && !self.sticky_registration_confirmed.load(Ordering::SeqCst)
7825        {
7826            return Err(Error::WorkerLoop("sticky_registration_unconfirmed: register the sticky-capable worker before polling".into()));
7827        }
7828        if self.client.local_activities_enabled
7829            && !self.local_registration_confirmed.load(Ordering::SeqCst)
7830        {
7831            return Err(Error::WorkerLoop("local_activity_registration_unconfirmed: register this local-capable worker before polling workflow tasks".into()));
7832        }
7833        if self.cooperative_cancellation_enabled {
7834            return self.poll_cooperative_workflow_once().await;
7835        }
7836        let poll_request_id = unique_request_id("rust-workflow-poll");
7837        let response = self
7838            .retry_worker_operation(|| self.poll_workflow_with_sticky_cache(&poll_request_id))
7839            .await;
7840        let Some((response, snapshot)) = self.settle_worker_poll_response(response).await? else {
7841            return Ok(ManagedPollOutcome::Idle);
7842        };
7843        if response.outcome().should_stop() {
7844            return Ok(ManagedPollOutcome::Stop);
7845        }
7846        let memo_updates_supported =
7847            runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
7848        let Some(task) = response.task else {
7849            return Ok(ManagedPollOutcome::Idle);
7850        };
7851
7852        let task_id = task.task_id.clone();
7853        let attempt = task.workflow_task_attempt;
7854        let run_id = task.run_id.clone();
7855        let lease_owner = task
7856            .lease_owner
7857            .clone()
7858            .unwrap_or_else(|| self.worker_id.clone());
7859
7860        let decision = match self.execute_workflow_with_local_activities(task).await {
7861            Ok(Some(decision)) => Ok(decision),
7862            Ok(None) => return Ok(ManagedPollOutcome::Handled),
7863            Err(error) => Err(error),
7864        };
7865        self.settle_workflow_task_decision(
7866            &task_id,
7867            &lease_owner,
7868            attempt,
7869            run_id.as_deref(),
7870            decision,
7871            memo_updates_supported,
7872            snapshot,
7873        )
7874        .await
7875    }
7876
7877    #[allow(clippy::too_many_arguments)]
7878    async fn settle_workflow_task_decision(
7879        &self,
7880        task_id: &str,
7881        lease_owner: &str,
7882        attempt: u64,
7883        run_id: Option<&str>,
7884        decision: Result<WorkflowTaskDecision>,
7885        memo_updates_supported: bool,
7886        snapshot: Option<sticky_worker::StickySnapshot>,
7887    ) -> Result<ManagedPollOutcome> {
7888        match decision {
7889            Ok(decision)
7890                if decision.cancellation_delivery.is_some()
7891                    || decision.cancellation_scope_delivery.is_some()
7892                    || decision.cancellation_scope_opening.is_some() =>
7893            {
7894                return Err(Error::CooperativeCancellationUnavailable(
7895                    "worker cancellation delivery has not been negotiated".into(),
7896                ));
7897            }
7898            Ok(decision)
7899                if commands_use_workflow_memo_updates(&decision.commands)
7900                    && !memo_updates_supported =>
7901            {
7902                self.client
7903                    .fail_workflow_task(
7904                        task_id,
7905                        lease_owner,
7906                        attempt,
7907                        Error::WorkflowMemoUpdatesUnavailable.to_string(),
7908                    )
7909                    .await?;
7910            }
7911            Ok(decision) if decision.commands.is_empty() => {
7912                // A replay can consume a recorded pending durable command
7913                // without producing a new command. The standalone protocol
7914                // acknowledges that state through the typed waiting outcome;
7915                // an empty completion is rejected by servers that require at
7916                // least one executable command.
7917                self.client
7918                    .fail_workflow_task_with_type(
7919                        task_id,
7920                        lease_owner,
7921                        attempt,
7922                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
7923                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
7924                    )
7925                    .await?;
7926            }
7927            Ok(decision) => {
7928                let sticky_claim = self.sticky_claim(snapshot.as_ref(), &decision.commands)?;
7929                let completion = self
7930                    .client
7931                    .complete_workflow_task_with_message_streams(
7932                        task_id,
7933                        lease_owner,
7934                        attempt,
7935                        decision.commands,
7936                        decision.message_stream_cursors,
7937                        decision.message_stream_waits,
7938                        sticky_claim,
7939                    )
7940                    .await;
7941                if let Err(error) = completion {
7942                    self.discard_sticky_snapshot(snapshot.as_ref())?;
7943                    if !workflow_task_completion_is_terminal_timeout(
7944                        &error, task_id, attempt, run_id,
7945                    ) {
7946                        return Err(error);
7947                    }
7948                }
7949            }
7950            Err(error @ Error::CancellationScopeExecutionUnavailable) => return Err(error),
7951            Err(error) => {
7952                self.discard_sticky_snapshot(snapshot.as_ref())?;
7953                self.client
7954                    .fail_workflow_task(task_id, lease_owner, attempt, error.to_string())
7955                    .await?;
7956            }
7957        }
7958
7959        Ok(ManagedPollOutcome::Handled)
7960    }
7961
7962    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7963        while !stop.load(Ordering::SeqCst) {
7964            match self.poll_workflow_once().await {
7965                Ok(ManagedPollOutcome::Stop) => {
7966                    stop.store(true, Ordering::SeqCst);
7967                    break;
7968                }
7969                Err(error) => {
7970                    stop.store(true, Ordering::SeqCst);
7971                    return Err(error);
7972                }
7973                _ => {}
7974            }
7975        }
7976
7977        Ok(())
7978    }
7979
7980    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
7981        if self.client.worker_sessions_enabled {
7982            self.require_session_registration()?;
7983        }
7984        if self.cooperative_cancellation_enabled {
7985            return self.poll_cooperative_activity_once().await;
7986        }
7987        let poll_request_id = unique_request_id("rust-activity-poll");
7988        let response = self
7989            .retry_worker_operation(|| {
7990                self.client.poll_activity_task_response_with_request_id(
7991                    &self.worker_id,
7992                    &self.task_queue,
7993                    self.poll_timeout,
7994                    &poll_request_id,
7995                    0,
7996                )
7997            })
7998            .await;
7999        let Some(response) = self.settle_worker_poll_response(response).await? else {
8000            return Ok(ManagedPollOutcome::Idle);
8001        };
8002        if response.outcome().should_stop() {
8003            return Ok(ManagedPollOutcome::Stop);
8004        }
8005        let Some(task) = response.task else {
8006            return Ok(ManagedPollOutcome::Idle);
8007        };
8008
8009        let session = self.track_session_task(task.worker_session.as_ref())?;
8010        let task = task.task;
8011
8012        let task_id = task.task_id.clone();
8013        let attempt_id = task
8014            .activity_attempt_id
8015            .clone()
8016            .or(task.attempt_id.clone())
8017            .unwrap_or_default();
8018        let lease_owner = task
8019            .lease_owner
8020            .clone()
8021            .unwrap_or_else(|| self.worker_id.clone());
8022        let codec = task.payload_codec.clone();
8023        let result = self.execute_session_activity_task(task, session).await;
8024        match result {
8025            Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
8026            Ok(value) => {
8027                let completion = self
8028                    .client
8029                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
8030                    .await;
8031                if let Err(error) = completion {
8032                    if !activity_task_rejection_is_final(&error) {
8033                        return Err(error);
8034                    }
8035                }
8036            }
8037            Err(error) => {
8038                let failure = self
8039                    .client
8040                    .fail_activity_task(
8041                        &task_id,
8042                        &attempt_id,
8043                        &lease_owner,
8044                        error.to_string(),
8045                        false,
8046                    )
8047                    .await;
8048                if let Err(error) = failure {
8049                    if !activity_task_rejection_is_final(&error) {
8050                        return Err(error);
8051                    }
8052                }
8053            }
8054        }
8055
8056        Ok(ManagedPollOutcome::Handled)
8057    }
8058
8059    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
8060        // Each lane retains its leased response through callback settlement.
8061        // A bounded independent activity must not occupy the cleanup lane too.
8062        // Preserve the existing serial loop for default protocol 1.19 workers.
8063        let lane_count = if self.cooperative_cancellation_enabled {
8064            self.max_concurrent_activity_tasks
8065        } else {
8066            1
8067        };
8068        let mut lanes = FuturesUnordered::new();
8069        for _ in 0..lane_count {
8070            let worker = self.clone();
8071            let stop = Arc::clone(&stop);
8072            lanes.push(async move {
8073                while !stop.load(Ordering::SeqCst) {
8074                    match worker.poll_activity_once().await {
8075                        Ok(ManagedPollOutcome::Stop) => {
8076                            stop.store(true, Ordering::SeqCst);
8077                            break;
8078                        }
8079                        Err(error) => {
8080                            stop.store(true, Ordering::SeqCst);
8081                            return Err(error);
8082                        }
8083                        _ => {}
8084                    }
8085                }
8086                Ok(())
8087            });
8088        }
8089        // Join every lane before deregistration, including after one fails.
8090        let mut first_error = None;
8091        while let Some(result) = lanes.next().await {
8092            if let Err(error) = result {
8093                first_error.get_or_insert(error);
8094            }
8095        }
8096        first_error.map_or(Ok(()), Err)
8097    }
8098
8099    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
8100        let poll_request_id = unique_request_id("rust-query-poll");
8101        let response = self
8102            .retry_worker_operation(|| {
8103                self.client.poll_query_task_response_with_request_id(
8104                    &self.worker_id,
8105                    &self.task_queue,
8106                    self.poll_timeout,
8107                    &poll_request_id,
8108                    0,
8109                )
8110            })
8111            .await;
8112        let Some(response) = self.settle_worker_poll_response(response).await? else {
8113            return Ok(ManagedPollOutcome::Idle);
8114        };
8115        if response.outcome().should_stop() {
8116            return Ok(ManagedPollOutcome::Stop);
8117        }
8118        let Some(task) = response.task else {
8119            return Ok(ManagedPollOutcome::Idle);
8120        };
8121
8122        let query_task_id = task.query_task_id.clone();
8123        let attempt = task.query_task_attempt;
8124        let lease_owner = task
8125            .lease_owner
8126            .clone()
8127            .unwrap_or_else(|| self.worker_id.clone());
8128        let codec = task.payload_codec.clone();
8129
8130        match self.execute_query_task(task).await {
8131            Ok(value) => {
8132                let result_envelope = match encode_typed_envelope(&value, &codec) {
8133                    Ok(result_envelope) => result_envelope,
8134                    Err(error) => {
8135                        let failure = self
8136                            .client
8137                            .fail_query_task(
8138                                &query_task_id,
8139                                &lease_owner,
8140                                attempt,
8141                                error.to_string(),
8142                                "query_result_encode_failed",
8143                                "QueryResultEncodeFailed",
8144                            )
8145                            .await;
8146                        if let Err(error) = failure {
8147                            if !query_task_rejection_is_final(&error) {
8148                                return Err(error);
8149                            }
8150                        }
8151                        return Ok(ManagedPollOutcome::Handled);
8152                    }
8153                };
8154
8155                if let Err(error) = self
8156                    .client
8157                    .complete_query_task_with_envelope(
8158                        &query_task_id,
8159                        &lease_owner,
8160                        attempt,
8161                        value.clone().into_json()?,
8162                        result_envelope,
8163                    )
8164                    .await
8165                {
8166                    if !query_task_rejection_is_final(&error) {
8167                        return Err(error);
8168                    }
8169                }
8170            }
8171            Err(failure) => {
8172                let result = self
8173                    .client
8174                    .fail_query_task(
8175                        &query_task_id,
8176                        &lease_owner,
8177                        attempt,
8178                        failure.message,
8179                        failure.reason,
8180                        failure.failure_type,
8181                    )
8182                    .await;
8183                if let Err(error) = result {
8184                    if !query_task_rejection_is_final(&error) {
8185                        return Err(error);
8186                    }
8187                }
8188            }
8189        }
8190
8191        Ok(ManagedPollOutcome::Handled)
8192    }
8193
8194    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
8195        while !stop.load(Ordering::SeqCst) {
8196            match self.poll_query_once().await {
8197                Ok(ManagedPollOutcome::Stop) => {
8198                    stop.store(true, Ordering::SeqCst);
8199                    break;
8200                }
8201                Err(error) => {
8202                    stop.store(true, Ordering::SeqCst);
8203                    return Err(error);
8204                }
8205                _ => {}
8206            }
8207        }
8208
8209        Ok(())
8210    }
8211
8212    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<Option<T>>
8213    where
8214        F: FnMut() -> Fut,
8215        Fut: Future<Output = Result<T>>,
8216    {
8217        let mut retries = 0;
8218
8219        loop {
8220            if self
8221                .run_stop
8222                .as_ref()
8223                .is_some_and(|stop| stop.load(Ordering::SeqCst))
8224            {
8225                return Ok(None);
8226            }
8227            match operation().await {
8228                Err(error)
8229                    if worker_operation_is_retryable(&error)
8230                        && (retries < self.retry_policy.max_retries
8231                            || (self.recover_transient_outages
8232                                && self.run_stop.is_some()
8233                                && self.retry_policy.max_retries > 0)) =>
8234                {
8235                    retries = retries.saturating_add(1);
8236                    let delay = worker_retry_delay(self.retry_policy, retries)
8237                        .max(Duration::from_millis(1));
8238                    if let Some(stop) = &self.run_stop {
8239                        tokio::select! {
8240                            _ = tokio::time::sleep(delay) => {}
8241                            _ = wait_for_worker_stop(stop) => return Ok(None),
8242                        }
8243                    } else {
8244                        tokio::time::sleep(delay).await;
8245                    }
8246                }
8247                result => return result.map(Some),
8248            }
8249        }
8250    }
8251
8252    async fn settle_worker_poll_response<T>(
8253        &self,
8254        response: Result<Option<T>>,
8255    ) -> Result<Option<T>> {
8256        match response {
8257            Ok(response) => Ok(response),
8258            Err(error) => {
8259                let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
8260                    return Err(error);
8261                };
8262                let minimum_delay = self
8263                    .retry_policy
8264                    .initial_backoff
8265                    .max(Duration::from_millis(1));
8266                let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
8267                tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
8268                Ok(None)
8269            }
8270        }
8271    }
8272
8273    async fn execute_query_task(
8274        &self,
8275        mut task: QueryTask,
8276    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
8277        validate_query_task_payloads(&task).map_err(|error| {
8278            QueryTaskExecutionFailure::new(
8279                "query_payload_decode_failed",
8280                error.to_string(),
8281                "QueryPayloadDecodeFailed",
8282            )
8283        })?;
8284
8285        if !self.workflows.contains_key(&task.workflow_type) {
8286            return Err(QueryTaskExecutionFailure::new(
8287                "query_workflow_type_not_registered",
8288                format!("no workflow registered for type {:?}", task.workflow_type),
8289                "WorkflowTypeNotRegistered",
8290            ));
8291        }
8292
8293        let Some(handlers) = self.queries.get(&task.workflow_type) else {
8294            return Err(QueryTaskExecutionFailure::new(
8295                "query_handler_unavailable",
8296                format!(
8297                    "query handlers are unavailable for workflow type {:?}",
8298                    task.workflow_type
8299                ),
8300                "QueryHandlerUnavailable",
8301            ));
8302        };
8303        let Some(query) = handlers.get(&task.query_name) else {
8304            return Err(QueryTaskExecutionFailure::new(
8305                "rejected_unknown_query",
8306                format!("unknown query {:?}", task.query_name),
8307                "QueryFailed",
8308            ));
8309        };
8310
8311        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
8312            .map_err(|error| {
8313            QueryTaskExecutionFailure::new(
8314                "query_payload_decode_failed",
8315                format!("cannot decode query arguments: {error}"),
8316                "QueryPayloadDecodeFailed",
8317            )
8318        })?;
8319        let workflow_input_typed =
8320            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
8321                .map_err(|error| {
8322                    QueryTaskExecutionFailure::new(
8323                        "query_workflow_state_unavailable",
8324                        format!("cannot decode workflow start input: {error}"),
8325                        "QueryWorkflowStateUnavailable",
8326                    )
8327                })?;
8328        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
8329            QueryTaskExecutionFailure::new(
8330                "query_workflow_state_unavailable",
8331                format!("cannot project workflow start input: {error}"),
8332                "QueryWorkflowStateUnavailable",
8333            )
8334        })?;
8335        hydrate_query_history_from_export(&mut task).map_err(|error| {
8336            QueryTaskExecutionFailure::new(
8337                "query_workflow_state_unavailable",
8338                format!("cannot restore query history snapshot: {error}"),
8339                "QueryWorkflowStateUnavailable",
8340            )
8341        })?;
8342        enrich_query_history_from_export(&mut task).map_err(|error| {
8343            QueryTaskExecutionFailure::new(
8344                "query_workflow_state_unavailable",
8345                format!("cannot restore compact query history payloads: {error}"),
8346                "QueryWorkflowStateUnavailable",
8347            )
8348        })?;
8349        let signal_events = query_signal_events(&task).map_err(|error| {
8350            QueryTaskExecutionFailure::new(
8351                "query_workflow_state_unavailable",
8352                format!("cannot decode committed workflow signals: {error}"),
8353                "QueryWorkflowStateUnavailable",
8354            )
8355        })?;
8356        let history_events = Arc::new(std::mem::take(&mut task.history_events));
8357        let context = QueryContext {
8358            workflow_id: task.workflow_id,
8359            run_id: task.run_id,
8360            workflow_type: task.workflow_type.clone(),
8361            run_status: task.run_status,
8362            workflow_input,
8363            workflow_input_avro_value: workflow_input_typed.clone(),
8364            history_events: Arc::clone(&history_events),
8365            signal_events: Arc::new(signal_events),
8366        };
8367
8368        let future = match query {
8369            RegisteredQuery::Snapshot(handler) => handler(context, args),
8370            RegisteredQuery::Replayed {
8371                state_type,
8372                handler,
8373            } => {
8374                let workflow = self
8375                    .workflows
8376                    .get(&task.workflow_type)
8377                    .expect("workflow registration was checked above");
8378                if workflow.state_type != Some(*state_type) {
8379                    return Err(QueryTaskExecutionFailure::new(
8380                        "query_workflow_state_unavailable",
8381                        "replayed query state type does not match its workflow registration",
8382                        "QueryWorkflowStateUnavailable",
8383                    ));
8384                }
8385                let replay = workflow.replay.as_ref().ok_or_else(|| {
8386                    QueryTaskExecutionFailure::new(
8387                        "query_workflow_state_unavailable",
8388                        format!(
8389                            "workflow type {:?} is not registered for instance-state replay",
8390                            task.workflow_type
8391                        ),
8392                        "QueryWorkflowStateUnavailable",
8393                    )
8394                })?;
8395                let workflow_state = Arc::new(Mutex::new(
8396                    WorkflowState::new_with_identity(
8397                        history_events.as_ref().clone(),
8398                        context.workflow_id.clone(),
8399                        context.run_id.clone(),
8400                        self.task_queue.clone(),
8401                        task.payload_codec,
8402                        None,
8403                    )
8404                    .map_err(|error| {
8405                        QueryTaskExecutionFailure::new(
8406                            "query_workflow_state_unavailable",
8407                            format!("workflow replay failed before query: {error}"),
8408                            "QueryWorkflowStateUnavailable",
8409                        )
8410                    })?,
8411                ));
8412                let workflow_context = WorkflowContext {
8413                    state: workflow_state,
8414                    cancellation_scope_id: "root".into(),
8415                };
8416                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
8417                let mut cx = TaskContext::from_waker(noop_waker_ref());
8418                let outcome = {
8419                    let _replay = cancellation_replay_clock::ReplayGuard::enter(&workflow_context)
8420                        .map_err(|error| {
8421                            QueryTaskExecutionFailure::new(
8422                                "query_workflow_state_unavailable",
8423                                error.to_string(),
8424                                "QueryWorkflowStateUnavailable",
8425                            )
8426                        })?;
8427                    invocation.future.as_mut().poll(&mut cx)
8428                };
8429                match outcome {
8430                    Poll::Ready(Ok(_)) => {
8431                        workflow_context
8432                            .ensure_history_consumed()
8433                            .map_err(|error| {
8434                                QueryTaskExecutionFailure::new(
8435                                    "query_workflow_state_unavailable",
8436                                    format!("workflow replay failed before query: {error}"),
8437                                    "QueryWorkflowStateUnavailable",
8438                                )
8439                            })?;
8440                    }
8441                    Poll::Ready(Err(error)) => {
8442                        return Err(QueryTaskExecutionFailure::new(
8443                            "query_workflow_state_unavailable",
8444                            format!("workflow replay failed before query: {error}"),
8445                            "QueryWorkflowStateUnavailable",
8446                        ));
8447                    }
8448                    Poll::Pending => {
8449                        let commands = workflow_context.take_commands().map_err(|error| {
8450                            QueryTaskExecutionFailure::new(
8451                                "query_workflow_state_unavailable",
8452                                format!("workflow replay failed before query: {error}"),
8453                                "QueryWorkflowStateUnavailable",
8454                            )
8455                        })?;
8456                        if commands.is_empty()
8457                            && !workflow_context
8458                                .matched_recorded_pending()
8459                                .map_err(|error| {
8460                                    QueryTaskExecutionFailure::new(
8461                                        "query_workflow_state_unavailable",
8462                                        format!("workflow replay failed before query: {error}"),
8463                                        "QueryWorkflowStateUnavailable",
8464                                    )
8465                                })?
8466                        {
8467                            return Err(QueryTaskExecutionFailure::new(
8468                                "query_workflow_state_unavailable",
8469                                "workflow replay yielded without a durable command",
8470                                "QueryWorkflowStateUnavailable",
8471                            ));
8472                        }
8473                    }
8474                }
8475                let state = (invocation.snapshot)().map_err(|error| {
8476                    QueryTaskExecutionFailure::new(
8477                        "query_workflow_state_unavailable",
8478                        format!("cannot snapshot replayed workflow state: {error}"),
8479                        "QueryWorkflowStateUnavailable",
8480                    )
8481                })?;
8482                handler(context, state, args).map_err(|message| {
8483                    QueryTaskExecutionFailure::new(
8484                        "query_workflow_state_unavailable",
8485                        message,
8486                        "QueryWorkflowStateUnavailable",
8487                    )
8488                })?
8489            }
8490        };
8491
8492        future.await.map_err(|error| {
8493            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
8494        })
8495    }
8496
8497    #[cfg(test)]
8498    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
8499        Ok(self.execute_workflow_task_decision(task)?.commands)
8500    }
8501
8502    fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
8503        self.execute_workflow_task_decision_with_cancellation(task, None)
8504    }
8505
8506    fn execute_workflow_task_decision_with_cancellation(
8507        &self,
8508        task: WorkflowTask,
8509        observation: Option<&CancellationRequest>,
8510    ) -> Result<WorkflowTaskDecision> {
8511        match self.prepare_workflow_task_execution(task, observation)? {
8512            PreparedWorkflowTask::Decision(decision) => Ok(decision),
8513            PreparedWorkflowTask::Execution {
8514                task,
8515                context,
8516                mut future,
8517            } => {
8518                let outcome = Self::poll_workflow_future(&context, &mut future)?;
8519                self.finish_workflow_task_execution(&task, &context, outcome)
8520            }
8521        }
8522    }
8523
8524    fn prepare_workflow_task_execution(
8525        &self,
8526        task: WorkflowTask,
8527        observation: Option<&CancellationRequest>,
8528    ) -> Result<PreparedWorkflowTask> {
8529        if (self.allow_cancellation_scope_authoring || self.allow_cancellation_scope_delivery)
8530            && !self.cooperative_cancellation_enabled
8531        {
8532            return Err(Error::CancellationScopeExecutionUnavailable);
8533        }
8534        validate_workflow_task_payloads(&task)?;
8535        cooperative_cancellation::assert_cancellation_scope_replay_supported(
8536            &task.history_events,
8537            self.allow_cancellation_scope_authoring,
8538            self.allow_cancellation_scope_delivery,
8539        )?;
8540        let cancellation_history = if let Some(observation) = observation {
8541            observation.validate_observation()?;
8542            cooperative_cancellation::cancellation_claim(&task)?;
8543            let canonical = CancellationHistory::from_events(
8544                &task.history_events,
8545                task.run_id.as_deref().unwrap_or_default(),
8546                Some(observation),
8547            )?;
8548            if canonical.request_index >= task.history_events.len() {
8549                return Err(Error::InvalidCooperativeCancellation(
8550                    "workflow replay requires the observed canonical request".into(),
8551                ));
8552            }
8553            Some(canonical)
8554        } else {
8555            None
8556        };
8557        if let Some(update_id) = task
8558            .workflow_update_id
8559            .as_deref()
8560            .filter(|update_id| !update_id.is_empty())
8561        {
8562            if self.allow_cancellation_scope_delivery {
8563                return Err(Error::CancellationScopeExecutionUnavailable);
8564            }
8565            CancellationHistory::from_events(
8566                &task.history_events,
8567                task.run_id.as_deref().unwrap_or_default(),
8568                None,
8569            )?;
8570            return self
8571                .execute_update_task(&task, update_id)
8572                .map(WorkflowTaskDecision::without_message_streams)
8573                .map(PreparedWorkflowTask::Decision);
8574        }
8575
8576        let workflow = self
8577            .workflows
8578            .get(&task.workflow_type)
8579            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
8580        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
8581        let resume_signal = decode_resume_signal(&task)?;
8582        let history_budget = WorkflowHistoryBudget {
8583            event_count: task
8584                .total_history_events
8585                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
8586            size_bytes: task.history_size_bytes,
8587            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
8588            pressure: task.history_budget_pressure.clone(),
8589        };
8590        let workflow_command_identity = task
8591            .workflow_command_id
8592            .clone()
8593            .filter(|identity| !identity.is_empty())
8594            .or_else(|| (!task.task_id.is_empty()).then(|| task.task_id.clone()))
8595            .unwrap_or_default();
8596        let mut workflow_state = WorkflowState::new_with_identity_and_scope_delivery(
8597            task.history_events.clone(),
8598            task.workflow_id.clone(),
8599            task.run_id.clone(),
8600            self.task_queue.clone(),
8601            task.payload_codec.clone(),
8602            resume_signal,
8603            self.allow_cancellation_scope_authoring,
8604            self.allow_cancellation_scope_delivery,
8605        )?;
8606        workflow_state.history_budget = history_budget;
8607        workflow_state.workflow_command_identity = workflow_command_identity;
8608        if let Some(canonical) = cancellation_history {
8609            workflow_state.cancellation_history = canonical;
8610            workflow_state.cancellation_delivery_enabled = true;
8611        }
8612        if workflow_state.cancellation_history.request.is_none() {
8613            workflow_state.cancel_requested = task.cancel_requested;
8614        }
8615        let state = Arc::new(Mutex::new(workflow_state));
8616        let ctx = WorkflowContext {
8617            state,
8618            cancellation_scope_id: "root".into(),
8619        };
8620        let future = (workflow.execute)(ctx.clone(), input);
8621        Ok(PreparedWorkflowTask::Execution {
8622            task,
8623            context: ctx,
8624            future,
8625        })
8626    }
8627
8628    fn poll_workflow_future(
8629        ctx: &WorkflowContext,
8630        future: &mut WorkflowFuture,
8631    ) -> Result<Poll<Result<AvroValue>>> {
8632        let mut cx = TaskContext::from_waker(noop_waker_ref());
8633        let _replay = cancellation_replay_clock::ReplayGuard::enter(ctx)?;
8634        Ok(future.as_mut().poll(&mut cx))
8635    }
8636
8637    fn finish_workflow_task_execution(
8638        &self,
8639        task: &WorkflowTask,
8640        ctx: &WorkflowContext,
8641        outcome: Poll<Result<AvroValue>>,
8642    ) -> Result<WorkflowTaskDecision> {
8643        if !ctx
8644            .state
8645            .lock()
8646            .map_err(|_| Error::WorkflowStatePoisoned)?
8647            .local_activity_requests
8648            .is_empty()
8649        {
8650            return Err(Error::CooperativeCancellationUnavailable(
8651                "local_activity_execution_unavailable: unresolved inline local calls require an ordinary workflow worker executor; cooperative execution requires prepared local supervision".into(),
8652            ));
8653        }
8654        let awaiting_canonical_boundary = {
8655            let state = ctx.state.lock().map_err(|_| Error::WorkflowStatePoisoned)?;
8656            state.cancellation_delivery_intent.is_some()
8657                || state.cancellation_scope_opening.is_some()
8658                || state
8659                    .scope_delivery
8660                    .as_ref()
8661                    .is_some_and(|replay| replay.intent.is_some())
8662        };
8663        if awaiting_canonical_boundary {
8664            // A private delivery intent never becomes an application error.
8665            // Even a custom future which ignores a pending durable call cannot
8666            // publish terminal commands before canonical delivery and replay.
8667            if let Poll::Ready(Err(error)) = outcome {
8668                if workflow_task_integrity_error(&error) {
8669                    return Err(error);
8670                }
8671            }
8672            return self.message_stream_decision(&ctx, ctx.take_commands()?);
8673        }
8674
8675        match outcome {
8676            Poll::Ready(Ok(result)) => {
8677                ctx.ensure_history_consumed()?;
8678                let result = encode_typed_envelope(&result, &task.payload_codec)?;
8679                let mut commands = ctx.take_commands()?;
8680                commands.push(json!({
8681                    "type": "complete_workflow",
8682                    "result": result
8683                }));
8684                self.message_stream_decision(&ctx, commands)
8685            }
8686            Poll::Ready(Err(error)) => {
8687                if let Error::ContinueAsNew(request) = error {
8688                    let mut commands = ctx.take_commands()?;
8689                    if let Some(command) = ctx.continue_as_new_command(request)? {
8690                        commands.push(command);
8691                    }
8692                    ctx.ensure_history_consumed()?;
8693                    return self.message_stream_decision(&ctx, commands);
8694                }
8695                if workflow_task_integrity_error(&error) {
8696                    // Replay and protocol failures describe the workflow-task
8697                    // decision itself. Preserve their specific failure reason
8698                    // instead of replacing it with the derivative fact that
8699                    // recorded commands remain unconsumed.
8700                    return Err(error);
8701                }
8702                // A handler error must not hide a committed durable command that
8703                // upgraded workflow code no longer consumes.
8704                ctx.ensure_history_consumed()?;
8705                let mut commands = ctx.take_commands()?;
8706                commands.push(workflow_failure_command(
8707                    &error,
8708                    recorded_activity_failure_boundary(&ctx, &error),
8709                ));
8710                self.message_stream_decision(&ctx, commands)
8711            }
8712            Poll::Pending => {
8713                let commands = ctx.take_commands()?;
8714                if commands.is_empty() && !ctx.matched_recorded_pending()? {
8715                    Err(Error::WorkflowYieldedWithoutCommand)
8716                } else {
8717                    self.message_stream_decision(&ctx, commands)
8718                }
8719            }
8720        }
8721    }
8722
8723    fn message_stream_decision(
8724        &self,
8725        ctx: &WorkflowContext,
8726        commands: Vec<Value>,
8727    ) -> Result<WorkflowTaskDecision> {
8728        if !self.cooperative_cancellation_enabled
8729            && commands.iter().any(|command| {
8730                command["type"] == "schedule_activity"
8731                    && command.get("cancellation_policy").is_some()
8732            })
8733        {
8734            return Err(Error::CooperativeCancellationUnavailable(format!(
8735                "activity_cancellation_policy_not_supported: Rust worker {} must enable cooperative cancellation with worker protocol 1.20 and a compatible Server/Native backend",
8736                self.worker_id,
8737            )));
8738        }
8739        if !self.cooperative_cancellation_enabled
8740            && commands.iter().any(|command| {
8741                command["type"] == "start_child_workflow"
8742                    && (command["parent_close_policy"] == "request_cancellation"
8743                        || matches!(
8744                            command["cancellation_policy"].as_str(),
8745                            Some("try_cancel" | "wait_cancellation_completed")
8746                        ))
8747            })
8748        {
8749            return Err(Error::CooperativeCancellationUnavailable(format!(
8750                "child_cancellation_policy_not_supported: Rust worker {} must enable cooperative cancellation with worker protocol 1.20 and a compatible Server/Native backend",
8751                self.worker_id,
8752            )));
8753        }
8754        let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
8755        let state = ctx.state.lock().map_err(|_| Error::WorkflowStatePoisoned)?;
8756        let scope_delivery = state
8757            .scope_delivery
8758            .as_ref()
8759            .and_then(|replay| replay.intent.clone());
8760        if scope_delivery
8761            .as_ref()
8762            .is_some_and(|intent| commands.len() != intent.command_count)
8763        {
8764            return Err(invalid_recorded_history(
8765                "cancellation_scope_pending_call_escaped",
8766                scope_delivery.as_ref().unwrap().boundary.sequence,
8767                "commands preceding the pending scoped call",
8768                "commands after suspension",
8769                "workflow cannot publish beyond an uncommitted scope delivery",
8770            ));
8771        }
8772        if let Some(opening) = &state.cancellation_scope_opening {
8773            if commands.len() != opening.command_count {
8774                return Err(invalid_recorded_history(
8775                    "cancellation_scope_pending_call_escaped",
8776                    opening.sequence,
8777                    "commands before original scope opening",
8778                    "commands after suspended opening",
8779                    "workflow cannot publish beyond an uncommitted scope opening",
8780                ));
8781            }
8782        }
8783        if state.cancellation_delivery_intent.is_some()
8784            && commands.len() != state.cancellation_delivery_command_count
8785        {
8786            return Err(invalid_recorded_history(
8787                "cooperative_cancellation_pending_call_escaped",
8788                state
8789                    .cancellation_delivery_intent
8790                    .as_ref()
8791                    .expect("pending delivery")
8792                    .sequence,
8793                "only commands preceding the pending cancellation call",
8794                "commands authored after a suspended call",
8795                "workflow code cannot publish work beyond an uncommitted cancellation boundary",
8796            ));
8797        }
8798        Ok(WorkflowTaskDecision {
8799            commands,
8800            message_stream_cursors,
8801            message_stream_waits,
8802            cancellation_delivery: state.cancellation_delivery_intent.clone(),
8803            cancellation_scope_opening: state.cancellation_scope_opening.clone(),
8804            cancellation_scope_delivery: scope_delivery,
8805        })
8806    }
8807
8808    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
8809        if !self.workflows.contains_key(&task.workflow_type) {
8810            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
8811        }
8812
8813        let accepted = task.history_events.iter().rev().find_map(|event| {
8814            (event.event_type == "UpdateAccepted"
8815                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
8816            .then_some(&event.payload)
8817        });
8818        let update_name = accepted
8819            .and_then(|payload| payload.get("update_name"))
8820            .and_then(Value::as_str)
8821            .or(task.update_name.as_deref())
8822            .unwrap_or_default();
8823        let Some(handler) = self
8824            .updates
8825            .get(&task.workflow_type)
8826            .and_then(|handlers| handlers.get(update_name))
8827        else {
8828            return Ok(vec![json!({
8829                "type": "fail_update",
8830                "update_id": update_id,
8831                "message": format!(
8832                    "no update handler is registered for {}.{update_name}",
8833                    task.workflow_type
8834                ),
8835                "exception_type": "UnknownUpdate",
8836                "non_retryable": true,
8837            })]);
8838        };
8839        let arguments = accepted
8840            .and_then(|payload| payload.get("arguments"))
8841            .or(task.arguments.as_ref());
8842        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
8843        let workflow_input_avro_value =
8844            decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
8845        let workflow_input = workflow_input_avro_value.clone().into_json()?;
8846        let signal_events = history_signal_events(&task.history_events, None, &task.payload_codec)?;
8847        let context = QueryContext {
8848            workflow_id: task.workflow_id.clone(),
8849            run_id: task.run_id.clone(),
8850            workflow_type: task.workflow_type.clone(),
8851            run_status: Some("running".to_string()),
8852            workflow_input,
8853            workflow_input_avro_value,
8854            history_events: Arc::new(task.history_events.clone()),
8855            signal_events: Arc::new(signal_events),
8856        };
8857        let mut future = handler(context, arguments);
8858        let mut cx = TaskContext::from_waker(noop_waker_ref());
8859
8860        match future.as_mut().poll(&mut cx) {
8861            Poll::Ready(Ok(result)) => Ok(vec![json!({
8862                "type": "complete_update",
8863                "update_id": update_id,
8864                "result": encode_typed_envelope(&result, &task.payload_codec)?,
8865            })]),
8866            Poll::Ready(Err(error)) => Ok(vec![json!({
8867                "type": "fail_update",
8868                "update_id": update_id,
8869                "message": error.to_string(),
8870                "exception_type": "UpdateFailed",
8871                "non_retryable": true,
8872            })]),
8873            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
8874        }
8875    }
8876
8877    #[cfg(test)]
8878    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
8879        self.execute_session_activity_task(task, None).await
8880    }
8881
8882    async fn execute_session_activity_task(
8883        &self,
8884        task: ActivityTask,
8885        worker_session: Option<WorkerSession>,
8886    ) -> Result<AvroValue> {
8887        validate_activity_task_payloads(&task)?;
8888        if worker_session.is_some() {
8889            if task.lease_owner.as_deref() != Some(self.worker_id.as_str())
8890                || task
8891                    .activity_attempt_id
8892                    .as_deref()
8893                    .or(task.attempt_id.as_deref())
8894                    .is_none_or(str::is_empty)
8895                || matches!((&task.activity_attempt_id,&task.attempt_id),(Some(a),Some(b)) if a != b)
8896            {
8897                return Err(Error::ActivityExecutionAbandoned(
8898                    "session activity requires this worker's exact immutable claim".into(),
8899                ));
8900            }
8901        }
8902
8903        let handler = self
8904            .activities
8905            .get(&task.activity_type)
8906            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
8907        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
8908        let attempt_id = task
8909            .activity_attempt_id
8910            .clone()
8911            .or(task.attempt_id.clone())
8912            .unwrap_or_default();
8913        let lease_owner = task
8914            .lease_owner
8915            .clone()
8916            .unwrap_or_else(|| self.worker_id.clone());
8917        let session = worker_session.clone();
8918        let ctx = ActivityContext {
8919            client: self.client.clone(),
8920            task_id: task.task_id,
8921            activity_attempt_id: attempt_id,
8922            lease_owner,
8923            activity_type: task.activity_type,
8924            attempt_number: task.attempt_number,
8925            task_queue: self.task_queue.clone(),
8926            worker_id: self.worker_id.clone(),
8927            claim_guard: None,
8928            local_heartbeats: None,
8929            worker_session,
8930        };
8931
8932        let callback = handler(ctx, args);
8933        if let Some(session) = session {
8934            tokio::select! {
8935                biased;
8936                _ = session.wait_until_unavailable() => Err(Error::ActivityExecutionAbandoned("worker-session lease or TTL expired, closed or became uncertain".into())),
8937                result = callback => if session.active() { result } else {
8938                    Err(Error::ActivityExecutionAbandoned("worker-session authority ended before callback settlement".into()))
8939                },
8940            }
8941        } else {
8942            callback.await
8943        }
8944    }
8945}
8946
8947fn poller_result(
8948    kind: &str,
8949    result: std::result::Result<Result<()>, tokio::task::JoinError>,
8950) -> Result<()> {
8951    match result {
8952        Ok(result) => result,
8953        Err(error) => Err(Error::WorkerLoop(format!(
8954            "{kind} poller join error: {error}"
8955        ))),
8956    }
8957}
8958
8959fn optional_poller_result(
8960    kind: &str,
8961    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
8962) -> Result<()> {
8963    match result {
8964        Some(result) => poller_result(kind, result),
8965        None => Ok(()),
8966    }
8967}
8968
8969async fn join_pollers(
8970    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
8971    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
8972    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
8973) -> Result<()> {
8974    let mut first_error = None;
8975
8976    if let Some(handle) = workflow_poller {
8977        if let Err(error) = poller_result("workflow", handle.await) {
8978            first_error.get_or_insert(error);
8979        }
8980    }
8981
8982    if let Some(handle) = activity_poller {
8983        if let Err(error) = poller_result("activity", handle.await) {
8984            first_error.get_or_insert(error);
8985        }
8986    }
8987
8988    if let Some(handle) = query_poller {
8989        if let Err(error) = poller_result("query", handle.await) {
8990            first_error.get_or_insert(error);
8991        }
8992    }
8993
8994    if let Some(error) = first_error {
8995        Err(error)
8996    } else {
8997        Ok(())
8998    }
8999}
9000
9001fn default_worker_id() -> String {
9002    let millis = SystemTime::now()
9003        .duration_since(UNIX_EPOCH)
9004        .unwrap_or_default()
9005        .as_millis();
9006    format!("rust-worker-{}-{millis}", std::process::id())
9007}
9008
9009fn percent_encode_path_segment(segment: &str) -> String {
9010    const HEX: &[u8; 16] = b"0123456789ABCDEF";
9011    let mut encoded = String::with_capacity(segment.len());
9012
9013    for byte in segment.bytes() {
9014        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
9015            encoded.push(char::from(byte));
9016        } else {
9017            encoded.push('%');
9018            encoded.push(char::from(HEX[(byte >> 4) as usize]));
9019            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
9020        }
9021    }
9022
9023    encoded
9024}
9025
9026fn unique_request_id(prefix: &str) -> String {
9027    let nanos = SystemTime::now()
9028        .duration_since(UNIX_EPOCH)
9029        .unwrap_or_default()
9030        .as_nanos();
9031    format!("{prefix}-{}-{nanos}", std::process::id())
9032}
9033
9034#[derive(Debug)]
9035struct QueryTaskExecutionFailure {
9036    reason: String,
9037    message: String,
9038    failure_type: String,
9039}
9040
9041impl QueryTaskExecutionFailure {
9042    fn new(
9043        reason: impl Into<String>,
9044        message: impl Into<String>,
9045        failure_type: impl Into<String>,
9046    ) -> Self {
9047        Self {
9048            reason: reason.into(),
9049            message: message.into(),
9050            failure_type: failure_type.into(),
9051        }
9052    }
9053}
9054
9055/// Typed local state owned by one deterministic workflow invocation.
9056///
9057/// Use [`WorkflowInstance::update`] for the same state transitions during
9058/// ordinary execution and replay. A replayed query receives a detached
9059/// immutable `Arc<S>` rather than this mutation-capable handle.
9060#[derive(Clone, Debug)]
9061pub struct WorkflowInstance<S> {
9062    state: Arc<Mutex<S>>,
9063}
9064
9065impl<S> WorkflowInstance<S> {
9066    fn new(state: S) -> Self {
9067        Self {
9068            state: Arc::new(Mutex::new(state)),
9069        }
9070    }
9071
9072    /// Read the current workflow-instance state without changing it.
9073    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
9074        let state = self
9075            .state
9076            .lock()
9077            .map_err(|_| Error::WorkflowStatePoisoned)?;
9078        Ok(reader(&state))
9079    }
9080
9081    /// Apply one deterministic workflow-instance state transition.
9082    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
9083        let mut state = self
9084            .state
9085            .lock()
9086            .map_err(|_| Error::WorkflowStatePoisoned)?;
9087        Ok(transition(&mut state))
9088    }
9089}
9090
9091impl<S: Clone> WorkflowInstance<S> {
9092    fn snapshot(&self) -> Result<S> {
9093        self.read(Clone::clone)
9094    }
9095}
9096
9097#[derive(Clone, Debug, PartialEq)]
9098pub struct MessageStreamMessage {
9099    pub stream_name: String,
9100    pub message_id: String,
9101    pub position: u64,
9102    pub arguments: Vec<AvroValue>,
9103}
9104
9105#[derive(Clone, Debug)]
9106pub struct MessageStream {
9107    ctx: WorkflowContext,
9108    name: String,
9109}
9110
9111impl MessageStream {
9112    /// Wait for one message, then return a bounded currently-available batch.
9113    pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
9114        if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
9115            return Err(Error::Codec(format!(
9116                "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
9117            )));
9118        }
9119        loop {
9120            if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
9121                return Ok(batch);
9122            }
9123
9124            self.ctx.record_message_stream_wait(&self.name)?;
9125            let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
9126            let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
9127            self.ctx.buffer_message_stream_delivery(arguments)?;
9128            if let Some(sequence) = replay_wait_sequence {
9129                self.ctx.buffer_message_stream_history_for_wait(sequence)?;
9130            }
9131        }
9132    }
9133
9134    pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
9135        self.receive(1)
9136            .await?
9137            .into_iter()
9138            .next()
9139            .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
9140    }
9141}
9142
9143#[derive(Clone, Debug)]
9144pub struct WorkflowContext {
9145    state: Arc<Mutex<WorkflowState>>,
9146    cancellation_scope_id: String,
9147}
9148
9149fn valid_memo_key(key: &str) -> bool {
9150    let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
9151
9152    !key.is_empty()
9153        && key.len() <= 64
9154        && (numeric_candidate.is_empty()
9155            || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
9156        && key
9157            .bytes()
9158            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
9159}
9160
9161fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
9162    BASE64
9163        .decode(encode_avro_value(value)?.blob)
9164        .map(|bytes| bytes.len())
9165        .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
9166}
9167
9168fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
9169    let AvroValue::Map(entries) = value else {
9170        return Err(Error::InvalidMemoUpdate(
9171            "entries must serialize to an Avro string-keyed map".to_string(),
9172        ));
9173    };
9174    if require_entries && entries.is_empty() {
9175        return Err(Error::InvalidMemoUpdate(
9176            "at least one entry is required".to_string(),
9177        ));
9178    }
9179    if entries.len() > MAX_MEMO_ENTRIES {
9180        return Err(Error::InvalidMemoUpdate(format!(
9181            "at most {MAX_MEMO_ENTRIES} entries are allowed"
9182        )));
9183    }
9184
9185    for (key, value) in &entries {
9186        if !valid_memo_key(&key) {
9187            return Err(Error::InvalidMemoUpdate(
9188                "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
9189            ));
9190        }
9191        if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
9192            return Err(Error::InvalidMemoUpdate(format!(
9193                "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
9194            )));
9195        }
9196    }
9197
9198    let value = AvroValue::Map(entries);
9199    if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
9200        return Err(Error::InvalidMemoUpdate(format!(
9201            "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
9202        )));
9203    }
9204    Ok(value)
9205}
9206
9207fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
9208    let object = envelope.as_object().ok_or_else(|| {
9209        Error::InvalidMemoUpdate(
9210            "history field must use the public {codec, blob} payload envelope".to_string(),
9211        )
9212    })?;
9213    if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
9214        return Err(Error::InvalidMemoUpdate(
9215            "history field must use exactly the public {codec, blob} payload envelope".to_string(),
9216        ));
9217    }
9218
9219    canonical_memo_entries(
9220        decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
9221        require_entries,
9222    )
9223}
9224
9225impl WorkflowContext {
9226    pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
9227        let name = name.into();
9228        if name.is_empty()
9229            || name.len() > 128
9230            || !name.bytes().all(|byte| {
9231                byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
9232            })
9233        {
9234            return Err(Error::Codec(
9235                "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
9236                    .to_string(),
9237            ));
9238        }
9239        Ok(MessageStream {
9240            ctx: self.clone(),
9241            name,
9242        })
9243    }
9244
9245    fn record_message_stream_wait(&self, name: &str) -> Result<()> {
9246        let mut state = self
9247            .state
9248            .lock()
9249            .map_err(|_| Error::WorkflowStatePoisoned)?;
9250        let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
9251        state
9252            .message_stream_waits
9253            .insert(name.to_string(), position);
9254        Ok(())
9255    }
9256
9257    fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
9258        let mut state = self
9259            .state
9260            .lock()
9261            .map_err(|_| Error::WorkflowStatePoisoned)?;
9262        let cursor = state
9263            .message_stream_cursors
9264            .get(&message.stream_name)
9265            .copied()
9266            .unwrap_or(0);
9267        if message.position <= cursor {
9268            return Ok(());
9269        }
9270        let pending = state
9271            .message_stream_messages
9272            .entry(message.stream_name.clone())
9273            .or_default();
9274        if pending.iter().any(|candidate| {
9275            candidate.position == message.position || candidate.message_id == message.message_id
9276        }) {
9277            return Ok(());
9278        }
9279        pending.push(message);
9280        pending.sort_by_key(|candidate| candidate.position);
9281        Ok(())
9282    }
9283
9284    fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
9285        if let Some(delivery) = decode_message_stream_delivery(arguments)? {
9286            match delivery {
9287                MessageStreamDelivery::Message(message) => {
9288                    let stream_name = message.stream_name.clone();
9289                    self.buffer_message_stream(message)?;
9290                    return Ok(Some(stream_name));
9291                }
9292                MessageStreamDelivery::Cursor {
9293                    stream_name,
9294                    through_position,
9295                } => self.apply_message_stream_cursor(&stream_name, through_position)?,
9296            }
9297        }
9298        Ok(None)
9299    }
9300
9301    fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
9302        let state = self
9303            .state
9304            .lock()
9305            .map_err(|_| Error::WorkflowStatePoisoned)?;
9306        Ok(match state.recorded_commands.get(state.command_cursor) {
9307            Some(RecordedCommand::SignalWait {
9308                sequence,
9309                signal_name,
9310                ..
9311            }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
9312            _ => None,
9313        })
9314    }
9315
9316    fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
9317        let (history, payload_codec) = {
9318            let state = self
9319                .state
9320                .lock()
9321                .map_err(|_| Error::WorkflowStatePoisoned)?;
9322            (
9323                Arc::clone(&state.history_events),
9324                state.payload_codec.clone(),
9325            )
9326        };
9327
9328        let Some(opened_index) = history.iter().position(|event| {
9329            event.event_type == "SignalWaitOpened"
9330                && durable_event_sequence(event) == Some(wait_sequence)
9331                && event.payload.get("signal_name").and_then(Value::as_str)
9332                    == Some(MESSAGE_STREAM_SIGNAL)
9333        }) else {
9334            return Ok(());
9335        };
9336        let boundary_index = history
9337            .iter()
9338            .enumerate()
9339            .skip(opened_index + 1)
9340            .find_map(|(index, event)| {
9341                (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
9342                    && is_authored_command_open_event(event))
9343                .then_some(index)
9344            })
9345            .unwrap_or(history.len());
9346
9347        for event in history[opened_index + 1..boundary_index]
9348            .iter()
9349            .filter(|event| {
9350                event.event_type == "SignalReceived"
9351                    && event.payload.get("signal_name").and_then(Value::as_str)
9352                        == Some(MESSAGE_STREAM_SIGNAL)
9353            })
9354        {
9355            let arguments = decode_signal_event_arguments(event, &payload_codec)?
9356                .into_iter()
9357                .map(AvroValue::into_json)
9358                .collect::<Result<Vec<_>>>()?;
9359            self.buffer_message_stream_delivery(arguments)?;
9360        }
9361        Ok(())
9362    }
9363
9364    fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
9365        let mut state = self
9366            .state
9367            .lock()
9368            .map_err(|_| Error::WorkflowStatePoisoned)?;
9369        let cursor = state
9370            .message_stream_cursors
9371            .entry(name.to_string())
9372            .or_default();
9373        *cursor = (*cursor).max(through_position);
9374        if let Some(pending) = state.message_stream_messages.get_mut(name) {
9375            pending.retain(|message| message.position > through_position);
9376        }
9377        Ok(())
9378    }
9379
9380    fn take_message_stream_batch(
9381        &self,
9382        name: &str,
9383        max_items: usize,
9384    ) -> Result<Option<Vec<MessageStreamMessage>>> {
9385        let mut state = self
9386            .state
9387            .lock()
9388            .map_err(|_| Error::WorkflowStatePoisoned)?;
9389        let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
9390        let pending = state
9391            .message_stream_messages
9392            .entry(name.to_string())
9393            .or_default();
9394        let count = contiguous_message_stream_count(pending, cursor, max_items);
9395        if count == 0 {
9396            return Ok(None);
9397        }
9398        let batch = pending.drain(..count).collect::<Vec<_>>();
9399        let position = batch.last().map(|message| message.position).unwrap_or(0);
9400        state
9401            .message_stream_cursors
9402            .insert(name.to_string(), position);
9403        state.message_stream_waits.remove(name);
9404        Ok(Some(batch))
9405    }
9406
9407    fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
9408        let state = self
9409            .state
9410            .lock()
9411            .map_err(|_| Error::WorkflowStatePoisoned)?;
9412        let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
9413        cursors.sort_by_key(|(name, _)| *name);
9414        let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
9415        waits.sort_by_key(|(name, _)| *name);
9416        Ok((
9417            cursors
9418                .into_iter()
9419                .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
9420                .collect(),
9421            waits
9422                .into_iter()
9423                .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
9424                .collect(),
9425        ))
9426    }
9427    /// Identity of the parent workflow currently being replayed.
9428    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
9429        let state = self
9430            .state
9431            .lock()
9432            .map_err(|_| Error::WorkflowStatePoisoned)?;
9433        Ok(WorkflowIdentity {
9434            workflow_id: state.workflow_id.clone(),
9435            run_id: state.run_id.clone(),
9436        })
9437    }
9438
9439    /// Return the server-published history budget for this workflow task.
9440    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
9441        let state = self
9442            .state
9443            .lock()
9444            .map_err(|_| Error::WorkflowStatePoisoned)?;
9445        Ok(state.history_budget.clone())
9446    }
9447
9448    /// Continue this workflow instance as a fresh run with replacement arguments.
9449    ///
9450    /// Return this value directly from the workflow handler. The worker converts
9451    /// it to the terminal protocol command only after replay has consumed every
9452    /// recorded durable command.
9453    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
9454        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
9455    }
9456
9457    /// Continue as new with optional workflow-type and task-queue overrides.
9458    pub fn continue_as_new_with_options<T: Serialize>(
9459        &self,
9460        options: ContinueAsNewOptions,
9461        args: T,
9462    ) -> Result<Value> {
9463        options.validate()?;
9464        Err(Error::ContinueAsNew(ContinueAsNewRequest {
9465            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
9466            options,
9467        }))
9468    }
9469
9470    pub fn activity<T: Serialize>(
9471        &self,
9472        activity_type: impl Into<String>,
9473        args: T,
9474    ) -> ActivityCall {
9475        self.activity_with_options(activity_type, ActivityOptions::new(), args)
9476    }
9477
9478    pub fn activity_on_queue<T, Q>(
9479        &self,
9480        activity_type: impl Into<String>,
9481        task_queue: Option<Q>,
9482        args: T,
9483    ) -> ActivityCall
9484    where
9485        T: Serialize,
9486        Q: Into<String>,
9487    {
9488        let mut options = ActivityOptions::new();
9489        options.task_queue = task_queue.map(Into::into);
9490        self.activity_with_options(activity_type, options, args)
9491    }
9492
9493    /// Schedule one durable activity with retry, routing, and timeout options.
9494    ///
9495    /// Options are validated before the command is emitted. Once the command is
9496    /// recorded, replay consumes the same activity lifecycle at this command
9497    /// position and never emits a duplicate schedule.
9498    ///
9499    /// ```no_run
9500    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
9501    /// # use std::time::Duration;
9502    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
9503    /// let result = ctx
9504    ///     .activity_with_options(
9505    ///         "charge-card",
9506    ///         ActivityOptions::new()
9507    ///             .task_queue("payments")
9508    ///             .retry_policy(
9509    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
9510    ///                     Duration::from_secs(1),
9511    ///                     2,
9512    ///                     Some(Duration::from_secs(30)),
9513    ///                 ),
9514    ///             )
9515    ///             .start_to_close_timeout(Duration::from_secs(60))
9516    ///             .schedule_to_close_timeout(Duration::from_secs(180))
9517    ///             .heartbeat_timeout(Duration::from_secs(15)),
9518    ///         json!([{"order_id": "order-42"}]),
9519    ///     )
9520    ///     .await;
9521    /// match result {
9522    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
9523    ///         "reason": failure.reason,
9524    ///         "timeout_kind": failure.timeout_kind,
9525    ///     })),
9526    ///     other => other,
9527    /// }
9528    /// # }
9529    /// ```
9530    pub fn activity_with_options<T: Serialize>(
9531        &self,
9532        activity_type: impl Into<String>,
9533        options: ActivityOptions,
9534        args: T,
9535    ) -> ActivityCall {
9536        ActivityCall {
9537            ctx: self.clone(),
9538            activity_type: activity_type.into(),
9539            options,
9540            worker_session: None,
9541            args: Some(AvroValue::from_serialize(&args)),
9542            scheduled: false,
9543            local: false,
9544            local_result: None,
9545            parallel_group_path: Vec::new(),
9546        }
9547    }
9548
9549    pub async fn activity_avro_value<T: Serialize>(
9550        &self,
9551        activity_type: impl Into<String>,
9552        args: T,
9553    ) -> Result<AvroValue> {
9554        let mut call = self.activity(activity_type, args);
9555        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
9556    }
9557
9558    pub async fn activity_avro_value_with_options<T: Serialize>(
9559        &self,
9560        activity_type: impl Into<String>,
9561        options: ActivityOptions,
9562        args: T,
9563    ) -> Result<AvroValue> {
9564        let mut call = self.activity_with_options(activity_type, options, args);
9565        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
9566    }
9567
9568    /// Schedule an activity with a Serde request and decode its Serde result.
9569    pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
9570    where
9571        I: Serialize,
9572        O: DeserializeOwned,
9573    {
9574        self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
9575            .await
9576    }
9577
9578    /// Schedule an activity with options and decode its result into `O`.
9579    ///
9580    /// Both directions use the fixed Avro Value codec. In particular, this
9581    /// method does not deserialize the JSON-safe inspection projection returned
9582    /// by the dynamic [`ActivityCall`] future.
9583    pub async fn activity_typed_with_options<I, O>(
9584        &self,
9585        activity_type: impl Into<String>,
9586        options: ActivityOptions,
9587        args: I,
9588    ) -> Result<O>
9589    where
9590        I: Serialize,
9591        O: DeserializeOwned,
9592    {
9593        let activity_type = activity_type.into();
9594        let encoded = AvroValue::from_serialize(&args).map_err(|error| {
9595            handler_type_error::<I>(
9596                HandlerKind::Activity,
9597                &activity_type,
9598                HandlerValueKind::Input,
9599                error.to_string(),
9600            )
9601        });
9602        let mut call = ActivityCall {
9603            ctx: self.clone(),
9604            activity_type: activity_type.clone(),
9605            options,
9606            worker_session: None,
9607            args: Some(encoded),
9608            scheduled: false,
9609            local: false,
9610            local_result: None,
9611            parallel_group_path: Vec::new(),
9612        };
9613        let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
9614        decode_handler_result(result, HandlerKind::Activity, &activity_type)
9615    }
9616
9617    /// Schedule and join a deterministic activity/child/timer group.
9618    ///
9619    /// Nested groups retain their input shape. Every durable leaf is scheduled
9620    /// before this future yields, results are assembled by declaration order,
9621    /// and a failure returns [`Error::ParallelFailed`] with typed cause,
9622    /// declaration path, stable group metadata, and completed siblings.
9623    pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
9624        ParallelCall::new(self.clone(), operations)
9625    }
9626
9627    /// Alias for [`WorkflowContext::parallel`].
9628    pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
9629        self.parallel(operations)
9630    }
9631
9632    /// Lossless fixed-Avro variant of [`WorkflowContext::parallel`].
9633    pub async fn parallel_avro_value(
9634        &self,
9635        operations: Vec<ParallelOperation>,
9636    ) -> Result<Vec<ParallelAvroResult>> {
9637        let mut call = self.parallel(operations);
9638        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
9639    }
9640
9641    /// Start every durable operation and resume from the one winner persisted
9642    /// by Server. Non-winning operations continue and remain addressable.
9643    pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
9644        let operations = operations
9645            .into_iter()
9646            .enumerate()
9647            .map(|(index, operation)| (SelectionKey::Index(index), operation))
9648            .collect();
9649        SelectCall::new(self.clone(), operations)
9650    }
9651
9652    /// Named-key variant of [`WorkflowContext::select`].
9653    pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
9654    where
9655        K: Into<SelectionKey>,
9656    {
9657        SelectCall::new(
9658            self.clone(),
9659            operations
9660                .into_iter()
9661                .map(|(key, operation)| (key.into(), operation))
9662                .collect(),
9663        )
9664    }
9665
9666    /// Create a workflow-local deterministic compensation registry.
9667    pub fn saga(&self) -> Saga {
9668        Saga::new(self.clone())
9669    }
9670
9671    /// Whether cancellation has reached this replay's authored boundary.
9672    ///
9673    /// Server's current `/cancel` route is terminal and does not set this flag.
9674    /// A canonical service request remains pending until its committed delivery
9675    /// is consumed. Request observation alone does not authorize an exception.
9676    pub fn is_cancellation_requested(&self) -> Result<bool> {
9677        let state = self
9678            .state
9679            .lock()
9680            .map_err(|_| Error::WorkflowStatePoisoned)?;
9681        if self.cancellation_scope_id != "root" && state.scope_delivery.is_some() {
9682            return Ok(state
9683                .scope_delivery
9684                .as_ref()
9685                .unwrap()
9686                .contexts
9687                .contains_key(&self.cancellation_scope_id));
9688        }
9689        Ok(state.cancel_requested
9690            || state
9691                .scope_delivery
9692                .as_ref()
9693                .is_some_and(|replay| replay.contexts.contains_key(&self.cancellation_scope_id)))
9694    }
9695
9696    /// Raise a typed cooperative cancellation at an author-controlled point.
9697    ///
9698    /// Passing this result to [`Saga::finish`] compensates already registered
9699    /// forward steps before the cancellation remains the initiating outcome.
9700    pub fn throw_if_cancellation_requested(&self) -> Result<()> {
9701        let mut state = self
9702            .state
9703            .lock()
9704            .map_err(|_| Error::WorkflowStatePoisoned)?;
9705        if state.cancellation_shield_depth == 0 {
9706            if let Some(replay) = &state.scope_delivery {
9707                if let Some((context, _)) = replay.contexts.get(&self.cancellation_scope_id) {
9708                    let delivered = replay
9709                        .consumed_delivery_for_scope(&self.cancellation_scope_id)
9710                        .ok_or_else(|| {
9711                            Error::InvalidCooperativeCancellation(
9712                                "scoped request lacks its consumed original delivery".into(),
9713                            )
9714                        })?;
9715                    return Err(Error::CancellationScopeRequested(
9716                        CancellationScopeRequested {
9717                            context: context
9718                                .clone()
9719                                .with_replay(Some(Arc::downgrade(&self.state))),
9720                            delivery: delivered.boundary.clone(),
9721                        },
9722                    ));
9723                }
9724            }
9725        }
9726        if state.cancel_requested && state.cancellation_shield_depth == 0 {
9727            if self.cancellation_scope_id != "root" && state.scope_delivery.is_some() {
9728                return Ok(());
9729            }
9730            return Err(state.cancellation_error());
9731        }
9732        Ok(())
9733    }
9734
9735    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
9736        SignalCall {
9737            ctx: self.clone(),
9738            signal_name: signal_name.into(),
9739            runtime_reserved_allowed: false,
9740            opened_wait: false,
9741            matched_pending: false,
9742            parallel_group_path: Vec::new(),
9743        }
9744    }
9745
9746    fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
9747        SignalCall {
9748            ctx: self.clone(),
9749            signal_name: signal_name.into(),
9750            runtime_reserved_allowed: true,
9751            opened_wait: false,
9752            matched_pending: false,
9753            parallel_group_path: Vec::new(),
9754        }
9755    }
9756
9757    pub async fn wait_signal_avro_value(
9758        &self,
9759        signal_name: impl Into<String>,
9760    ) -> Result<Vec<AvroValue>> {
9761        let mut call = self.wait_signal(signal_name);
9762        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
9763    }
9764
9765    /// Return every committed signal argument list with the given name.
9766    ///
9767    /// This history-backed view is deterministic and is intended for
9768    /// condition predicates that must be re-evaluated after a signal while the
9769    /// workflow is blocked on [`WorkflowContext::wait_condition`].
9770    pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
9771        self.signals_avro_value(signal_name)?
9772            .into_iter()
9773            .map(|arguments| {
9774                arguments
9775                    .into_iter()
9776                    .map(AvroValue::into_json)
9777                    .collect::<Result<Vec<_>>>()
9778            })
9779            .collect()
9780    }
9781
9782    /// Lossless fixed Avro Value view of committed signals with the given name.
9783    pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
9784        let state = self
9785            .state
9786            .lock()
9787            .map_err(|_| Error::WorkflowStatePoisoned)?;
9788        state
9789            .history_events
9790            .iter()
9791            .filter(|event| {
9792                event.event_type == "SignalReceived"
9793                    && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
9794            })
9795            .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
9796            .collect()
9797    }
9798
9799    /// Return every committed update argument list with the given name.
9800    ///
9801    /// Accepted and applied records for the same update ID are de-duplicated.
9802    /// A Server task created after an update therefore replays the workflow and
9803    /// re-evaluates an open condition without application polling.
9804    pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
9805        self.updates_avro_value(update_name)?
9806            .into_iter()
9807            .map(|arguments| {
9808                arguments
9809                    .into_iter()
9810                    .map(AvroValue::into_json)
9811                    .collect::<Result<Vec<_>>>()
9812            })
9813            .collect()
9814    }
9815
9816    /// Lossless fixed Avro Value view of committed updates with the given name.
9817    pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
9818        let state = self
9819            .state
9820            .lock()
9821            .map_err(|_| Error::WorkflowStatePoisoned)?;
9822        let mut seen = Vec::new();
9823        let mut updates = Vec::new();
9824        for event in state.history_events.iter() {
9825            if !matches!(
9826                event.event_type.as_str(),
9827                "UpdateAccepted" | "UpdateApplied"
9828            ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
9829                || event.payload.get("arguments").is_none()
9830            {
9831                continue;
9832            }
9833            if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
9834                if seen.iter().any(|recorded| recorded == update_id) {
9835                    continue;
9836                }
9837                seen.push(update_id.to_string());
9838            }
9839            updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
9840        }
9841        Ok(updates)
9842    }
9843
9844    /// Wait for a deterministic predicate to become true or for its durable
9845    /// timeout to elapse.
9846    ///
9847    /// Prefer [`wait_condition!`] for inline predicates so changes to the Rust
9848    /// predicate tokens automatically change the recorded definition
9849    /// fingerprint. Direct callers must provide an equally stable identity in
9850    /// [`ConditionWaitOptions`].
9851    pub fn wait_condition<F>(
9852        &self,
9853        options: ConditionWaitOptions,
9854        predicate: F,
9855    ) -> ConditionWaitCall
9856    where
9857        F: Fn() -> Result<bool> + Send + 'static,
9858    {
9859        ConditionWaitCall {
9860            ctx: self.clone(),
9861            options,
9862            predicate: Box::new(predicate),
9863            occurrence_id: None,
9864            opened_wait: false,
9865            parallel_group_path: Vec::new(),
9866        }
9867    }
9868
9869    /// Wait for server-backed durable time without blocking the worker executor.
9870    ///
9871    /// Polling this future emits one `start_timer` command and yields. The
9872    /// server records the deadline, so neither worker nor server restarts reset
9873    /// the wait. Replay resolves the future only from a `TimerScheduled` and
9874    /// `TimerFired` pair at the same position in the shared durable-command
9875    /// stream, with matching sequence, timer identity, and delay. Sub-second
9876    /// durations round up because protocol deadlines use whole seconds.
9877    ///
9878    /// ```no_run
9879    /// # use durable_workflow::{json, Client, Worker};
9880    /// # use std::time::Duration;
9881    /// # fn configure(client: Client) {
9882    /// let mut worker = Worker::new(client, "rust-workers");
9883    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
9884    ///     ctx.sleep(Duration::from_secs(5)).await?;
9885    ///     Ok(json!({"status": "timer fired"}))
9886    /// });
9887    /// # }
9888    /// ```
9889    pub fn sleep(&self, duration: Duration) -> TimerCall {
9890        let delay_seconds = duration
9891            .as_secs()
9892            .checked_add(u64::from(duration.subsec_nanos() > 0));
9893        TimerCall {
9894            ctx: self.clone(),
9895            delay_seconds,
9896            scheduled: false,
9897            matched_pending: false,
9898            parallel_group_path: Vec::new(),
9899        }
9900    }
9901
9902    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
9903    pub fn start_timer(&self, duration: Duration) -> TimerCall {
9904        self.sleep(duration)
9905    }
9906
9907    /// Evaluate a non-deterministic callback once and durably record its typed value.
9908    ///
9909    /// On replay the callback is not invoked: the value is decoded from the
9910    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
9911    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
9912    /// small values that must remain fixed for the lifetime of a workflow run.
9913    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
9914    where
9915        T: Serialize + DeserializeOwned,
9916        F: FnOnce() -> T,
9917    {
9918        {
9919            let mut state = self
9920                .state
9921                .lock()
9922                .map_err(|_| Error::WorkflowStatePoisoned)?;
9923            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9924                return match recorded {
9925                    RecordedCommand::SideEffect { sequence, value } => {
9926                        state.command_cursor += 1;
9927                        value.deserialize().map_err(|error| {
9928                            Error::NonDeterministicReplay(ReplayFailure::new(
9929                                "side_effect_type_mismatch",
9930                                Some(sequence),
9931                                Some(std::any::type_name::<T>().to_string()),
9932                                Some(error.to_string()),
9933                                "recorded side-effect value is incompatible with the requested Rust type",
9934                            ))
9935                        })
9936                    }
9937                    other => Err(command_mismatch(&other, "side effect")),
9938                };
9939            }
9940        }
9941
9942        let value = callback();
9943        let avro_value = AvroValue::from_serialize(&value)?;
9944        let mut state = self
9945            .state
9946            .lock()
9947            .map_err(|_| Error::WorkflowStatePoisoned)?;
9948        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
9949        state.commands.push(json!({
9950            "type": "record_side_effect",
9951            "result": result,
9952        }));
9953        Ok(value)
9954    }
9955
9956    /// Record or replay a lossless fixed Avro Value side effect.
9957    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
9958    where
9959        F: FnOnce() -> AvroValue,
9960    {
9961        {
9962            let mut state = self
9963                .state
9964                .lock()
9965                .map_err(|_| Error::WorkflowStatePoisoned)?;
9966            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9967                return match recorded {
9968                    RecordedCommand::SideEffect { value, .. } => {
9969                        state.command_cursor += 1;
9970                        Ok(value)
9971                    }
9972                    other => Err(command_mismatch(&other, "side effect")),
9973                };
9974            }
9975        }
9976
9977        let value = callback();
9978        let mut state = self
9979            .state
9980            .lock()
9981            .map_err(|_| Error::WorkflowStatePoisoned)?;
9982        let result = encode_typed_envelope(&value, &state.payload_codec)?;
9983        state.commands.push(json!({
9984            "type": "record_side_effect",
9985            "result": result,
9986        }));
9987        Ok(value)
9988    }
9989
9990    /// Append output items at a deterministic workflow command boundary.
9991    ///
9992    /// Stable idempotency keys are derived from the server-provided durable
9993    /// workflow command identity, command ordinal, and item index. Replay
9994    /// consumes the recorded side effect and never emits another append.
9995    pub fn append_workflow_stream(
9996        &self,
9997        stream_name: impl Into<String>,
9998        items: &[WorkflowStreamAppendItem],
9999        max_pending_items: Option<u64>,
10000    ) -> Result<()> {
10001        if items.is_empty() {
10002            return Err(Error::Codec(
10003                "workflow_stream_items_empty: append requires at least one item".to_string(),
10004            ));
10005        }
10006        if max_pending_items == Some(0) {
10007            return Err(Error::Codec(
10008                "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
10009                    .to_string(),
10010            ));
10011        }
10012        let stream_name = stream_name.into();
10013        if stream_name.is_empty() {
10014            return Err(Error::Codec(
10015                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
10016            ));
10017        }
10018
10019        let mut state = self
10020            .state
10021            .lock()
10022            .map_err(|_| Error::WorkflowStatePoisoned)?;
10023        let command_ordinal = state.workflow_stream_command_counter;
10024        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10025            state.workflow_stream_command_counter += 1;
10026            return match recorded {
10027                RecordedCommand::SideEffect { .. } => {
10028                    state.command_cursor += 1;
10029                    Ok(())
10030                }
10031                other => Err(command_mismatch(&other, "workflow stream append")),
10032            };
10033        }
10034
10035        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
10036        state.workflow_stream_command_counter += 1;
10037        let wire_items = items
10038            .iter()
10039            .enumerate()
10040            .map(|(item_index, item)| {
10041                item.wire_value(Some(format!(
10042                    "dw-stream:{identity}:{command_ordinal}:{item_index}"
10043                )))
10044            })
10045            .collect::<Vec<_>>();
10046        let mut directive = json!({
10047            "operation": "append",
10048            "stream_name": stream_name,
10049            "command_identity": identity,
10050            "command_ordinal": command_ordinal,
10051            "items": wire_items,
10052        });
10053        if let Some(max_pending_items) = max_pending_items {
10054            directive["max_pending_items"] = json!(max_pending_items);
10055        }
10056        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
10057        state.commands.push(json!({
10058            "type": "record_side_effect",
10059            "result": result,
10060            "workflow_stream": directive,
10061        }));
10062        Ok(())
10063    }
10064
10065    /// Close a run-scoped output stream at a deterministic command boundary.
10066    pub fn close_workflow_stream(
10067        &self,
10068        stream_name: impl Into<String>,
10069        retention_seconds: Option<u64>,
10070    ) -> Result<()> {
10071        self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
10072    }
10073
10074    /// Mark a run-scoped output stream errored at a deterministic command boundary.
10075    pub fn error_workflow_stream(
10076        &self,
10077        stream_name: impl Into<String>,
10078        error_reason: impl Into<String>,
10079        retention_seconds: Option<u64>,
10080    ) -> Result<()> {
10081        let error_reason = error_reason.into();
10082        if error_reason.is_empty() {
10083            return Err(Error::Codec(
10084                "workflow_stream_error_invalid: error reason must not be empty".to_string(),
10085            ));
10086        }
10087        self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
10088    }
10089
10090    fn finish_workflow_stream(
10091        &self,
10092        stream_name: String,
10093        error_reason: Option<String>,
10094        retention_seconds: Option<u64>,
10095    ) -> Result<()> {
10096        if stream_name.is_empty() {
10097            return Err(Error::Codec(
10098                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
10099            ));
10100        }
10101        if retention_seconds == Some(0) {
10102            return Err(Error::Codec(
10103                "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
10104            ));
10105        }
10106        let mut state = self
10107            .state
10108            .lock()
10109            .map_err(|_| Error::WorkflowStatePoisoned)?;
10110        let command_ordinal = state.workflow_stream_command_counter;
10111        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10112            state.workflow_stream_command_counter += 1;
10113            return match recorded {
10114                RecordedCommand::SideEffect { .. } => {
10115                    state.command_cursor += 1;
10116                    Ok(())
10117                }
10118                other => Err(command_mismatch(&other, "workflow stream close")),
10119            };
10120        }
10121        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
10122        state.workflow_stream_command_counter += 1;
10123        let mut directive = json!({
10124            "operation": if error_reason.is_some() { "error" } else { "close" },
10125            "stream_name": stream_name,
10126            "command_identity": identity,
10127            "command_ordinal": command_ordinal,
10128        });
10129        if let Some(error_reason) = error_reason {
10130            directive["error_reason"] = json!(error_reason);
10131        }
10132        if let Some(retention_seconds) = retention_seconds {
10133            directive["retention_seconds"] = json!(retention_seconds);
10134        }
10135        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
10136        state.commands.push(json!({
10137            "type": "record_side_effect",
10138            "result": result,
10139            "workflow_stream": directive,
10140        }));
10141        Ok(())
10142    }
10143
10144    fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
10145        let identity = state.workflow_command_identity.as_str();
10146        if identity.is_empty() {
10147            return Err(Error::MissingWorkflowCommandIdentity);
10148        }
10149        Ok(identity)
10150    }
10151
10152    /// Validate, emit, or replay a typed workflow search-attribute update.
10153    ///
10154    /// The command is non-blocking within a workflow decision, but its
10155    /// `SearchAttributesUpserted` event occupies the same deterministic command
10156    /// stream as activities, timers, conditions, and other durable operations.
10157    pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
10158        update.validate()?;
10159        let (attributes, attribute_types) = update.into_wire_parts();
10160        let mut state = self
10161            .state
10162            .lock()
10163            .map_err(|_| Error::WorkflowStatePoisoned)?;
10164
10165        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10166            return match recorded {
10167                RecordedCommand::SearchAttributes {
10168                    sequence,
10169                    attributes: recorded_attributes,
10170                    attribute_types: recorded_attribute_types,
10171                } => {
10172                    if recorded_attributes != attributes {
10173                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10174                            "search_attribute_value_mismatch",
10175                            Some(sequence),
10176                            Some(recorded_attributes.to_string()),
10177                            Some(attributes.to_string()),
10178                            "search-attribute values differ from the recorded durable command",
10179                        )));
10180                    }
10181                    if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
10182                        if recorded_types != attribute_types {
10183                            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10184                                "search_attribute_type_mismatch",
10185                                Some(sequence),
10186                                Some(json!(recorded_types).to_string()),
10187                                Some(json!(attribute_types).to_string()),
10188                                "search-attribute declared types differ from the recorded durable command",
10189                            )));
10190                        }
10191                    }
10192                    state.command_cursor += 1;
10193                    Ok(())
10194                }
10195                other => Err(command_mismatch(&other, "search-attribute update")),
10196            };
10197        }
10198
10199        let mut command = serde_json::Map::from_iter([
10200            ("type".to_string(), json!("upsert_search_attributes")),
10201            ("attributes".to_string(), attributes),
10202        ]);
10203        if !attribute_types.is_empty() {
10204            command.insert("attribute_types".to_string(), json!(attribute_types));
10205        }
10206        state.commands.push(Value::Object(command));
10207        Ok(())
10208    }
10209
10210    /// Record a UUIDv4 once and return the same UUID on every replay.
10211    pub fn uuid_v4(&self) -> Result<Uuid> {
10212        self.side_effect(Uuid::new_v4)
10213    }
10214
10215    /// Select the newest supported version for a change, or replay the version
10216    /// already committed for that stable change ID.
10217    ///
10218    /// An unmarked old history that reached its next durable operation, or
10219    /// already completed, selects `-1` when that version is supported. This
10220    /// leaves the original operation available and writes no additional marker.
10221    /// Consistent markers written by older workers retain their original ordered
10222    /// positions. Repeated calls consume a matching marker at the current cursor.
10223    /// Conflicting recorded decisions and unsupported ranges remain replay faults.
10224    pub fn get_version(
10225        &self,
10226        change_id: impl Into<String>,
10227        min_supported: i32,
10228        max_supported: i32,
10229    ) -> Result<i32> {
10230        let change_id = change_id.into();
10231        if change_id.trim().is_empty() {
10232            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10233                "version_change_id_invalid",
10234                None,
10235                Some("non-empty change ID".to_string()),
10236                Some(change_id),
10237                "version markers require a stable non-empty change ID",
10238            )));
10239        }
10240        if min_supported > max_supported {
10241            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10242                "version_range_invalid",
10243                None,
10244                Some("min_supported <= max_supported".to_string()),
10245                Some(format!("{min_supported}..={max_supported}")),
10246                "version marker supported range is invalid",
10247            )));
10248        }
10249
10250        let mut state = self
10251            .state
10252            .lock()
10253            .map_err(|_| Error::WorkflowStatePoisoned)?;
10254        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
10255            if let Some(RecordedCommand::VersionMarker {
10256                sequence: alias_sequence,
10257                change_id: alias_change_id,
10258                version: alias_version,
10259            }) = state.recorded_commands.get(state.command_cursor)
10260            {
10261                if alias_change_id == &change_id {
10262                    if *alias_version != version {
10263                        return Err(invalid_recorded_history(
10264                            "version_marker_conflict",
10265                            *alias_sequence,
10266                            &format!("selected version {version} for change ID {change_id:?}"),
10267                            &format!("recorded version {alias_version}"),
10268                            "recorded marker conflicts with the decision already selected during replay",
10269                        ));
10270                    }
10271                    ensure_version_supported(
10272                        &change_id,
10273                        version,
10274                        min_supported,
10275                        max_supported,
10276                        *alias_sequence,
10277                    )?;
10278                    // Older workers could record the same decision again. Keep
10279                    // each physical boundary without skipping intervening work.
10280                    state.command_cursor += 1;
10281                    return Ok(version);
10282                }
10283            }
10284            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
10285            return Ok(version);
10286        }
10287
10288        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10289            return match recorded {
10290                RecordedCommand::VersionMarker {
10291                    sequence,
10292                    change_id: recorded_change_id,
10293                    version,
10294                    ..
10295                } => {
10296                    if recorded_change_id != change_id {
10297                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10298                            "version_change_id_mismatch",
10299                            Some(sequence),
10300                            Some(recorded_change_id),
10301                            Some(change_id),
10302                            "recorded version marker change ID differs from current workflow code",
10303                        )));
10304                    }
10305                    ensure_version_supported(
10306                        &change_id,
10307                        version,
10308                        min_supported,
10309                        max_supported,
10310                        sequence,
10311                    )?;
10312                    state.command_cursor += 1;
10313                    state.version_markers.insert(change_id, (version, sequence));
10314                    Ok(version)
10315                }
10316                other => {
10317                    let version = -1;
10318                    let sequence = other.sequence();
10319                    ensure_version_supported(
10320                        &change_id,
10321                        version,
10322                        min_supported,
10323                        max_supported,
10324                        sequence,
10325                    )?;
10326                    // The existing operation proves this patch site was reached
10327                    // before its marker existed. Keep its original cursor.
10328                    state.version_markers.insert(change_id, (version, sequence));
10329                    Ok(version)
10330                }
10331            };
10332        }
10333
10334        if state
10335            .history_events
10336            .iter()
10337            .any(|event| event.event_type == "WorkflowCompleted")
10338        {
10339            let version = -1;
10340            ensure_version_supported(&change_id, version, min_supported, max_supported, 0)?;
10341            state.version_markers.insert(change_id, (version, 0));
10342            return Ok(version);
10343        }
10344
10345        let version = max_supported;
10346        state.commands.push(json!({
10347            "type": "record_version_marker",
10348            "change_id": change_id,
10349            "version": version,
10350            "min_supported": min_supported,
10351            "max_supported": max_supported,
10352        }));
10353        // Sequence numbers are assigned by the server. Zero identifies a marker
10354        // selected in this uncommitted decision batch for duplicate-call checks.
10355        state.version_markers.insert(change_id, (version, 0));
10356        Ok(version)
10357    }
10358
10359    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
10360    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
10361        Ok(self.get_version(change_id, -1, 1)? == 1)
10362    }
10363
10364    /// Keep a patch marker in history after the legacy branch has been removed.
10365    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
10366        self.get_version(change_id, -1, 1).map(|_| ())
10367    }
10368
10369    /// Merge non-indexed workflow memo metadata through durable history.
10370    ///
10371    /// Avro `null` deletes a key. The SDK encodes the complete patch in the
10372    /// public Avro payload envelope consumed by Server and Cloud runtimes.
10373    pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
10374        let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
10375        let mut state = self
10376            .state
10377            .lock()
10378            .map_err(|_| Error::WorkflowStatePoisoned)?;
10379
10380        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10381            return match recorded {
10382                RecordedCommand::Memo {
10383                    sequence,
10384                    entries: recorded_entries,
10385                } => {
10386                    if recorded_entries != entries {
10387                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10388                            "memo_update_mismatch",
10389                            Some(sequence),
10390                            Some(format!("{recorded_entries:?}")),
10391                            Some(format!("{entries:?}")),
10392                            "recorded memo entries differ from the current workflow update",
10393                        )));
10394                    }
10395                    state.command_cursor += 1;
10396                    Ok(())
10397                }
10398                other => Err(command_mismatch(&other, "memo upsert")),
10399            };
10400        }
10401
10402        let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
10403        state.commands.push(json!({
10404            "type": "upsert_memo",
10405            "entries": entries_envelope,
10406        }));
10407        Ok(())
10408    }
10409
10410    /// Start a named durable child on an explicit queue and await its result.
10411    ///
10412    /// The command is recorded in the parent's sequence-ordered durable command
10413    /// stream. Replay keeps a scheduled child pending without emitting another
10414    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
10415    /// values preserve the history payload codec and include both sides of the
10416    /// durable relationship; failures are returned as
10417    /// [`Error::ChildWorkflowFailed`].
10418    ///
10419    /// ```no_run
10420    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
10421    /// # fn configure(client: Client) {
10422    /// let mut worker = Worker::new(client, "parent-workers");
10423    /// worker.register_workflow("order-parent", |ctx, _input| async move {
10424    ///     let child = ctx
10425    ///         .start_child_workflow(
10426    ///             "fulfil-order",
10427    ///             ChildWorkflowOptions::new("fulfilment-workers")
10428    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
10429    ///             json!([{"order_id": "order-42"}]),
10430    ///         )
10431    ///         .await?;
10432    ///     Ok(child.result)
10433    /// });
10434    /// # }
10435    /// ```
10436    pub fn start_child_workflow<T: Serialize>(
10437        &self,
10438        workflow_type: impl Into<String>,
10439        options: ChildWorkflowOptions,
10440        args: T,
10441    ) -> ChildWorkflowCall {
10442        ChildWorkflowCall {
10443            ctx: self.clone(),
10444            workflow_type: workflow_type.into(),
10445            options,
10446            args: Some(AvroValue::from_serialize(&args)),
10447            scheduled: false,
10448            matched_pending: false,
10449            parallel_group_path: Vec::new(),
10450        }
10451    }
10452
10453    pub async fn start_child_workflow_avro_value<T: Serialize>(
10454        &self,
10455        workflow_type: impl Into<String>,
10456        options: ChildWorkflowOptions,
10457        args: T,
10458    ) -> Result<ChildWorkflowAvroResult> {
10459        let mut call = self.start_child_workflow(workflow_type, options, args);
10460        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
10461    }
10462
10463    fn take_commands(&self) -> Result<Vec<Value>> {
10464        let mut state = self
10465            .state
10466            .lock()
10467            .map_err(|_| Error::WorkflowStatePoisoned)?;
10468        Ok(std::mem::take(&mut state.commands))
10469    }
10470
10471    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
10472        let mut state = self
10473            .state
10474            .lock()
10475            .map_err(|_| Error::WorkflowStatePoisoned)?;
10476
10477        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10478            return Err(command_mismatch(&recorded, "continue as new"));
10479        }
10480        if state.recorded_continue_as_new_sequence.is_some() {
10481            state.continue_as_new_consumed = true;
10482            return Ok(None);
10483        }
10484
10485        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
10486        let mut command = serde_json::Map::from_iter([
10487            ("type".to_string(), json!("continue_as_new")),
10488            ("arguments".to_string(), arguments),
10489            ("queue".to_string(), json!(state.task_queue.clone())),
10490        ]);
10491        if let Some(workflow_type) = request.options.workflow_type {
10492            command.insert("workflow_type".to_string(), json!(workflow_type));
10493        }
10494        if let Some(task_queue) = request.options.task_queue {
10495            command.insert("queue".to_string(), json!(task_queue));
10496        }
10497        Ok(Some(Value::Object(command)))
10498    }
10499
10500    fn matched_recorded_pending(&self) -> Result<bool> {
10501        let state = self
10502            .state
10503            .lock()
10504            .map_err(|_| Error::WorkflowStatePoisoned)?;
10505        Ok(state.matched_recorded_pending)
10506    }
10507
10508    fn ensure_history_consumed(&self) -> Result<()> {
10509        let state = self
10510            .state
10511            .lock()
10512            .map_err(|_| Error::WorkflowStatePoisoned)?;
10513        if let Some(replay) = &state.scope_delivery {
10514            if let Some((&sequence, _)) = replay
10515                .canonical
10516                .deliveries
10517                .iter()
10518                .find(|(sequence, _)| !replay.consumed.contains(sequence))
10519            {
10520                return Err(invalid_recorded_history(
10521                    "cancellation_scope_delivery_unconsumed",
10522                    sequence,
10523                    "original scoped delivery",
10524                    "workflow completion",
10525                    "workflow completed without consuming its committed scope delivery",
10526                ));
10527            }
10528        }
10529        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
10530            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10531                "recorded_commands_unconsumed",
10532                Some(command.sequence()),
10533                Some(command.shape().to_string()),
10534                Some("workflow completion".to_string()),
10535                "workflow completed before consuming all recorded durable commands",
10536            )));
10537        }
10538        if let Some(delivery) = state
10539            .cancellation_history
10540            .delivery
10541            .as_ref()
10542            .filter(|_| !state.cancellation_consumed)
10543        {
10544            return Err(invalid_recorded_history(
10545                "cooperative_cancellation_unconsumed",
10546                delivery.sequence,
10547                "committed cancellation call",
10548                "workflow completion",
10549                "workflow completed without reaching its committed cancellation boundary",
10550            ));
10551        }
10552        if let Some(sequence) = state
10553            .recorded_continue_as_new_sequence
10554            .filter(|_| !state.continue_as_new_consumed)
10555        {
10556            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10557                "recorded_continue_as_new_unconsumed",
10558                Some(sequence),
10559                Some("continue as new".to_string()),
10560                Some("workflow completion".to_string()),
10561                "workflow completed without consuming its recorded continue-as-new transition",
10562            )));
10563        }
10564        Ok(())
10565    }
10566}
10567
10568fn contiguous_message_stream_count(
10569    pending: &[MessageStreamMessage],
10570    cursor: u64,
10571    max_items: usize,
10572) -> usize {
10573    pending
10574        .iter()
10575        .take(max_items)
10576        .enumerate()
10577        .take_while(|(offset, message)| {
10578            u64::try_from(*offset)
10579                .ok()
10580                .and_then(|offset| cursor.checked_add(offset + 1))
10581                == Some(message.position)
10582        })
10583        .count()
10584}
10585
10586fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
10587    matches!(
10588        event.event_type.as_str(),
10589        "ActivityScheduled"
10590            | "TimerScheduled"
10591            | "ChildWorkflowScheduled"
10592            | "SignalWaitOpened"
10593            | "ConditionWaitOpened"
10594            | "SearchAttributesUpserted"
10595            | "SideEffectRecorded"
10596            | "VersionMarkerRecorded"
10597            | "MemoUpserted"
10598            | "WorkflowContinuedAsNew"
10599    )
10600}
10601
10602#[derive(Debug)]
10603struct WorkflowState {
10604    workflow_id: Option<String>,
10605    run_id: Option<String>,
10606    task_queue: String,
10607    payload_codec: String,
10608    history_events: Arc<Vec<HistoryEvent>>,
10609    history_budget: WorkflowHistoryBudget,
10610    cancel_requested: bool,
10611    cancellation_history: CancellationHistory,
10612    cancellation_consumed: bool,
10613    cancellation_clock: Option<cancellation_replay_clock::ReplayClock>,
10614    cancellation_shield_depth: u64,
10615    cancellation_delivery_enabled: bool,
10616    cancellation_delivery_intent: Option<CancellationDelivery>,
10617    cancellation_delivery_command_count: usize,
10618    allow_cancellation_scope_authoring: bool,
10619    cancellation_scope_opening: Option<cancellation_scope::CancellationScopeOpening>,
10620    cancellation_scope_memberships: BTreeMap<u64, String>,
10621    scope_delivery: Option<cancellation_scope_replay::ScopeReplay>,
10622    resume_signal: Option<ResumeSignal>,
10623    recorded_commands: Vec<RecordedCommand>,
10624    selection_markers: Vec<SelectionMarker>,
10625    selection_marker_cursor: usize,
10626    cancelled_selection_members: Vec<SelectionCancellation>,
10627    recorded_continue_as_new_sequence: Option<u64>,
10628    continue_as_new_consumed: bool,
10629    command_cursor: usize,
10630    condition_wait_occurrence_counter: u64,
10631    matched_recorded_pending: bool,
10632    version_markers: HashMap<String, (i32, u64)>,
10633    workflow_command_identity: String,
10634    workflow_stream_command_counter: u64,
10635    commands: Vec<Value>,
10636    local_activity_requests: Vec<local_activity::Request>,
10637    message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
10638    message_stream_cursors: HashMap<String, u64>,
10639    message_stream_waits: HashMap<String, u64>,
10640}
10641
10642impl WorkflowState {
10643    #[cfg(test)]
10644    fn new(
10645        history: Vec<HistoryEvent>,
10646        task_queue: String,
10647        payload_codec: String,
10648        resume_signal: Option<ResumeSignal>,
10649    ) -> Result<Self> {
10650        Self::new_with_identity(
10651            history,
10652            None,
10653            None,
10654            task_queue,
10655            payload_codec,
10656            resume_signal,
10657        )
10658    }
10659
10660    fn new_with_identity(
10661        history: Vec<HistoryEvent>,
10662        workflow_id: Option<String>,
10663        run_id: Option<String>,
10664        task_queue: String,
10665        payload_codec: String,
10666        resume_signal: Option<ResumeSignal>,
10667    ) -> Result<Self> {
10668        Self::new_with_identity_and_scopes(
10669            history,
10670            workflow_id,
10671            run_id,
10672            task_queue,
10673            payload_codec,
10674            resume_signal,
10675            false,
10676        )
10677    }
10678
10679    fn new_with_identity_and_scopes(
10680        history: Vec<HistoryEvent>,
10681        workflow_id: Option<String>,
10682        run_id: Option<String>,
10683        task_queue: String,
10684        payload_codec: String,
10685        resume_signal: Option<ResumeSignal>,
10686        allow_cancellation_scope_authoring: bool,
10687    ) -> Result<Self> {
10688        Self::new_with_identity_and_scope_delivery(
10689            history,
10690            workflow_id,
10691            run_id,
10692            task_queue,
10693            payload_codec,
10694            resume_signal,
10695            allow_cancellation_scope_authoring,
10696            false,
10697        )
10698    }
10699
10700    fn new_with_identity_and_scope_delivery(
10701        history: Vec<HistoryEvent>,
10702        workflow_id: Option<String>,
10703        run_id: Option<String>,
10704        task_queue: String,
10705        payload_codec: String,
10706        resume_signal: Option<ResumeSignal>,
10707        allow_cancellation_scope_authoring: bool,
10708        allow_cancellation_scope_delivery: bool,
10709    ) -> Result<Self> {
10710        cooperative_cancellation::assert_cancellation_scope_replay_supported(
10711            &history,
10712            allow_cancellation_scope_authoring,
10713            allow_cancellation_scope_delivery,
10714        )?;
10715        let scopes = if allow_cancellation_scope_authoring {
10716            cancellation_scope::CancellationScopeHistory::read(
10717                &history,
10718                run_id.as_deref().unwrap_or_default(),
10719            )?
10720        } else {
10721            cancellation_scope::CancellationScopeHistory::default()
10722        };
10723        let cancellation_history = CancellationHistory::from_events(
10724            &history,
10725            run_id.as_deref().unwrap_or_default(),
10726            None,
10727        )?;
10728        if allow_cancellation_scope_authoring
10729            && cancellation_history.request.is_some()
10730            && !allow_cancellation_scope_delivery
10731        {
10732            return Err(Error::CancellationScopeExecutionUnavailable);
10733        }
10734        let scope_delivery = if allow_cancellation_scope_delivery {
10735            Some(cancellation_scope_replay::ScopeReplay::read(
10736                &history,
10737                &scopes,
10738                run_id.as_deref().unwrap_or_default(),
10739                workflow_id.as_deref().unwrap_or_default(),
10740            )?)
10741        } else {
10742            None
10743        };
10744        let mut recorded_commands = cancellation_history.bind_commands(recorded_commands(
10745            &history,
10746            &payload_codec,
10747            WorkflowIdentity {
10748                workflow_id: workflow_id.clone(),
10749                run_id: run_id.clone(),
10750            },
10751        )?)?;
10752        for (sequence, opening) in &scopes.openings {
10753            if recorded_commands
10754                .iter()
10755                .any(|command| command.sequence() == *sequence)
10756            {
10757                return Err(invalid_recorded_history(
10758                    "invalid_cancellation_scope_history",
10759                    *sequence,
10760                    "distinct authored scope opening",
10761                    "operation collision",
10762                    "scope opening collides with a command",
10763                ));
10764            }
10765            recorded_commands.push(RecordedCommand::CancellationScope {
10766                sequence: *sequence,
10767                scope_id: opening.scope_id.clone(),
10768                parent_scope_id: opening.parent_scope_id.clone(),
10769                shield_parent: opening.shield_parent,
10770            });
10771        }
10772        recorded_commands.sort_by_key(RecordedCommand::sequence);
10773        if let Some(replay) = &scope_delivery {
10774            replay.bind_commands(&mut recorded_commands)?;
10775        }
10776        let selection_markers = recorded_selection_markers(&history)?;
10777        let cancelled_selection_members = recorded_selection_cancellations(&history)?;
10778        let recorded_continue_as_new = history
10779            .iter()
10780            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
10781            .collect::<Vec<_>>();
10782        if recorded_continue_as_new.len() > 1 {
10783            return Err(invalid_recorded_history(
10784                "duplicate_continue_as_new_transition",
10785                recorded_continue_as_new
10786                    .last()
10787                    .and_then(|event| durable_event_sequence(event))
10788                    .unwrap_or(0),
10789                "one WorkflowContinuedAsNew event",
10790                &format!(
10791                    "{} WorkflowContinuedAsNew events",
10792                    recorded_continue_as_new.len()
10793                ),
10794                "workflow history records one continue-as-new transition more than once",
10795            ));
10796        }
10797        let recorded_continue_as_new_sequence = recorded_continue_as_new
10798            .first()
10799            .map(|event| {
10800                durable_event_sequence(event).ok_or_else(|| {
10801                    Error::NonDeterministicReplay(ReplayFailure::new(
10802                        "continue_as_new_sequence_missing",
10803                        None,
10804                        Some("recorded transition sequence".to_string()),
10805                        Some("missing sequence".to_string()),
10806                        "WorkflowContinuedAsNew history is missing its recorded sequence",
10807                    ))
10808                })
10809            })
10810            .transpose()?;
10811        let mut message_stream_cursors = HashMap::new();
10812        for event in &history {
10813            if !matches!(
10814                event.event_type.as_str(),
10815                "SignalReceived" | "SignalApplied"
10816            ) || event.payload.get("signal_name").and_then(Value::as_str)
10817                != Some(MESSAGE_STREAM_SIGNAL)
10818            {
10819                continue;
10820            }
10821            let arguments = decode_signal_event_arguments(event, &payload_codec)?;
10822            if arguments.len() != 1 {
10823                continue;
10824            }
10825            let envelope = arguments[0].clone().into_json()?;
10826            let Some(envelope) = envelope.as_object() else {
10827                continue;
10828            };
10829            if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
10830            {
10831                continue;
10832            }
10833            let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
10834                continue;
10835            };
10836            let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
10837            else {
10838                continue;
10839            };
10840            let cursor = message_stream_cursors
10841                .entry(stream_name.to_string())
10842                .or_insert(0);
10843            *cursor = (*cursor).max(through_position);
10844        }
10845        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
10846        let cancel_requested = history.iter().any(|event| {
10847            matches!(
10848                event.event_type.as_str(),
10849                "WorkflowCancellationRequested" | "WorkflowCancelRequested"
10850            )
10851        });
10852        let cancellation_clock = cancellation_history
10853            .request
10854            .as_ref()
10855            .and_then(|request| request.context.as_ref())
10856            .map(|_| {
10857                cancellation_replay_clock::ReplayClock::new(
10858                    &history,
10859                    &recorded_commands,
10860                    cancellation_history.delivery_index,
10861                )
10862            })
10863            .or_else(|| {
10864                scope_delivery.as_ref().map(|_| {
10865                    cancellation_replay_clock::ReplayClock::new(&history, &recorded_commands, None)
10866                })
10867            });
10868        Ok(Self {
10869            workflow_command_identity: String::new(),
10870            workflow_stream_command_counter: 0,
10871            workflow_id,
10872            run_id,
10873            task_queue,
10874            payload_codec,
10875            history_events: Arc::new(history),
10876            history_budget: WorkflowHistoryBudget {
10877                event_count,
10878                ..WorkflowHistoryBudget::default()
10879            },
10880            cancel_requested,
10881            cancellation_history,
10882            cancellation_consumed: false,
10883            cancellation_clock,
10884            cancellation_shield_depth: 0,
10885            cancellation_delivery_enabled: false,
10886            cancellation_delivery_intent: None,
10887            cancellation_delivery_command_count: 0,
10888            allow_cancellation_scope_authoring,
10889            cancellation_scope_opening: None,
10890            cancellation_scope_memberships: scopes.memberships,
10891            scope_delivery,
10892            resume_signal,
10893            recorded_commands,
10894            selection_markers,
10895            selection_marker_cursor: 0,
10896            cancelled_selection_members,
10897            recorded_continue_as_new_sequence,
10898            continue_as_new_consumed: false,
10899            command_cursor: 0,
10900            condition_wait_occurrence_counter: 0,
10901            matched_recorded_pending: false,
10902            version_markers: HashMap::new(),
10903            commands: Vec::new(),
10904            local_activity_requests: Vec::new(),
10905            message_stream_messages: HashMap::new(),
10906            message_stream_cursors,
10907            message_stream_waits: HashMap::new(),
10908        })
10909    }
10910}
10911
10912enum MessageStreamDelivery {
10913    Message(MessageStreamMessage),
10914    Cursor {
10915        stream_name: String,
10916        through_position: u64,
10917    },
10918}
10919
10920fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
10921    if arguments.len() != 1 {
10922        return Ok(None);
10923    }
10924    let envelope = arguments
10925        .into_iter()
10926        .next()
10927        .expect("one argument was checked");
10928    let Some(envelope) = envelope.as_object() else {
10929        return Ok(None);
10930    };
10931    let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
10932        return Ok(None);
10933    };
10934    if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
10935        let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
10936        else {
10937            return Ok(None);
10938        };
10939        return Ok(Some(MessageStreamDelivery::Cursor {
10940            stream_name: stream_name.to_string(),
10941            through_position,
10942        }));
10943    }
10944    if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
10945        return Ok(None);
10946    }
10947    let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
10948        return Ok(None);
10949    };
10950    let Some(position) = envelope
10951        .get("position")
10952        .and_then(Value::as_u64)
10953        .filter(|value| *value > 0)
10954    else {
10955        return Ok(None);
10956    };
10957    let Some(payload_envelope) = envelope.get("payload_envelope") else {
10958        return Ok(None);
10959    };
10960    let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
10961    else {
10962        return Ok(None);
10963    };
10964    let decoded = decode_avro_value(&payload_envelope)?;
10965    let AvroValue::Array(values) = decoded else {
10966        return Ok(None);
10967    };
10968    Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
10969        stream_name: stream_name.to_string(),
10970        message_id: message_id.to_string(),
10971        position,
10972        arguments: values,
10973    })))
10974}
10975
10976#[derive(Clone, Debug)]
10977enum RecordedCommand {
10978    CancellationScope {
10979        sequence: u64,
10980        scope_id: String,
10981        parent_scope_id: String,
10982        shield_parent: bool,
10983    },
10984    CancellationGroup {
10985        sequence: u64,
10986        span: u64,
10987        original: Vec<RecordedCommand>,
10988    },
10989    CancellationBoundary {
10990        sequence: u64,
10991        call_kind: CancellationCallKind,
10992        original: Option<Box<RecordedCommand>>,
10993    },
10994    Activity {
10995        sequence: u64,
10996        activity_type: Option<String>,
10997        cancellation_policy: String,
10998        options: Option<RecordedActivityOptions>,
10999        outcome: Option<ActivityOutcome>,
11000        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
11001    },
11002    Timer {
11003        sequence: u64,
11004        delay_seconds: u64,
11005        fired: bool,
11006        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
11007    },
11008    ChildWorkflow {
11009        sequence: u64,
11010        workflow_type: Option<String>,
11011        policies: RecordedChildPolicies,
11012        outcome: Option<ChildWorkflowOutcome>,
11013        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
11014    },
11015    SignalWait {
11016        sequence: u64,
11017        signal_name: String,
11018        value: Option<Vec<AvroValue>>,
11019        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
11020    },
11021    ConditionWait {
11022        sequence: u64,
11023        occurrence_id: String,
11024        condition_key: Option<String>,
11025        predicate_identity: String,
11026        timeout_seconds: Option<u64>,
11027        result: Option<ConditionWaitResult>,
11028        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
11029    },
11030    SearchAttributes {
11031        sequence: u64,
11032        attributes: Value,
11033        attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
11034    },
11035    SideEffect {
11036        sequence: u64,
11037        value: AvroValue,
11038    },
11039    VersionMarker {
11040        sequence: u64,
11041        change_id: String,
11042        version: i32,
11043    },
11044    Memo {
11045        sequence: u64,
11046        entries: AvroValue,
11047    },
11048}
11049
11050#[derive(Clone, Debug, PartialEq, Eq)]
11051struct SelectionMarker {
11052    selection_group_id: String,
11053    selection_group_base_sequence: u64,
11054    selection_group_size: usize,
11055    member_key: SelectionKey,
11056    member_index: usize,
11057    member_base_sequence: u64,
11058    member_size: usize,
11059    operation_kind: String,
11060    operation_identity: String,
11061    outcome: String,
11062    resolution_event_id: String,
11063    resolution_event_type: String,
11064}
11065
11066#[derive(Clone, Debug, PartialEq, Eq)]
11067struct SelectionCancellation {
11068    selection_group_id: String,
11069    member_key: SelectionKey,
11070    member_index: usize,
11071    member_base_sequence: u64,
11072    member_size: usize,
11073    operation_kind: String,
11074    operation_identity: String,
11075}
11076
11077fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
11078    let mut markers: Vec<SelectionMarker> = Vec::new();
11079    for event in events
11080        .iter()
11081        .filter(|event| event.event_type == "SelectionResolved")
11082    {
11083        let payload = &event.payload;
11084        let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
11085        let group_size = required_selection_usize(payload, "selection_group_size")?;
11086        let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
11087        let member_size = required_selection_usize(payload, "member_size")?;
11088        let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
11089        let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
11090            invalid_recorded_history(
11091                "selection_marker_invalid",
11092                base_sequence,
11093                "non-empty selection_group_id",
11094                &payload.to_string(),
11095                "selection winner history is missing its durable group identity",
11096            )
11097        })?;
11098        let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
11099        if group_id != expected_group_id {
11100            return Err(invalid_recorded_history(
11101                "selection_marker_invalid",
11102                base_sequence,
11103                &expected_group_id,
11104                &group_id,
11105                "selection winner history contains an incompatible group identity",
11106            ));
11107        }
11108        let group_end = base_sequence
11109            .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
11110            .unwrap_or(u64::MAX);
11111        let member_end = member_base_sequence
11112            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
11113            .unwrap_or(u64::MAX);
11114        if member_index >= group_size
11115            || member_base_sequence < base_sequence
11116            || member_end > group_end
11117        {
11118            return Err(invalid_recorded_history(
11119                "selection_marker_invalid",
11120                base_sequence,
11121                "winner member within selection group bounds",
11122                &payload.to_string(),
11123                "selection winner history contains an invalid member range",
11124            ));
11125        }
11126        let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
11127            invalid_recorded_history(
11128                "selection_marker_invalid",
11129                base_sequence,
11130                "selection operation kind",
11131                &payload.to_string(),
11132                "selection winner history is missing its operation kind",
11133            )
11134        })?;
11135        if !matches!(
11136            operation_kind.as_str(),
11137            "activity" | "child" | "timer" | "signal" | "condition" | "group"
11138        ) {
11139            return Err(invalid_recorded_history(
11140                "selection_marker_invalid",
11141                base_sequence,
11142                "activity, child, timer, signal, condition, or group",
11143                &operation_kind,
11144                "selection winner history contains an unsupported operation kind",
11145            ));
11146        }
11147        let operation_identity =
11148            payload_string(payload, "operation_identity").ok_or_else(|| {
11149                invalid_recorded_history(
11150                    "selection_marker_invalid",
11151                    base_sequence,
11152                    "non-empty operation identity",
11153                    &payload.to_string(),
11154                    "selection winner history is missing its durable operation identity",
11155                )
11156            })?;
11157        let outcome = payload_string(payload, "outcome").ok_or_else(|| {
11158            invalid_recorded_history(
11159                "selection_marker_invalid",
11160                base_sequence,
11161                "completed or failed selection outcome",
11162                &payload.to_string(),
11163                "selection winner history is missing its outcome",
11164            )
11165        })?;
11166        if !matches!(outcome.as_str(), "completed" | "failed") {
11167            return Err(invalid_recorded_history(
11168                "selection_marker_invalid",
11169                base_sequence,
11170                "completed or failed selection outcome",
11171                &outcome,
11172                "selection winner history contains an unsupported outcome",
11173            ));
11174        }
11175        let marker = SelectionMarker {
11176            selection_group_id: group_id,
11177            selection_group_base_sequence: base_sequence,
11178            selection_group_size: group_size,
11179            member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
11180            member_index,
11181            member_base_sequence,
11182            member_size,
11183            operation_kind,
11184            operation_identity,
11185            outcome,
11186            resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
11187                || {
11188                    invalid_recorded_history(
11189                        "selection_marker_invalid",
11190                        base_sequence,
11191                        "durable resolution_event_id",
11192                        &payload.to_string(),
11193                        "selection winner history is missing its terminal event identity",
11194                    )
11195                },
11196            )?,
11197            resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
11198                || {
11199                    invalid_recorded_history(
11200                        "selection_marker_invalid",
11201                        base_sequence,
11202                        "durable resolution_event_type",
11203                        &payload.to_string(),
11204                        "selection winner history is missing its terminal event type",
11205                    )
11206                },
11207            )?,
11208        };
11209        if let Some(existing) = markers
11210            .iter()
11211            .find(|existing| existing.selection_group_id == marker.selection_group_id)
11212        {
11213            if existing != &marker {
11214                return Err(invalid_recorded_history(
11215                    "selection_marker_conflict",
11216                    base_sequence,
11217                    &format!("one winner for {}", marker.selection_group_id),
11218                    &payload.to_string(),
11219                    "selection history records conflicting winners for one durable group",
11220                ));
11221            }
11222            continue;
11223        }
11224        markers.push(marker);
11225    }
11226    Ok(markers)
11227}
11228
11229fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
11230    let mut cancelled: Vec<SelectionCancellation> = Vec::new();
11231    for event in events
11232        .iter()
11233        .filter(|event| event.event_type == "SelectionOperationCancelled")
11234    {
11235        let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
11236            invalid_recorded_history(
11237                "selection_cancellation_invalid",
11238                0,
11239                "non-empty selection_group_id",
11240                &event.payload.to_string(),
11241                "selection cancellation history is missing its group identity",
11242            )
11243        })?;
11244        let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
11245        let marker = SelectionCancellation {
11246            selection_group_id: group_id,
11247            member_key: selection_key_from_value(
11248                event.payload.get("member_key"),
11249                member_base_sequence,
11250            )?,
11251            member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
11252            member_base_sequence,
11253            member_size: required_selection_usize(&event.payload, "member_size")?,
11254            operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
11255                invalid_recorded_history(
11256                    "selection_cancellation_invalid",
11257                    member_base_sequence,
11258                    "selection operation kind",
11259                    &event.payload.to_string(),
11260                    "selection cancellation is missing its operation kind",
11261                )
11262            })?,
11263            operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
11264                || {
11265                    invalid_recorded_history(
11266                        "selection_cancellation_invalid",
11267                        member_base_sequence,
11268                        "selection operation identity",
11269                        &event.payload.to_string(),
11270                        "selection cancellation is missing its operation identity",
11271                    )
11272                },
11273            )?,
11274        };
11275        if let Some(existing) = cancelled.iter().find(|recorded| {
11276            recorded.selection_group_id == marker.selection_group_id
11277                && recorded.member_base_sequence == marker.member_base_sequence
11278        }) {
11279            if existing != &marker {
11280                return Err(invalid_recorded_history(
11281                    "selection_cancellation_conflict",
11282                    member_base_sequence,
11283                    "one stable SelectionOperationCancelled marker",
11284                    &event.payload.to_string(),
11285                    "selection cancellation history contains conflicting member metadata",
11286                ));
11287            }
11288        } else {
11289            cancelled.push(marker);
11290        }
11291    }
11292    Ok(cancelled)
11293}
11294
11295fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
11296    payload
11297        .get(field)
11298        .and_then(value_as_u64)
11299        .filter(|value| *value > 0)
11300        .ok_or_else(|| {
11301            invalid_recorded_history(
11302                "selection_marker_invalid",
11303                0,
11304                &format!("positive integer {field}"),
11305                &payload.to_string(),
11306                "selection history contains invalid durable identity metadata",
11307            )
11308        })
11309}
11310
11311fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
11312    required_selection_usize_allow_zero(payload, field).and_then(|value| {
11313        if value > 0 {
11314            Ok(value)
11315        } else {
11316            Err(invalid_recorded_history(
11317                "selection_marker_invalid",
11318                0,
11319                &format!("positive integer {field}"),
11320                &payload.to_string(),
11321                "selection history contains invalid durable identity metadata",
11322            ))
11323        }
11324    })
11325}
11326
11327fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
11328    payload
11329        .get(field)
11330        .and_then(value_as_u64)
11331        .and_then(|value| usize::try_from(value).ok())
11332        .ok_or_else(|| {
11333            invalid_recorded_history(
11334                "selection_marker_invalid",
11335                0,
11336                &format!("non-negative integer {field}"),
11337                &payload.to_string(),
11338                "selection history contains invalid durable identity metadata",
11339            )
11340        })
11341}
11342
11343#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
11344struct RecordedActivityOptions {
11345    task_queue: RecordedSnapshotValue<Option<String>>,
11346    execution_mode: RecordedSnapshotValue<Option<String>>,
11347    retry_policy: ActivityRetrySnapshot,
11348    worker_session: Option<Value>,
11349}
11350
11351#[derive(Clone, Debug)]
11352struct RecordedChildPolicies {
11353    parent_close_policy: String,
11354    cancellation_policy: String,
11355}
11356
11357fn recorded_activity_cancellation_policy(
11358    events: &[&HistoryEvent],
11359    sequence: u64,
11360) -> Result<String> {
11361    let mut policy: Option<String> = None;
11362    for event in events.iter().filter(|event| {
11363        matches!(
11364            event.event_type.as_str(),
11365            "ActivityScheduled"
11366                | "ActivityStarted"
11367                | "ActivityCompleted"
11368                | "ActivityFailed"
11369                | "ActivityTimedOut"
11370                | "ActivityCancelled"
11371        )
11372    }) {
11373        for source in [Some(&event.payload), event.payload.get("activity")]
11374            .into_iter()
11375            .flatten()
11376        {
11377            let Some(value) = source.get("cancellation_policy") else {
11378                continue;
11379            };
11380            let Some(incoming @ ("try_cancel" | "wait_cancellation_completed" | "abandon")) =
11381                value.as_str()
11382            else {
11383                return Err(invalid_recorded_history(
11384                    "invalid_activity_cancellation_policy_history",
11385                    sequence,
11386                    "supported Activity cancellation policy",
11387                    &value.to_string(),
11388                    "Activity history contains an invalid cancellation policy",
11389                ));
11390            };
11391            if let Some(previous) = policy.as_deref() {
11392                if previous != incoming {
11393                    return Err(invalid_recorded_history(
11394                        "activity_cancellation_policy_history_conflict",
11395                        sequence,
11396                        previous,
11397                        incoming,
11398                        "Activity cancellation policy changed between history events",
11399                    ));
11400                }
11401            }
11402            policy = Some(incoming.to_string());
11403        }
11404        policy.get_or_insert_with(|| "try_cancel".to_string());
11405    }
11406    Ok(policy.unwrap_or_else(|| "try_cancel".to_string()))
11407}
11408
11409fn recorded_child_policy_value<'a>(
11410    payload: &'a Value,
11411    field: &str,
11412    sequence: u64,
11413) -> Result<Option<&'a str>> {
11414    let Some(value) = payload.get(field).filter(|value| !value.is_null()) else {
11415        return Ok(None);
11416    };
11417    let valid = match (field, value.as_str()) {
11418        (
11419            "parent_close_policy",
11420            Some("abandon" | "request_cancel" | "request_cancellation" | "terminate"),
11421        ) => true,
11422        ("cancellation_policy", Some("abandon" | "try_cancel" | "wait_cancellation_completed")) => {
11423            true
11424        }
11425        _ => false,
11426    };
11427    if !valid {
11428        return Err(invalid_recorded_history(
11429            "invalid_child_workflow_policy_history",
11430            sequence,
11431            "supported child workflow policy",
11432            &value.to_string(),
11433            "child workflow history contains an invalid policy",
11434        ));
11435    }
11436    Ok(value.as_str())
11437}
11438
11439fn recorded_child_policies(
11440    events: &[&HistoryEvent],
11441    scheduled: &HistoryEvent,
11442    sequence: u64,
11443) -> Result<RecordedChildPolicies> {
11444    let parent_close_policy =
11445        recorded_child_policy_value(&scheduled.payload, "parent_close_policy", sequence)?
11446            .unwrap_or("abandon")
11447            .to_string();
11448    let cancellation_policy =
11449        recorded_child_policy_value(&scheduled.payload, "cancellation_policy", sequence)?
11450            .unwrap_or("abandon")
11451            .to_string();
11452    for event in events {
11453        for (field, expected) in [
11454            ("parent_close_policy", parent_close_policy.as_str()),
11455            ("cancellation_policy", cancellation_policy.as_str()),
11456        ] {
11457            if let Some(actual) = recorded_child_policy_value(&event.payload, field, sequence)? {
11458                if actual != expected {
11459                    return Err(invalid_recorded_history(
11460                        "child_workflow_policy_history_conflict",
11461                        sequence,
11462                        expected,
11463                        actual,
11464                        "child workflow policy changed between history events",
11465                    ));
11466                }
11467            }
11468        }
11469    }
11470    Ok(RecordedChildPolicies {
11471        parent_close_policy,
11472        cancellation_policy,
11473    })
11474}
11475
11476fn ensure_child_policies_match(
11477    sequence: u64,
11478    recorded: &RecordedChildPolicies,
11479    current: &ChildWorkflowOptions,
11480) -> Result<()> {
11481    for (field, expected, actual) in [
11482        (
11483            "parent_close_policy",
11484            recorded.parent_close_policy.as_str(),
11485            current.parent_close_policy.as_str(),
11486        ),
11487        (
11488            "cancellation_policy",
11489            recorded.cancellation_policy.as_str(),
11490            current.cancellation_policy.as_str(),
11491        ),
11492    ] {
11493        if expected != actual {
11494            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11495                "child_workflow_policy_changed",
11496                Some(sequence),
11497                Some(expected.to_string()),
11498                Some(actual.to_string()),
11499                format!("child workflow {field} changed during replay"),
11500            )));
11501        }
11502    }
11503    Ok(())
11504}
11505
11506#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
11507enum RecordedSnapshotValue<T> {
11508    /// Older history did not persist this field, so it cannot constrain replay.
11509    Unknown,
11510    Known(T),
11511}
11512
11513impl<T: PartialEq> RecordedSnapshotValue<T> {
11514    fn matches_current(&self, current: &Self) -> bool {
11515        match self {
11516            Self::Unknown => true,
11517            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
11518        }
11519    }
11520}
11521
11522#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
11523struct ActivityRetrySnapshot {
11524    snapshot_version: RecordedSnapshotValue<Option<u64>>,
11525    max_attempts: RecordedSnapshotValue<Option<u64>>,
11526    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
11527    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
11528    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
11529    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
11530    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
11531    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
11532}
11533
11534impl ActivityRetrySnapshot {
11535    fn matches_current(&self, current: &Self) -> bool {
11536        self.snapshot_version
11537            .matches_current(&current.snapshot_version)
11538            && self.max_attempts.matches_current(&current.max_attempts)
11539            && self
11540                .backoff_seconds
11541                .matches_current(&current.backoff_seconds)
11542            && self
11543                .start_to_close_timeout
11544                .matches_current(&current.start_to_close_timeout)
11545            && self
11546                .schedule_to_start_timeout
11547                .matches_current(&current.schedule_to_start_timeout)
11548            && self
11549                .schedule_to_close_timeout
11550                .matches_current(&current.schedule_to_close_timeout)
11551            && self
11552                .heartbeat_timeout
11553                .matches_current(&current.heartbeat_timeout)
11554            && self
11555                .non_retryable_error_types
11556                .matches_current(&current.non_retryable_error_types)
11557    }
11558}
11559
11560fn recorded_optional_u64(
11561    object: Option<&serde_json::Map<String, Value>>,
11562    field: &str,
11563) -> RecordedSnapshotValue<Option<u64>> {
11564    match object.and_then(|object| object.get(field)) {
11565        None => RecordedSnapshotValue::Unknown,
11566        Some(Value::Null) => RecordedSnapshotValue::Known(None),
11567        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
11568    }
11569}
11570
11571fn recorded_optional_string(
11572    object: &serde_json::Map<String, Value>,
11573    field: &str,
11574) -> RecordedSnapshotValue<Option<String>> {
11575    match object.get(field) {
11576        None => RecordedSnapshotValue::Unknown,
11577        Some(Value::Null) => RecordedSnapshotValue::Known(None),
11578        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
11579    }
11580}
11581
11582fn recorded_activity_execution_mode(
11583    events: &[&HistoryEvent],
11584    sequence: u64,
11585) -> Result<RecordedSnapshotValue<Option<String>>> {
11586    let mut recorded = RecordedSnapshotValue::Unknown;
11587    for event in events {
11588        for object in [
11589            event.payload.as_object(),
11590            event.payload.get("activity").and_then(Value::as_object),
11591        ]
11592        .into_iter()
11593        .flatten()
11594        {
11595            let mode = match object.get("execution_mode") {
11596                None => None,
11597                Some(Value::Null) => Some(None),
11598                Some(Value::String(mode)) if !mode.is_empty() => Some(Some(mode.clone())),
11599                Some(_) => {
11600                    return Err(invalid_recorded_history(
11601                        "activity_execution_mode_invalid",
11602                        sequence,
11603                        "null or a non-empty execution mode",
11604                        "invalid execution_mode",
11605                        "activity history contains an invalid execution mode",
11606                    ))
11607                }
11608            };
11609            let legacy_mode = match object.get("local_activity") {
11610                None => None,
11611                Some(Value::Bool(local)) => Some(local.then(|| "local".to_string())),
11612                Some(_) => {
11613                    return Err(invalid_recorded_history(
11614                        "activity_execution_mode_invalid",
11615                        sequence,
11616                        "boolean local_activity",
11617                        "invalid local_activity",
11618                        "activity history contains an invalid local-activity marker",
11619                    ))
11620                }
11621            };
11622            for mode in [mode, legacy_mode].into_iter().flatten() {
11623                if matches!(&recorded, RecordedSnapshotValue::Known(previous) if previous != &mode)
11624                {
11625                    return Err(invalid_recorded_history(
11626                        "activity_execution_mode_conflict", sequence, "one activity execution mode", "conflicting execution modes",
11627                        "activity lifecycle events at one workflow sequence disagree on execution mode",
11628                    ));
11629                }
11630                recorded = RecordedSnapshotValue::Known(mode);
11631            }
11632        }
11633    }
11634    Ok(recorded)
11635}
11636
11637fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
11638    let policy = policy.and_then(Value::as_object);
11639    let backoff_seconds = policy
11640        .and_then(|policy| policy.get("backoff_seconds"))
11641        .and_then(Value::as_array)
11642        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
11643        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
11644    let mut non_retryable_error_types = Vec::new();
11645    for error_type in policy
11646        .and_then(|policy| policy.get("non_retryable_error_types"))
11647        .and_then(Value::as_array)
11648        .into_iter()
11649        .flatten()
11650        .filter_map(Value::as_str)
11651        .map(str::trim)
11652        .filter(|error_type| !error_type.is_empty())
11653    {
11654        if !non_retryable_error_types
11655            .iter()
11656            .any(|recorded| recorded == error_type)
11657        {
11658            non_retryable_error_types.push(error_type.to_string());
11659        }
11660    }
11661
11662    ActivityRetrySnapshot {
11663        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
11664        max_attempts: recorded_optional_u64(policy, "max_attempts"),
11665        backoff_seconds,
11666        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
11667        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
11668        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
11669        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
11670        non_retryable_error_types: if policy
11671            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
11672        {
11673            RecordedSnapshotValue::Known(non_retryable_error_types)
11674        } else {
11675            RecordedSnapshotValue::Unknown
11676        },
11677    }
11678}
11679
11680fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
11681    let policy = options.retry_policy.as_ref();
11682    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
11683        Some(Value::Null) => None,
11684        Some(value) => value_as_u64(value),
11685        None => Some(1),
11686    };
11687    let backoff_seconds = policy
11688        .and_then(|policy| policy.get("backoff_seconds"))
11689        .and_then(Value::as_array)
11690        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
11691        .unwrap_or_default();
11692    let non_retryable_error_types = policy
11693        .and_then(|policy| policy.get("non_retryable_error_types"))
11694        .and_then(Value::as_array)
11695        .into_iter()
11696        .flatten()
11697        .filter_map(Value::as_str)
11698        .map(str::to_string)
11699        .collect();
11700
11701    ActivityRetrySnapshot {
11702        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
11703        max_attempts: RecordedSnapshotValue::Known(max_attempts),
11704        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
11705        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
11706        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
11707        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
11708        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
11709        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
11710    }
11711}
11712
11713fn activity_options_description(options: &RecordedActivityOptions) -> String {
11714    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
11715}
11716
11717impl RecordedCommand {
11718    fn sequence(&self) -> u64 {
11719        match self {
11720            Self::CancellationGroup { sequence, .. }
11721            | Self::CancellationScope { sequence, .. }
11722            | Self::CancellationBoundary { sequence, .. }
11723            | Self::Activity { sequence, .. }
11724            | Self::Timer { sequence, .. }
11725            | Self::ChildWorkflow { sequence, .. }
11726            | Self::SignalWait { sequence, .. }
11727            | Self::ConditionWait { sequence, .. }
11728            | Self::SearchAttributes { sequence, .. }
11729            | Self::SideEffect { sequence, .. }
11730            | Self::VersionMarker { sequence, .. }
11731            | Self::Memo { sequence, .. } => *sequence,
11732        }
11733    }
11734
11735    fn shape(&self) -> &'static str {
11736        match self {
11737            Self::CancellationScope { .. } => "cancellation scope",
11738            Self::CancellationBoundary { .. } | Self::CancellationGroup { .. } => {
11739                "cooperative cancellation"
11740            }
11741            Self::Activity { .. } => "activity",
11742            Self::Timer { .. } => "timer",
11743            Self::ChildWorkflow { .. } => "child workflow",
11744            Self::SignalWait { .. } => "signal wait",
11745            Self::ConditionWait { .. } => "condition wait",
11746            Self::SearchAttributes { .. } => "search-attribute update",
11747            Self::SideEffect { .. } => "side effect",
11748            Self::VersionMarker { .. } => "version marker",
11749            Self::Memo { .. } => "memo upsert",
11750        }
11751    }
11752}
11753
11754fn ensure_version_supported(
11755    change_id: &str,
11756    version: i32,
11757    min_supported: i32,
11758    max_supported: i32,
11759    sequence: u64,
11760) -> Result<()> {
11761    if (min_supported..=max_supported).contains(&version) {
11762        return Ok(());
11763    }
11764    Err(Error::NonDeterministicReplay(ReplayFailure::new(
11765        "version_marker_incompatible_range",
11766        (sequence != 0).then_some(sequence),
11767        Some(format!("{min_supported}..={max_supported}")),
11768        Some(format!("{change_id}:{version}")),
11769        "recorded workflow version is outside the range supported by current code",
11770    )))
11771}
11772
11773#[derive(Clone, Debug)]
11774struct ResumeSignal {
11775    signal_name: String,
11776    arguments: Vec<AvroValue>,
11777}
11778
11779const MAX_PARALLEL_OPERATIONS: usize = 1000;
11780
11781fn parallel_group_prefix(kind: &str) -> &'static str {
11782    match kind {
11783        "activity" => "parallel-activities",
11784        "child" => "parallel-children",
11785        "timer" => "parallel-timers",
11786        _ => "parallel-calls",
11787    }
11788}
11789
11790fn parallel_group_entry(
11791    base_sequence: u64,
11792    size: usize,
11793    index: usize,
11794    kind: &str,
11795) -> ParallelGroupMetadata {
11796    ParallelGroupMetadata {
11797        parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
11798        parallel_group_kind: kind.to_string(),
11799        parallel_group_base_sequence: base_sequence,
11800        parallel_group_size: size,
11801        parallel_group_index: index,
11802        parallel_group_mode: None,
11803        selection_member_key: None,
11804        selection_member_index: None,
11805        selection_member_base_sequence: None,
11806        selection_member_size: None,
11807        selection_member_kind: None,
11808    }
11809}
11810
11811struct SelectionMemberMetadata {
11812    key: SelectionKey,
11813    index: usize,
11814    base_sequence: u64,
11815    size: usize,
11816    kind: String,
11817}
11818
11819fn selection_group_entry(
11820    base_sequence: u64,
11821    size: usize,
11822    index: usize,
11823    kind: &str,
11824    member: &SelectionMemberMetadata,
11825) -> ParallelGroupMetadata {
11826    ParallelGroupMetadata {
11827        parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
11828        parallel_group_kind: kind.to_string(),
11829        parallel_group_base_sequence: base_sequence,
11830        parallel_group_size: size,
11831        parallel_group_index: index,
11832        parallel_group_mode: Some("select".to_string()),
11833        selection_member_key: Some(member.key.clone()),
11834        selection_member_index: Some(member.index),
11835        selection_member_base_sequence: Some(member.base_sequence),
11836        selection_member_size: Some(member.size),
11837        selection_member_kind: Some(member.kind.clone()),
11838    }
11839}
11840
11841fn apply_parallel_group_path(
11842    command: &mut serde_json::Map<String, Value>,
11843    path: &[ParallelGroupMetadata],
11844) {
11845    let Some(inner) = path.last() else {
11846        return;
11847    };
11848    command.insert(
11849        "parallel_group_id".to_string(),
11850        json!(inner.parallel_group_id),
11851    );
11852    command.insert(
11853        "parallel_group_kind".to_string(),
11854        json!(inner.parallel_group_kind),
11855    );
11856    command.insert(
11857        "parallel_group_base_sequence".to_string(),
11858        json!(inner.parallel_group_base_sequence),
11859    );
11860    command.insert(
11861        "parallel_group_size".to_string(),
11862        json!(inner.parallel_group_size),
11863    );
11864    command.insert(
11865        "parallel_group_index".to_string(),
11866        json!(inner.parallel_group_index),
11867    );
11868    if let Some(mode) = &inner.parallel_group_mode {
11869        command.insert("parallel_group_mode".to_string(), json!(mode));
11870    }
11871    if let Some(key) = &inner.selection_member_key {
11872        command.insert("selection_member_key".to_string(), json!(key));
11873    }
11874    if let Some(index) = inner.selection_member_index {
11875        command.insert("selection_member_index".to_string(), json!(index));
11876    }
11877    if let Some(base_sequence) = inner.selection_member_base_sequence {
11878        command.insert(
11879            "selection_member_base_sequence".to_string(),
11880            json!(base_sequence),
11881        );
11882    }
11883    if let Some(size) = inner.selection_member_size {
11884        command.insert("selection_member_size".to_string(), json!(size));
11885    }
11886    if let Some(kind) = &inner.selection_member_kind {
11887        command.insert("selection_member_kind".to_string(), json!(kind));
11888    }
11889    command.insert("parallel_group_path".to_string(), json!(path));
11890}
11891
11892fn ensure_parallel_path_matches(
11893    sequence: u64,
11894    recorded: Option<&[ParallelGroupMetadata]>,
11895    expected: &[ParallelGroupMetadata],
11896) -> Result<()> {
11897    match (recorded, expected.is_empty()) {
11898        (None, true) => Ok(()),
11899        (Some(recorded), false) if recorded == expected => Ok(()),
11900        (None, false) => Err(invalid_recorded_history(
11901            "parallel_group_metadata_missing",
11902            sequence,
11903            &serde_json::to_string(expected).unwrap_or_default(),
11904            "<missing>",
11905            "recorded parallel member is missing its durable group path",
11906        )),
11907        (Some(recorded), true) => Err(invalid_recorded_history(
11908            "parallel_group_shape_mismatch",
11909            sequence,
11910            "sequential command",
11911            &serde_json::to_string(recorded).unwrap_or_default(),
11912            "recorded command belonged to a parallel group but current code schedules it sequentially",
11913        )),
11914        (Some(recorded), false) => Err(invalid_recorded_history(
11915            "parallel_group_shape_mismatch",
11916            sequence,
11917            &serde_json::to_string(recorded).unwrap_or_default(),
11918            &serde_json::to_string(expected).unwrap_or_default(),
11919            "recorded parallel-group identity or path changed during replay",
11920        )),
11921    }
11922}
11923
11924#[derive(Clone, Debug)]
11925enum ParallelShape {
11926    Leaf,
11927    Group(Vec<ParallelShape>),
11928}
11929
11930struct ParallelDescriptor {
11931    operation: ParallelOperation,
11932    offset: usize,
11933    member_path: Vec<usize>,
11934    group_path: Vec<ParallelGroupMetadata>,
11935}
11936
11937fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
11938    operations
11939        .iter()
11940        .map(|operation| match operation {
11941            ParallelOperation::Group(children) => parallel_leaf_count(children),
11942            _ => 1,
11943        })
11944        .sum()
11945}
11946
11947fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
11948    match operation {
11949        ParallelOperation::Activity { .. } => Some("activity"),
11950        ParallelOperation::ChildWorkflow { .. } => Some("child"),
11951        ParallelOperation::Timer(_) => Some("timer"),
11952        ParallelOperation::Signal(_) => Some("signal"),
11953        ParallelOperation::Condition { .. } => Some("condition"),
11954        ParallelOperation::Group(children) => parallel_group_kind(children),
11955    }
11956}
11957
11958fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
11959    let mut kind = None;
11960    for operation in operations {
11961        let Some(operation_kind) = parallel_operation_kind(operation) else {
11962            continue;
11963        };
11964        match kind {
11965            None => kind = Some(operation_kind),
11966            Some(current) if current == operation_kind => {}
11967            Some(_) => return Some("mixed"),
11968        }
11969    }
11970    kind
11971}
11972
11973fn validate_parallel_operations(
11974    operations: &[ParallelOperation],
11975    member_path: &mut Vec<usize>,
11976    root: bool,
11977) -> Result<()> {
11978    let leaves = parallel_leaf_count(operations);
11979    if leaves > MAX_PARALLEL_OPERATIONS {
11980        return Err(Error::InvalidParallelGroup(ParallelGroupError {
11981            reason: "fan_out_limit_exceeded",
11982            member_path: member_path.clone(),
11983            message: format!(
11984                "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
11985            ),
11986        }));
11987    }
11988    if !root && operations.is_empty() {
11989        return Err(Error::InvalidParallelGroup(ParallelGroupError {
11990            reason: "nested_group_empty",
11991            member_path: member_path.clone(),
11992            message: "a nested group must contain at least one durable leaf".to_string(),
11993        }));
11994    }
11995
11996    for (index, operation) in operations.iter().enumerate() {
11997        member_path.push(index);
11998        match operation {
11999            ParallelOperation::Activity {
12000                options, arguments, ..
12001            } => {
12002                options
12003                    .validate()
12004                    .map_err(|error| Error::InvalidActivityOptions(error))?;
12005                if let Err(error) = arguments {
12006                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
12007                        reason: "arguments_invalid",
12008                        member_path: member_path.clone(),
12009                        message: error.to_string(),
12010                    }));
12011                }
12012            }
12013            ParallelOperation::ChildWorkflow {
12014                options, arguments, ..
12015            } => {
12016                validate_parallel_child_options(options)?;
12017                if let Err(error) = arguments {
12018                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
12019                        reason: "arguments_invalid",
12020                        member_path: member_path.clone(),
12021                        message: error.to_string(),
12022                    }));
12023                }
12024            }
12025            ParallelOperation::Timer(duration)
12026                if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
12027            {
12028                return Err(Error::TimerDurationOverflow);
12029            }
12030            ParallelOperation::Timer(_) => {}
12031            ParallelOperation::Signal(signal_name) => {
12032                validate_user_signal_name(signal_name)?;
12033                if signal_name.trim().is_empty() {
12034                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
12035                        reason: "signal_name_empty",
12036                        member_path: member_path.clone(),
12037                        message: "signal wait name must not be empty".to_string(),
12038                    }));
12039                }
12040            }
12041            ParallelOperation::Condition { options, .. } => {
12042                options.validate()?;
12043            }
12044            ParallelOperation::Group(children) => {
12045                validate_parallel_operations(children, member_path, false)?;
12046            }
12047        }
12048        member_path.pop();
12049    }
12050    Ok(())
12051}
12052
12053fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
12054    if options.task_queue.trim().is_empty() {
12055        return Err(Error::InvalidChildWorkflowOptions(
12056            "task_queue must not be empty".to_string(),
12057        ));
12058    }
12059    for (name, value) in [
12060        (
12061            "execution_timeout_seconds",
12062            options.execution_timeout_seconds,
12063        ),
12064        ("run_timeout_seconds", options.run_timeout_seconds),
12065    ] {
12066        if value == Some(0) {
12067            return Err(Error::InvalidChildWorkflowOptions(format!(
12068                "{name} must be at least 1"
12069            )));
12070        }
12071    }
12072    if options
12073        .retry_policy
12074        .as_ref()
12075        .is_some_and(|policy| policy.max_attempts == Some(0))
12076    {
12077        return Err(Error::InvalidChildWorkflowOptions(
12078            "retry_policy.max_attempts must be at least 1".to_string(),
12079        ));
12080    }
12081    Ok(())
12082}
12083
12084fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
12085    ParallelShape::Group(
12086        operations
12087            .iter()
12088            .map(|operation| match operation {
12089                ParallelOperation::Group(children) => parallel_shape(children),
12090                _ => ParallelShape::Leaf,
12091            })
12092            .collect(),
12093    )
12094}
12095
12096fn parallel_descriptors(
12097    operations: Vec<ParallelOperation>,
12098    base_sequence: u64,
12099) -> Result<Vec<ParallelDescriptor>> {
12100    let size = parallel_leaf_count(&operations);
12101    let kind = parallel_group_kind(&operations).unwrap_or("activity");
12102    let mut descriptors = Vec::with_capacity(size);
12103    let mut cursor = 0;
12104
12105    for (index, operation) in operations.into_iter().enumerate() {
12106        match operation {
12107            ParallelOperation::Group(children) => {
12108                let child_base = base_sequence
12109                    .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
12110                    .ok_or(Error::TimerDurationOverflow)?;
12111                for mut descriptor in parallel_descriptors(children, child_base)? {
12112                    let outer_index = cursor + descriptor.offset;
12113                    descriptor.group_path.insert(
12114                        0,
12115                        parallel_group_entry(base_sequence, size, outer_index, kind),
12116                    );
12117                    descriptor.member_path.insert(0, index);
12118                    descriptor.offset = outer_index;
12119                    descriptors.push(descriptor);
12120                }
12121                cursor = descriptors.len();
12122            }
12123            operation => {
12124                descriptors.push(ParallelDescriptor {
12125                    operation,
12126                    offset: cursor,
12127                    member_path: vec![index],
12128                    group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
12129                });
12130                cursor += 1;
12131            }
12132        }
12133    }
12134    Ok(descriptors)
12135}
12136
12137enum ParallelLeafCall {
12138    Activity(ActivityCall),
12139    ChildWorkflow(ChildWorkflowCall),
12140    Timer(TimerCall),
12141    Signal(SignalCall),
12142    Condition(ConditionWaitCall),
12143}
12144
12145fn parallel_leaf_call(
12146    ctx: &WorkflowContext,
12147    operation: ParallelOperation,
12148    parallel_group_path: Vec<ParallelGroupMetadata>,
12149) -> ParallelLeafCall {
12150    match operation {
12151        ParallelOperation::Activity {
12152            activity_type,
12153            options,
12154            arguments,
12155        } => ParallelLeafCall::Activity(ActivityCall {
12156            ctx: ctx.clone(),
12157            activity_type,
12158            options,
12159            worker_session: None,
12160            args: Some(arguments),
12161            scheduled: false,
12162            local: false,
12163            local_result: None,
12164            parallel_group_path,
12165        }),
12166        ParallelOperation::ChildWorkflow {
12167            workflow_type,
12168            options,
12169            arguments,
12170        } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
12171            ctx: ctx.clone(),
12172            workflow_type,
12173            options,
12174            args: Some(arguments),
12175            scheduled: false,
12176            matched_pending: false,
12177            parallel_group_path,
12178        }),
12179        ParallelOperation::Timer(duration) => {
12180            let delay_seconds = duration
12181                .as_secs()
12182                .checked_add(u64::from(duration.subsec_nanos() > 0));
12183            ParallelLeafCall::Timer(TimerCall {
12184                ctx: ctx.clone(),
12185                delay_seconds,
12186                scheduled: false,
12187                matched_pending: false,
12188                parallel_group_path,
12189            })
12190        }
12191        ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
12192            ctx: ctx.clone(),
12193            signal_name,
12194            runtime_reserved_allowed: false,
12195            opened_wait: false,
12196            matched_pending: false,
12197            parallel_group_path,
12198        }),
12199        ParallelOperation::Condition { options, predicate } => {
12200            ParallelLeafCall::Condition(ConditionWaitCall {
12201                ctx: ctx.clone(),
12202                options,
12203                predicate,
12204                occurrence_id: None,
12205                opened_wait: false,
12206                parallel_group_path,
12207            })
12208        }
12209        ParallelOperation::Group(_) => {
12210            unreachable!("parallel descriptors contain only durable leaves")
12211        }
12212    }
12213}
12214
12215impl ParallelLeafCall {
12216    fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
12217        match self {
12218            Self::Activity(call) => Pin::new(call)
12219                .poll_avro_value(cx)
12220                .map_ok(ParallelAvroResult::Activity),
12221            Self::ChildWorkflow(call) => Pin::new(call)
12222                .poll_avro_value(cx)
12223                .map_ok(ParallelAvroResult::ChildWorkflow),
12224            Self::Timer(call) => Pin::new(call)
12225                .poll(cx)
12226                .map_ok(|()| ParallelAvroResult::Timer),
12227            Self::Signal(call) => Pin::new(call)
12228                .poll_avro_value(cx)
12229                .map_ok(ParallelAvroResult::Signal),
12230            Self::Condition(call) => Pin::new(call)
12231                .poll(cx)
12232                .map_ok(ParallelAvroResult::Condition),
12233        }
12234    }
12235}
12236
12237struct ParallelLeaf {
12238    call: ParallelLeafCall,
12239    sequence: u64,
12240    member_path: Vec<usize>,
12241    group_path: Vec<ParallelGroupMetadata>,
12242    result: Option<ParallelAvroResult>,
12243}
12244
12245/// Future returned by [`WorkflowContext::parallel`].
12246pub struct ParallelCall {
12247    ctx: WorkflowContext,
12248    operations: Option<Vec<ParallelOperation>>,
12249    shape: Option<ParallelShape>,
12250    leaves: Vec<ParallelLeaf>,
12251    pending_scope_delivery: bool,
12252    worker_session: Option<WorkerSessionOptions>,
12253}
12254
12255impl ParallelCall {
12256    fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
12257        Self {
12258            ctx,
12259            operations: Some(operations),
12260            shape: None,
12261            leaves: Vec::new(),
12262            pending_scope_delivery: false,
12263            worker_session: None,
12264        }
12265    }
12266
12267    fn initialize(&mut self) -> Result<()> {
12268        if let Some(session) = &self.worker_session {
12269            session.to_wire()?;
12270        }
12271        let operations = self.operations.take().unwrap_or_default();
12272        validate_parallel_operations(&operations, &mut Vec::new(), true)?;
12273        self.shape = Some(parallel_shape(&operations));
12274        if operations.is_empty() {
12275            return Ok(());
12276        }
12277
12278        let base_sequence = {
12279            let state = self
12280                .ctx
12281                .state
12282                .lock()
12283                .map_err(|_| Error::WorkflowStatePoisoned)?;
12284            if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
12285                recorded.sequence()
12286            } else {
12287                let last = state
12288                    .recorded_commands
12289                    .last()
12290                    .map(RecordedCommand::sequence)
12291                    .unwrap_or(0);
12292                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
12293                    .and_then(|sequence| sequence.checked_add(1))
12294                    .ok_or_else(|| {
12295                        Error::InvalidParallelGroup(ParallelGroupError {
12296                            reason: "sequence_overflow",
12297                            member_path: Vec::new(),
12298                            message: "parallel group sequence identity overflowed u64".to_string(),
12299                        })
12300                    })?
12301            }
12302        };
12303
12304        let descriptors = parallel_descriptors(operations, base_sequence)?;
12305        {
12306            let mut state = self
12307                .ctx
12308                .state
12309                .lock()
12310                .map_err(|_| Error::WorkflowStatePoisoned)?;
12311            let cursor = state.command_cursor;
12312            self.ctx.validate_scope_membership(&mut state, cursor)?;
12313            if state.prepare_parallel_scope_cancellation(&descriptors)? {
12314                self.pending_scope_delivery = true;
12315                return Ok(());
12316            }
12317            state.expand_cancellation_group(&descriptors)?;
12318            state.prepare_group_cancellation(&descriptors)?;
12319        }
12320        self.leaves = descriptors
12321            .into_iter()
12322            .map(|descriptor| {
12323                let mut call = parallel_leaf_call(
12324                    &self.ctx,
12325                    descriptor.operation,
12326                    descriptor.group_path.clone(),
12327                );
12328                if let (Some(session), ParallelLeafCall::Activity(activity)) =
12329                    (&self.worker_session, &mut call)
12330                {
12331                    activity.worker_session = Some(session.clone());
12332                }
12333                ParallelLeaf {
12334                    call,
12335                    sequence: base_sequence + descriptor.offset as u64,
12336                    member_path: descriptor.member_path,
12337                    group_path: descriptor.group_path,
12338                    result: None,
12339                }
12340            })
12341            .collect();
12342        Ok(())
12343    }
12344
12345    fn poll_avro_value(
12346        mut self: Pin<&mut Self>,
12347        cx: &mut TaskContext<'_>,
12348    ) -> Poll<Result<Vec<ParallelAvroResult>>> {
12349        if self.operations.is_some() {
12350            if let Err(error) = self.initialize() {
12351                return Poll::Ready(Err(error));
12352            }
12353        }
12354        if self.pending_scope_delivery {
12355            return Poll::Pending;
12356        }
12357        if self.leaves.is_empty() {
12358            return Poll::Ready(Ok(Vec::new()));
12359        }
12360
12361        let mut failures = Vec::new();
12362        let mut pending = false;
12363        for (index, leaf) in self.leaves.iter_mut().enumerate() {
12364            if leaf.result.is_some() {
12365                continue;
12366            }
12367            match leaf.call.poll_avro_value(cx) {
12368                Poll::Ready(Ok(result)) => leaf.result = Some(result),
12369                Poll::Ready(Err(error)) => failures.push((index, error)),
12370                Poll::Pending => pending = true,
12371            }
12372        }
12373
12374        if !failures.is_empty() {
12375            if let Some(position) = failures
12376                .iter()
12377                .position(|(_, error)| workflow_task_integrity_error(error))
12378            {
12379                return Poll::Ready(Err(failures.remove(position).1));
12380            }
12381            if self
12382                .ctx
12383                .state
12384                .lock()
12385                .map_err(|_| Error::WorkflowStatePoisoned)?
12386                .cancellation_delivery_intent
12387                .is_some()
12388            {
12389                return Poll::Pending;
12390            }
12391            if let Some(position) = failures
12392                .iter()
12393                .position(|(_, error)| matches!(error, Error::CooperativeCancellationRequested(_)))
12394            {
12395                return Poll::Ready(Err(failures.remove(position).1));
12396            }
12397            failures.sort_by_key(|(index, _)| *index);
12398            let (failed_index, cause) = failures.remove(0);
12399            let failed = &self.leaves[failed_index];
12400            let completed = self
12401                .leaves
12402                .iter()
12403                .filter_map(|leaf| {
12404                    leaf.result
12405                        .clone()
12406                        .and_then(|result| result.into_json_result().ok())
12407                        .map(|result| ParallelCompletion {
12408                            member_path: leaf.member_path.clone(),
12409                            result,
12410                        })
12411                })
12412                .collect();
12413            let group_id = failed
12414                .group_path
12415                .first()
12416                .map(|entry| entry.parallel_group_id.clone())
12417                .unwrap_or_default();
12418            let sequences = self
12419                .leaves
12420                .iter()
12421                .map(|leaf| leaf.sequence)
12422                .collect::<Vec<_>>();
12423            self.ctx
12424                .state
12425                .lock()
12426                .map_err(|_| Error::WorkflowStatePoisoned)?
12427                .advance_cancellation_sequences(&sequences, Some(failed.sequence));
12428            return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
12429                group_id,
12430                member_path: failed.member_path.clone(),
12431                group_path: failed.group_path.clone(),
12432                completed,
12433                cause: Box::new(cause),
12434            })));
12435        }
12436        if pending
12437            || self
12438                .ctx
12439                .state
12440                .lock()
12441                .map_err(|_| Error::WorkflowStatePoisoned)?
12442                .cancellation_delivery_intent
12443                .is_some()
12444        {
12445            return Poll::Pending;
12446        }
12447
12448        let mut flat_results = self
12449            .leaves
12450            .iter_mut()
12451            .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
12452            .collect::<Vec<_>>()
12453            .into_iter();
12454        let results = parallel_results_for_shape(
12455            self.shape.as_ref().expect("initialized parallel shape"),
12456            &mut flat_results,
12457        );
12458        let sequences = self
12459            .leaves
12460            .iter()
12461            .map(|leaf| leaf.sequence)
12462            .collect::<Vec<_>>();
12463        self.ctx
12464            .state
12465            .lock()
12466            .map_err(|_| Error::WorkflowStatePoisoned)?
12467            .advance_cancellation_sequences(&sequences, None);
12468        Poll::Ready(Ok(match results {
12469            ParallelAvroResult::Group(results) => results,
12470            ParallelAvroResult::Activity(_)
12471            | ParallelAvroResult::ChildWorkflow(_)
12472            | ParallelAvroResult::Timer
12473            | ParallelAvroResult::Signal(_)
12474            | ParallelAvroResult::Condition(_) => {
12475                unreachable!("root parallel shape is a group")
12476            }
12477        }))
12478    }
12479}
12480
12481fn parallel_results_for_shape(
12482    shape: &ParallelShape,
12483    flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
12484) -> ParallelAvroResult {
12485    match shape {
12486        ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
12487        ParallelShape::Group(children) => ParallelAvroResult::Group(
12488            children
12489                .iter()
12490                .map(|child| parallel_results_for_shape(child, flat_results))
12491                .collect(),
12492        ),
12493    }
12494}
12495
12496impl Future for ParallelCall {
12497    type Output = Result<Vec<ParallelResult>>;
12498
12499    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12500        self.poll_avro_value(cx)
12501            .map_ok(|results| {
12502                results
12503                    .into_iter()
12504                    .map(ParallelAvroResult::into_json_result)
12505                    .collect::<Result<Vec<_>>>()
12506            })
12507            .map_ok(|result| result)
12508            .flatten_result()
12509    }
12510}
12511
12512#[derive(Clone, Debug)]
12513struct SelectionMemberPlan {
12514    key: SelectionKey,
12515    index: usize,
12516    base_sequence: u64,
12517    size: usize,
12518    kind: String,
12519    shape: ParallelShape,
12520    leaf_start: usize,
12521}
12522
12523fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
12524    match operation {
12525        ParallelOperation::Activity { .. } => "activity",
12526        ParallelOperation::ChildWorkflow { .. } => "child",
12527        ParallelOperation::Timer(_) => "timer",
12528        ParallelOperation::Signal(_) => "signal",
12529        ParallelOperation::Condition { .. } => "condition",
12530        ParallelOperation::Group(_) => "group",
12531    }
12532}
12533
12534fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
12535    match operation {
12536        ParallelOperation::Group(children) => parallel_shape(children),
12537        _ => ParallelShape::Leaf,
12538    }
12539}
12540
12541fn selection_descriptors(
12542    operations: Vec<(SelectionKey, ParallelOperation)>,
12543    base_sequence: u64,
12544) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
12545    if operations.is_empty() {
12546        return Err(Error::InvalidParallelGroup(ParallelGroupError {
12547            reason: "selection_empty",
12548            member_path: Vec::new(),
12549            message: "durable selection requires at least one operation".to_string(),
12550        }));
12551    }
12552    let operation_refs = operations
12553        .iter()
12554        .map(|(_, operation)| operation)
12555        .collect::<Vec<_>>();
12556    let total_size = operation_refs
12557        .iter()
12558        .map(|operation| match operation {
12559            ParallelOperation::Group(children) => parallel_leaf_count(children),
12560            _ => 1,
12561        })
12562        .sum::<usize>();
12563    if total_size > MAX_PARALLEL_OPERATIONS {
12564        return Err(Error::InvalidParallelGroup(ParallelGroupError {
12565            reason: "fan_out_limit_exceeded",
12566            member_path: Vec::new(),
12567            message: format!(
12568                "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
12569            ),
12570        }));
12571    }
12572    let group_kind = {
12573        let mut kind = None;
12574        for operation in &operation_refs {
12575            let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
12576            match kind {
12577                None => kind = Some(operation_kind),
12578                Some(current) if current == operation_kind => {}
12579                Some(_) => {
12580                    kind = Some("mixed");
12581                    break;
12582                }
12583            }
12584        }
12585        kind.unwrap_or("mixed")
12586    };
12587
12588    let mut descriptors = Vec::with_capacity(total_size);
12589    let mut members = Vec::with_capacity(operations.len());
12590    let mut cursor = 0usize;
12591    let mut seen_keys: Vec<SelectionKey> = Vec::new();
12592    for (member_index, (key, operation)) in operations.into_iter().enumerate() {
12593        if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
12594            return Err(Error::InvalidParallelGroup(ParallelGroupError {
12595                reason: "selection_key_invalid",
12596                member_path: vec![member_index],
12597                message: "selection member keys must be non-empty strings or non-negative integers"
12598                    .to_string(),
12599            }));
12600        }
12601        if seen_keys.contains(&key) {
12602            return Err(Error::InvalidParallelGroup(ParallelGroupError {
12603                reason: "selection_key_duplicate",
12604                member_path: vec![member_index],
12605                message: format!("selection member key {key:?} is duplicated"),
12606            }));
12607        }
12608        seen_keys.push(key.clone());
12609        let member_size = match &operation {
12610            ParallelOperation::Group(children) => parallel_leaf_count(children),
12611            _ => 1,
12612        };
12613        if member_size == 0 {
12614            return Err(Error::InvalidParallelGroup(ParallelGroupError {
12615                reason: "selection_member_empty",
12616                member_path: vec![member_index],
12617                message: "a selection member must contain at least one durable leaf".to_string(),
12618            }));
12619        }
12620        let member_base = base_sequence
12621            .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
12622            .ok_or(Error::TimerDurationOverflow)?;
12623        let member_kind = selection_operation_kind(&operation).to_string();
12624        let member_shape = selection_operation_shape(&operation);
12625        let leaf_start = descriptors.len();
12626        match operation {
12627            ParallelOperation::Group(children) => {
12628                validate_parallel_operations(&children, &mut vec![member_index], false)?;
12629                for mut descriptor in parallel_descriptors(children, member_base)? {
12630                    let flat_index = cursor + descriptor.offset;
12631                    descriptor.group_path.insert(
12632                        0,
12633                        selection_group_entry(
12634                            base_sequence,
12635                            total_size,
12636                            flat_index,
12637                            group_kind,
12638                            &SelectionMemberMetadata {
12639                                key: key.clone(),
12640                                index: member_index,
12641                                base_sequence: member_base,
12642                                size: member_size,
12643                                kind: member_kind.clone(),
12644                            },
12645                        ),
12646                    );
12647                    descriptor.member_path.insert(0, member_index);
12648                    descriptor.offset = flat_index;
12649                    descriptors.push(descriptor);
12650                }
12651            }
12652            operation => {
12653                validate_parallel_operations(
12654                    std::slice::from_ref(&operation),
12655                    &mut Vec::new(),
12656                    true,
12657                )?;
12658                descriptors.push(ParallelDescriptor {
12659                    operation,
12660                    offset: cursor,
12661                    member_path: vec![member_index],
12662                    group_path: vec![selection_group_entry(
12663                        base_sequence,
12664                        total_size,
12665                        cursor,
12666                        group_kind,
12667                        &SelectionMemberMetadata {
12668                            key: key.clone(),
12669                            index: member_index,
12670                            base_sequence: member_base,
12671                            size: member_size,
12672                            kind: member_kind.clone(),
12673                        },
12674                    )],
12675                });
12676            }
12677        }
12678        members.push(SelectionMemberPlan {
12679            key,
12680            index: member_index,
12681            base_sequence: member_base,
12682            size: member_size,
12683            kind: member_kind,
12684            shape: member_shape,
12685            leaf_start,
12686        });
12687        cursor += member_size;
12688    }
12689    Ok((descriptors, members))
12690}
12691
12692struct SelectionLeaf {
12693    call: ParallelLeafCall,
12694    outcome: Option<Result<ParallelAvroResult>>,
12695}
12696
12697/// Stable reference to one member of a durable selection group.
12698#[derive(Clone)]
12699pub struct DurableOperationHandle {
12700    ctx: WorkflowContext,
12701    pub key: SelectionKey,
12702    pub index: usize,
12703    pub kind: String,
12704    pub identity: String,
12705    pub base_sequence: u64,
12706    pub size: usize,
12707    pub selection_group_id: String,
12708    shape: ParallelShape,
12709}
12710
12711impl std::fmt::Debug for DurableOperationHandle {
12712    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
12713        formatter
12714            .debug_struct("DurableOperationHandle")
12715            .field("key", &self.key)
12716            .field("index", &self.index)
12717            .field("kind", &self.kind)
12718            .field("identity", &self.identity)
12719            .field("base_sequence", &self.base_sequence)
12720            .field("size", &self.size)
12721            .field("selection_group_id", &self.selection_group_id)
12722            .finish()
12723    }
12724}
12725
12726impl DurableOperationHandle {
12727    /// Await this member after another member has already won the selection.
12728    pub fn await_result(&self) -> DurableOperationAwaitCall {
12729        DurableOperationAwaitCall {
12730            handle: self.clone(),
12731        }
12732    }
12733
12734    /// Request cancellation without affecting siblings. The future resolves
12735    /// to unit; only `SelectionOperationCancelled` history proves cancellation
12736    /// beat a concurrently committed terminal result.
12737    pub fn cancel(&self) -> CancelDurableOperationCall {
12738        CancelDurableOperationCall {
12739            handle: self.clone(),
12740            emitted: false,
12741        }
12742    }
12743}
12744
12745/// The one winner committed for a durable selection group.
12746#[derive(Debug)]
12747pub struct SelectionResult {
12748    pub key: SelectionKey,
12749    pub index: usize,
12750    pub kind: String,
12751    pub identity: String,
12752    pub value: Option<ParallelResult>,
12753    pub failure: Option<Error>,
12754    pub winner: DurableOperationHandle,
12755    pub handles: Vec<DurableOperationHandle>,
12756}
12757
12758impl SelectionResult {
12759    pub fn succeeded(&self) -> bool {
12760        self.failure.is_none()
12761    }
12762
12763    pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
12764        self.handles.iter().find(|handle| &handle.key == key)
12765    }
12766
12767    pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
12768        self.handles
12769            .iter()
12770            .filter(|handle| handle.index != self.index)
12771            .collect()
12772    }
12773
12774    pub fn into_result(self) -> Result<ParallelResult> {
12775        match (self.value, self.failure) {
12776            (Some(value), None) => Ok(value),
12777            (_, Some(error)) => Err(error),
12778            _ => Err(Error::WorkerLoop(
12779                "selection result contained neither a value nor a failure".to_string(),
12780            )),
12781        }
12782    }
12783}
12784
12785/// Future returned by [`WorkflowContext::select`].
12786pub struct SelectCall {
12787    ctx: WorkflowContext,
12788    operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
12789    members: Vec<SelectionMemberPlan>,
12790    leaves: Vec<SelectionLeaf>,
12791    group_id: Option<String>,
12792}
12793
12794impl SelectCall {
12795    fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
12796        Self {
12797            ctx,
12798            operations: Some(operations),
12799            members: Vec::new(),
12800            leaves: Vec::new(),
12801            group_id: None,
12802        }
12803    }
12804
12805    fn initialize(&mut self) -> Result<()> {
12806        let operations = self.operations.take().unwrap_or_default();
12807        let base_sequence = {
12808            let state = self
12809                .ctx
12810                .state
12811                .lock()
12812                .map_err(|_| Error::WorkflowStatePoisoned)?;
12813            if let Some(RecordedCommand::CancellationGroup { sequence, .. }) =
12814                state.recorded_commands.get(state.command_cursor)
12815            {
12816                *sequence
12817            } else if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor)
12818            {
12819                marker.selection_group_base_sequence
12820            } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
12821                recorded.sequence()
12822            } else {
12823                let last = state
12824                    .recorded_commands
12825                    .last()
12826                    .map(RecordedCommand::sequence)
12827                    .unwrap_or(0);
12828                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
12829                    .and_then(|sequence| sequence.checked_add(1))
12830                    .ok_or(Error::TimerDurationOverflow)?
12831            }
12832        };
12833        let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
12834        let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
12835        {
12836            let mut state = self
12837                .ctx
12838                .state
12839                .lock()
12840                .map_err(|_| Error::WorkflowStatePoisoned)?;
12841            state.expand_cancellation_group(&descriptors)?;
12842            state.prepare_group_cancellation(&descriptors)?;
12843        }
12844        self.leaves = descriptors
12845            .into_iter()
12846            .map(|descriptor| SelectionLeaf {
12847                call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
12848                outcome: None,
12849            })
12850            .collect();
12851        self.members = members;
12852        self.group_id = Some(group_id);
12853        Ok(())
12854    }
12855}
12856
12857impl Future for SelectCall {
12858    type Output = Result<SelectionResult>;
12859
12860    fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12861        if self.operations.is_some() {
12862            if let Err(error) = self.initialize() {
12863                return Poll::Ready(Err(error));
12864            }
12865        }
12866
12867        for leaf in &mut self.leaves {
12868            if leaf.outcome.is_some() {
12869                continue;
12870            }
12871            if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
12872                if outcome
12873                    .as_ref()
12874                    .err()
12875                    .is_some_and(workflow_task_integrity_error)
12876                {
12877                    return Poll::Ready(outcome.map(|_| unreachable!()));
12878                }
12879                leaf.outcome = Some(outcome);
12880            }
12881        }
12882
12883        if self
12884            .ctx
12885            .state
12886            .lock()
12887            .map_err(|_| Error::WorkflowStatePoisoned)?
12888            .cancellation_delivery_intent
12889            .is_some()
12890        {
12891            return Poll::Pending;
12892        }
12893
12894        if let Some(leaf) = self.leaves.iter_mut().find(|leaf| {
12895            matches!(
12896                leaf.outcome,
12897                Some(Err(Error::CooperativeCancellationRequested(_)))
12898            )
12899        }) {
12900            return Poll::Ready(
12901                leaf.outcome
12902                    .take()
12903                    .expect("matched cancellation")
12904                    .map(|_| unreachable!()),
12905            );
12906        }
12907
12908        let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
12909        let selection_member_range = self
12910            .members
12911            .first()
12912            .map(|member| member.base_sequence)
12913            .zip(self.leaves.len().try_into().ok())
12914            .map(|(base_sequence, size): (u64, u64)| {
12915                base_sequence..base_sequence.saturating_add(size)
12916            });
12917        let marker = {
12918            let mut state = match self.ctx.state.lock() {
12919                Ok(state) => state,
12920                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12921            };
12922            let marker = state
12923                .selection_markers
12924                .get(state.selection_marker_cursor)
12925                .cloned();
12926            if marker.is_none()
12927                && all_members_terminal
12928                && selection_member_range.as_ref().is_some_and(|member_range| {
12929                    state
12930                        .recorded_commands
12931                        .iter()
12932                        .any(|command| member_range.contains(&command.sequence()))
12933                })
12934            {
12935                // Terminal member history can be committed before the server's
12936                // SelectionResolved marker becomes visible to this task. That
12937                // marker is the durable barrier the selection is still waiting
12938                // on, so an otherwise commandless replay remains legitimately
12939                // pending instead of failing as an untracked yield.
12940                state.matched_recorded_pending = true;
12941            }
12942            marker
12943        };
12944        let Some(marker) = marker else {
12945            return Poll::Pending;
12946        };
12947        if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
12948            || marker.selection_group_size != self.leaves.len()
12949            || self.members.first().map(|member| member.base_sequence)
12950                != Some(marker.selection_group_base_sequence)
12951        {
12952            return Poll::Ready(Err(invalid_recorded_history(
12953                "selection_group_shape_mismatch",
12954                marker.selection_group_base_sequence,
12955                self.group_id
12956                    .as_deref()
12957                    .unwrap_or("initialized selection group"),
12958                &marker.selection_group_id,
12959                "recorded selection group differs from current workflow code",
12960            )));
12961        }
12962        let Some(member_position) = self.members.iter().position(|member| {
12963            member.key == marker.member_key
12964                && member.index == marker.member_index
12965                && member.base_sequence == marker.member_base_sequence
12966                && member.size == marker.member_size
12967                && member.kind == marker.operation_kind
12968        }) else {
12969            return Poll::Ready(Err(invalid_recorded_history(
12970                "selection_member_shape_mismatch",
12971                marker.member_base_sequence,
12972                "winner member matching current workflow code",
12973                &format!("{:?}", marker.member_key),
12974                "recorded selection winner differs from the authored member identity",
12975            )));
12976        };
12977        let member = self.members[member_position].clone();
12978        let (handles, resolution_sequence) = {
12979            let mut state = match self.ctx.state.lock() {
12980                Ok(state) => state,
12981                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12982            };
12983            let identities = self
12984                .members
12985                .iter()
12986                .map(|candidate| {
12987                    selection_operation_identity(
12988                        &state,
12989                        &candidate.kind,
12990                        candidate.base_sequence,
12991                        candidate.size,
12992                    )
12993                })
12994                .collect::<Vec<_>>();
12995            if let Some((position, missing)) = identities
12996                .iter()
12997                .enumerate()
12998                .find(|(_, identity)| identity.is_empty())
12999                .map(|(position, identity)| (position, identity.clone()))
13000            {
13001                let candidate = &self.members[position];
13002                return Poll::Ready(Err(invalid_recorded_history(
13003                    "selection_operation_identity_missing",
13004                    candidate.base_sequence,
13005                    &format!(
13006                        "durable {} resource identity from scheduled/open history",
13007                        candidate.kind
13008                    ),
13009                    &missing,
13010                    "selection member history is missing its canonical durable identity",
13011                )));
13012            }
13013            let expected_winner_identity = &identities[member_position];
13014            let resolution_sequence = match validated_selection_resolution_sequence(
13015                &state,
13016                &marker,
13017                &member,
13018                expected_winner_identity,
13019            ) {
13020                Ok(sequence) => sequence,
13021                Err(error) => return Poll::Ready(Err(error)),
13022            };
13023            let handles = self
13024                .members
13025                .iter()
13026                .zip(identities)
13027                .map(|(member, identity)| DurableOperationHandle {
13028                    ctx: self.ctx.clone(),
13029                    key: member.key.clone(),
13030                    index: member.index,
13031                    kind: member.kind.clone(),
13032                    identity,
13033                    base_sequence: member.base_sequence,
13034                    size: member.size,
13035                    selection_group_id: marker.selection_group_id.clone(),
13036                    shape: member.shape.clone(),
13037                })
13038                .collect::<Vec<_>>();
13039            if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
13040                return Poll::Ready(Err(error));
13041            }
13042            state.selection_marker_cursor += 1;
13043            (handles, resolution_sequence)
13044        };
13045
13046        let mut winner_failure = None;
13047        let mut flat_results = Vec::with_capacity(member.size);
13048        if marker.outcome == "failed" {
13049            let resolution_offset = match resolution_sequence
13050                .checked_sub(member.base_sequence)
13051                .and_then(|offset| usize::try_from(offset).ok())
13052            {
13053                Some(offset) if offset < member.size => offset,
13054                _ => {
13055                    return Poll::Ready(Err(invalid_recorded_history(
13056                        "selection_resolution_event_mismatch",
13057                        member.base_sequence,
13058                        "failure event within selected member bounds",
13059                        &resolution_sequence.to_string(),
13060                        "selection failure event is outside the authored member",
13061                    )))
13062                }
13063            };
13064            let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
13065            match leaf.outcome.take() {
13066                Some(Err(error)) => winner_failure = Some(error),
13067                _ => {
13068                    return Poll::Ready(Err(invalid_recorded_history(
13069                        "selection_winner_outcome_mismatch",
13070                        member.base_sequence,
13071                        "exact failed terminal history referenced by SelectionResolved",
13072                        "missing or successful resolution event",
13073                        "selection winner marker disagrees with terminal operation history",
13074                    )))
13075                }
13076            }
13077        } else {
13078            for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
13079                match leaf.outcome.take() {
13080                    Some(Ok(result)) => flat_results.push(result),
13081                    Some(Err(_)) => {
13082                        return Poll::Ready(Err(invalid_recorded_history(
13083                            "selection_winner_outcome_mismatch",
13084                            member.base_sequence,
13085                            "fully completed nested selection member",
13086                            "failed durable leaf",
13087                            "completed selection winner contains a failed leaf",
13088                        )))
13089                    }
13090                    None => {
13091                        return Poll::Ready(Err(invalid_recorded_history(
13092                            "selection_winner_unresolved",
13093                            member.base_sequence,
13094                            "terminal history for every completed winner leaf",
13095                            "pending member history",
13096                            "completed SelectionResolved member has an unfinished durable barrier",
13097                        )))
13098                    }
13099                }
13100            }
13101        }
13102        let value = if winner_failure.is_none() {
13103            let mut flat_results = flat_results.into_iter();
13104            let value = parallel_results_for_shape(&member.shape, &mut flat_results);
13105            match value.into_json_result() {
13106                Ok(value) => Some(value),
13107                Err(error) => return Poll::Ready(Err(error)),
13108            }
13109        } else {
13110            None
13111        };
13112        let winner = handles[member_position].clone();
13113        self.ctx
13114            .state
13115            .lock()
13116            .map_err(|_| Error::WorkflowStatePoisoned)?
13117            .advance_cancellation_selection(&marker.selection_group_id);
13118        Poll::Ready(Ok(SelectionResult {
13119            key: winner.key.clone(),
13120            index: winner.index,
13121            kind: winner.kind.clone(),
13122            identity: winner.identity.clone(),
13123            value,
13124            failure: winner_failure,
13125            winner,
13126            handles,
13127        }))
13128    }
13129}
13130
13131fn selection_operation_identity(
13132    state: &WorkflowState,
13133    kind: &str,
13134    base_sequence: u64,
13135    size: usize,
13136) -> String {
13137    if kind == "group" {
13138        return format!("group:{base_sequence}:{size}");
13139    }
13140    let fields: &[&str] = match kind {
13141        "activity" => &["activity_execution_id"],
13142        "child" => &["child_workflow_run_id"],
13143        "timer" => &["timer_id"],
13144        "signal" => &["signal_wait_id"],
13145        "condition" => &["condition_wait_id"],
13146        _ => &[],
13147    };
13148    for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
13149        for event in state
13150            .history_events
13151            .iter()
13152            .filter(|event| durable_event_sequence(event) == Some(sequence))
13153        {
13154            for field in fields {
13155                if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
13156                    if !identity.is_empty() {
13157                        return identity.to_string();
13158                    }
13159                }
13160            }
13161        }
13162    }
13163    String::new()
13164}
13165
13166fn validated_selection_resolution_sequence(
13167    state: &WorkflowState,
13168    marker: &SelectionMarker,
13169    member: &SelectionMemberPlan,
13170    expected_identity: &str,
13171) -> Result<u64> {
13172    if expected_identity.is_empty() {
13173        return Err(invalid_recorded_history(
13174            "selection_operation_identity_missing",
13175            member.base_sequence,
13176            &format!(
13177                "durable {} resource identity from scheduled/open history",
13178                member.kind
13179            ),
13180            "missing operation identity",
13181            "selection member history is missing its canonical durable identity",
13182        ));
13183    }
13184    if marker.operation_identity != expected_identity {
13185        return Err(invalid_recorded_history(
13186            "selection_operation_identity_mismatch",
13187            member.base_sequence,
13188            expected_identity,
13189            &marker.operation_identity,
13190            "selection winner identity does not match durable scheduled/open history",
13191        ));
13192    }
13193
13194    let failure_types = [
13195        "ActivityFailed",
13196        "ActivityCancelled",
13197        "ActivityTimedOut",
13198        "ChildRunFailed",
13199        "ChildRunCancelled",
13200        "ChildRunTerminated",
13201    ];
13202    let success_types = [
13203        "ActivityCompleted",
13204        "ChildRunCompleted",
13205        "TimerFired",
13206        "SignalApplied",
13207        "ConditionWaitSatisfied",
13208        "ConditionWaitTimedOut",
13209    ];
13210    let terminal_types: &[&str] = if marker.outcome == "failed" {
13211        &failure_types
13212    } else {
13213        &success_types
13214    };
13215    let mut candidates = Vec::new();
13216    let member_commands = (member.base_sequence
13217        ..member.base_sequence.saturating_add(member.size as u64))
13218        .filter_map(|sequence| selection_member_recorded_command(state, sequence))
13219        .map(|command| (command.sequence(), command))
13220        .collect::<BTreeMap<_, _>>();
13221    let mut resolved_conditions = BTreeSet::new();
13222    for event in state.history_events.iter() {
13223        let Some(sequence) = durable_event_sequence(event) else {
13224            continue;
13225        };
13226        let Some(command) = member_commands.get(&sequence) else {
13227            continue;
13228        };
13229        if !terminal_types.contains(&event.event_type.as_str()) {
13230            continue;
13231        }
13232        if matches!(command, RecordedCommand::ConditionWait { .. }) {
13233            if !matches!(
13234                event.event_type.as_str(),
13235                "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13236            ) && !(event.event_type == "TimerFired"
13237                && event.payload.get("timer_kind").and_then(Value::as_str)
13238                    == Some("condition_timeout"))
13239            {
13240                continue;
13241            }
13242            if !resolved_conditions.insert(sequence) {
13243                continue;
13244            }
13245        }
13246        let event_id = event
13247            .raw
13248            .get("id")
13249            .or_else(|| event.raw.get("event_id"))
13250            .and_then(Value::as_str)
13251            .filter(|value| !value.is_empty())
13252            .ok_or_else(|| {
13253                invalid_recorded_history(
13254                    "selection_resolution_event_id_missing",
13255                    member.base_sequence,
13256                    "terminal selection history with a durable event id",
13257                    &event.payload.to_string(),
13258                    "selection terminal history cannot be bound to its winner marker",
13259                )
13260            })?;
13261        candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
13262    }
13263    let resolution = if marker.outcome == "failed" {
13264        candidates.first()
13265    } else {
13266        candidates.last()
13267    };
13268    let Some((event_id, event_type, sequence)) = resolution else {
13269        return Err(invalid_recorded_history(
13270            "selection_resolution_event_missing",
13271            member.base_sequence,
13272            "terminal history for the selected member",
13273            &format!("{:?}", marker.member_key),
13274            "selection winner marker has no matching durable terminal event",
13275        ));
13276    };
13277    if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
13278        return Err(invalid_recorded_history(
13279            "selection_resolution_event_mismatch",
13280            member.base_sequence,
13281            &format!("{event_type}:{event_id}"),
13282            &format!(
13283                "{}:{}",
13284                marker.resolution_event_type, marker.resolution_event_id
13285            ),
13286            "selection winner marker does not reference the event that made its member terminal",
13287        ));
13288    }
13289    Ok(*sequence)
13290}
13291
13292fn selection_member_recorded_command(
13293    state: &WorkflowState,
13294    authored_sequence: u64,
13295) -> Option<&RecordedCommand> {
13296    let original = state
13297        .recorded_commands
13298        .iter()
13299        .find(|command| command.sequence() == authored_sequence)?;
13300    let RecordedCommand::ConditionWait {
13301        occurrence_id,
13302        parallel_group_path: Some(_),
13303        ..
13304    } = original
13305    else {
13306        return Some(original);
13307    };
13308    // Decoding proves each physical reopen has the same authored identity and
13309    // group path after a satisfied predecessor. Only its latest wait can make
13310    // this logical member terminal.
13311    state
13312        .recorded_commands
13313        .iter()
13314        .filter(|command| {
13315            matches!(command, RecordedCommand::ConditionWait {
13316                occurrence_id: candidate, ..
13317            } if candidate == occurrence_id)
13318        })
13319        .max_by_key(|command| command.sequence())
13320}
13321
13322fn recorded_selection_member_outcome(
13323    state: &WorkflowState,
13324    handle: &DurableOperationHandle,
13325) -> Result<Option<ParallelResult>> {
13326    for event in state.history_events.iter() {
13327        let Some(sequence) = durable_event_sequence(event) else {
13328            continue;
13329        };
13330        if sequence < handle.base_sequence
13331            || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
13332            || !matches!(
13333                event.event_type.as_str(),
13334                "ActivityFailed"
13335                    | "ActivityCancelled"
13336                    | "ActivityTimedOut"
13337                    | "ChildRunFailed"
13338                    | "ChildRunCancelled"
13339                    | "ChildRunTerminated"
13340            )
13341        {
13342            continue;
13343        }
13344        let Some(command) = state
13345            .recorded_commands
13346            .iter()
13347            .find(|command| command.sequence() == sequence)
13348        else {
13349            continue;
13350        };
13351        match command {
13352            RecordedCommand::Activity {
13353                outcome: Some(Err(failure)),
13354                ..
13355            } => return Err(Error::ActivityFailed(failure.clone())),
13356            RecordedCommand::ChildWorkflow {
13357                outcome: Some(Err(failure)),
13358                ..
13359            } => return Err(Error::ChildWorkflowFailed(failure.clone())),
13360            _ => {}
13361        }
13362    }
13363
13364    let mut results = Vec::with_capacity(handle.size);
13365    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
13366        let Some(command) = selection_member_recorded_command(state, sequence) else {
13367            return Ok(None);
13368        };
13369        let result = match command {
13370            RecordedCommand::Activity { outcome, .. } => match outcome {
13371                Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
13372                Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
13373                None => return Ok(None),
13374            },
13375            RecordedCommand::Timer { fired, .. } => {
13376                if !fired {
13377                    return Ok(None);
13378                }
13379                ParallelAvroResult::Timer
13380            }
13381            RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
13382                Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
13383                Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
13384                None => return Ok(None),
13385            },
13386            RecordedCommand::SignalWait { value, .. } => match value {
13387                Some(value) => ParallelAvroResult::Signal(value.clone()),
13388                None => return Ok(None),
13389            },
13390            RecordedCommand::ConditionWait { result, .. } => match result {
13391                Some(result) => ParallelAvroResult::Condition(*result),
13392                None => return Ok(None),
13393            },
13394            other => {
13395                return Err(command_mismatch(
13396                    other,
13397                    format!("selected {} member", handle.kind),
13398                ))
13399            }
13400        };
13401        results.push(result);
13402    }
13403    let mut results = results.into_iter();
13404    parallel_results_for_shape(&handle.shape, &mut results)
13405        .into_json_result()
13406        .map(Some)
13407}
13408
13409fn recorded_selection_member_is_terminal(
13410    state: &WorkflowState,
13411    handle: &DurableOperationHandle,
13412) -> bool {
13413    let mut completed = 0usize;
13414    let mut all_completed = true;
13415    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
13416        let Some(command) = selection_member_recorded_command(state, sequence) else {
13417            all_completed = false;
13418            continue;
13419        };
13420        let terminal = match command {
13421            RecordedCommand::Activity {
13422                outcome: Some(Err(_)),
13423                ..
13424            }
13425            | RecordedCommand::ChildWorkflow {
13426                outcome: Some(Err(_)),
13427                ..
13428            } => return true,
13429            RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
13430            RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
13431            RecordedCommand::Timer { fired, .. } => *fired,
13432            RecordedCommand::SignalWait { value, .. } => value.is_some(),
13433            RecordedCommand::ConditionWait { result, .. } => result.is_some(),
13434            RecordedCommand::SearchAttributes { .. }
13435            | RecordedCommand::CancellationScope { .. }
13436            | RecordedCommand::SideEffect { .. }
13437            | RecordedCommand::VersionMarker { .. }
13438            | RecordedCommand::Memo { .. }
13439            | RecordedCommand::CancellationBoundary { .. }
13440            | RecordedCommand::CancellationGroup { .. } => false,
13441        };
13442        if !terminal {
13443            all_completed = false;
13444            continue;
13445        }
13446        completed += 1;
13447    }
13448    all_completed && completed == handle.size
13449}
13450
13451fn selection_cancellation_for_handle(
13452    state: &WorkflowState,
13453    handle: &DurableOperationHandle,
13454) -> Result<bool> {
13455    let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
13456        recorded.selection_group_id == handle.selection_group_id
13457            && recorded.member_base_sequence == handle.base_sequence
13458    }) else {
13459        return Ok(false);
13460    };
13461    validate_selection_cancellation_marker(marker, handle)?;
13462    Ok(true)
13463}
13464
13465fn validate_selection_cancellations_for_handles(
13466    state: &WorkflowState,
13467    handles: &[DurableOperationHandle],
13468) -> Result<()> {
13469    let Some(group_id) = handles
13470        .first()
13471        .map(|handle| handle.selection_group_id.as_str())
13472    else {
13473        return Ok(());
13474    };
13475    for marker in state
13476        .cancelled_selection_members
13477        .iter()
13478        .filter(|marker| marker.selection_group_id == group_id)
13479    {
13480        let Some(handle) = handles
13481            .iter()
13482            .find(|handle| handle.base_sequence == marker.member_base_sequence)
13483        else {
13484            return Err(invalid_recorded_history(
13485                "selection_cancellation_member_mismatch",
13486                marker.member_base_sequence,
13487                "SelectionOperationCancelled matching an authored selection handle",
13488                &format!("{marker:?}"),
13489                "selection cancellation member base does not name an authored member",
13490            ));
13491        };
13492        validate_selection_cancellation_marker(marker, handle)?;
13493    }
13494    Ok(())
13495}
13496
13497fn validate_selection_cancellation_marker(
13498    marker: &SelectionCancellation,
13499    handle: &DurableOperationHandle,
13500) -> Result<()> {
13501    if marker.selection_group_id != handle.selection_group_id
13502        || marker.member_key != handle.key
13503        || marker.member_index != handle.index
13504        || marker.member_base_sequence != handle.base_sequence
13505        || marker.member_size != handle.size
13506        || marker.operation_kind != handle.kind
13507        || marker.operation_identity != handle.identity
13508    {
13509        return Err(invalid_recorded_history(
13510            "selection_cancellation_member_mismatch",
13511            handle.base_sequence,
13512            "SelectionOperationCancelled matching the authored selection handle",
13513            &format!("{marker:?}"),
13514            "selection cancellation history targets different authored member metadata",
13515        ));
13516    }
13517    Ok(())
13518}
13519
13520/// Future returned by [`DurableOperationHandle::await_result`].
13521pub struct DurableOperationAwaitCall {
13522    handle: DurableOperationHandle,
13523}
13524
13525impl Future for DurableOperationAwaitCall {
13526    type Output = Result<ParallelResult>;
13527
13528    fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
13529        let mut state = match self.handle.ctx.state.lock() {
13530            Ok(state) => state,
13531            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
13532        };
13533        if let Err(error) = state.replay_selection_handle_cancellation(&self.handle) {
13534            return Poll::Ready(Err(error));
13535        }
13536        match state.prepare_selection_handle_cancellation(&self.handle) {
13537            Ok(true) => return Poll::Pending,
13538            Ok(false) => {}
13539            Err(error) => return Poll::Ready(Err(error)),
13540        }
13541        match selection_cancellation_for_handle(&state, &self.handle) {
13542            Err(error) => return Poll::Ready(Err(error)),
13543            Ok(false) => {}
13544            Ok(true) => {
13545                state.advance_cancellation_receipt(
13546                    &self.handle.selection_group_id,
13547                    self.handle.base_sequence,
13548                );
13549                return Poll::Ready(Err(Error::DurableOperationCancelled(
13550                    DurableOperationCancelled {
13551                        selection_group_id: self.handle.selection_group_id.clone(),
13552                        member_key: self.handle.key.clone(),
13553                        member_index: self.handle.index,
13554                        operation_kind: self.handle.kind.clone(),
13555                        operation_identity: self.handle.identity.clone(),
13556                    },
13557                )));
13558            }
13559        }
13560        match recorded_selection_member_outcome(&state, &self.handle) {
13561            Ok(Some(result)) => {
13562                state.advance_cancellation_handle(&self.handle, false);
13563                Poll::Ready(Ok(result))
13564            }
13565            Ok(None) => Poll::Pending,
13566            Err(error) => {
13567                if !workflow_task_integrity_error(&error) {
13568                    state.advance_cancellation_handle(&self.handle, true);
13569                }
13570                Poll::Ready(Err(error))
13571            }
13572        }
13573    }
13574}
13575
13576fn validate_selection_delivery_handle(
13577    state: &WorkflowState,
13578    handle: &DurableOperationHandle,
13579) -> Result<()> {
13580    let path = state
13581        .recorded_commands
13582        .iter()
13583        .find(|command| command.sequence() == handle.base_sequence)
13584        .and_then(|command| match command {
13585            RecordedCommand::Activity {
13586                parallel_group_path,
13587                ..
13588            }
13589            | RecordedCommand::ChildWorkflow {
13590                parallel_group_path,
13591                ..
13592            }
13593            | RecordedCommand::Timer {
13594                parallel_group_path,
13595                ..
13596            }
13597            | RecordedCommand::SignalWait {
13598                parallel_group_path,
13599                ..
13600            }
13601            | RecordedCommand::ConditionWait {
13602                parallel_group_path,
13603                ..
13604            } => parallel_group_path.as_ref(),
13605            _ => None,
13606        })
13607        .and_then(|path| path.first());
13608    let matches = path.is_some_and(|entry| {
13609        entry.parallel_group_mode.as_deref() == Some("select")
13610            && entry.parallel_group_id == handle.selection_group_id
13611            && entry.selection_member_key.as_ref() == Some(&handle.key)
13612            && entry.selection_member_index == Some(handle.index)
13613            && entry.selection_member_base_sequence == Some(handle.base_sequence)
13614            && entry.selection_member_size == Some(handle.size)
13615            && entry.selection_member_kind.as_deref() == Some(handle.kind.as_str())
13616    });
13617    if !matches
13618        || selection_operation_identity(state, &handle.kind, handle.base_sequence, handle.size)
13619            != handle.identity
13620    {
13621        return Err(invalid_recorded_history(
13622            "cooperative_cancellation_call_mismatch",
13623            handle.base_sequence,
13624            "selection handle matching its authored durable identity",
13625            &format!("{handle:?}"),
13626            "selection handle identity or member metadata changed before cancellation replay",
13627        ));
13628    }
13629    Ok(())
13630}
13631
13632/// Future returned by [`DurableOperationHandle::cancel`].
13633pub struct CancelDurableOperationCall {
13634    handle: DurableOperationHandle,
13635    emitted: bool,
13636}
13637
13638impl Future for CancelDurableOperationCall {
13639    type Output = Result<()>;
13640
13641    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
13642        let ctx = self.handle.ctx.clone();
13643        let mut state = match ctx.state.lock() {
13644            Ok(state) => state,
13645            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
13646        };
13647        match selection_cancellation_for_handle(&state, &self.handle) {
13648            Err(error) => return Poll::Ready(Err(error)),
13649            Ok(true) => {
13650                state.advance_cancellation_receipt(
13651                    &self.handle.selection_group_id,
13652                    self.handle.base_sequence,
13653                );
13654                return Poll::Ready(Ok(()));
13655            }
13656            Ok(false) => {}
13657        }
13658        if recorded_selection_member_is_terminal(&state, &self.handle) {
13659            return Poll::Ready(Ok(()));
13660        }
13661        if !self.emitted {
13662            state.commands.push(json!({
13663                "type": "cancel_selection_operation",
13664                "selection_group_id": self.handle.selection_group_id,
13665                "member_key": self.handle.key,
13666                "member_index": self.handle.index,
13667                "member_base_sequence": self.handle.base_sequence,
13668                "member_size": self.handle.size,
13669                "operation_kind": self.handle.kind,
13670                "operation_identity": self.handle.identity,
13671            }));
13672            self.emitted = true;
13673        }
13674        // The cancellation command is only a request. Do not expose workflow
13675        // state after cancel until committed SelectionOperationCancelled
13676        // history proves that the request won the race with member completion.
13677        Poll::Pending
13678    }
13679}
13680
13681trait PollNestedResultExt<T> {
13682    fn flatten_result(self) -> Poll<Result<T>>;
13683}
13684
13685impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
13686    fn flatten_result(self) -> Poll<Result<T>> {
13687        match self {
13688            Poll::Ready(Ok(result)) => Poll::Ready(result),
13689            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
13690            Poll::Pending => Poll::Pending,
13691        }
13692    }
13693}
13694
13695struct SagaCompensation {
13696    activity_type: String,
13697    options: ActivityOptions,
13698    arguments: AvroValue,
13699    registration_order: usize,
13700}
13701
13702/// Workflow-local deterministic saga compensation helper.
13703///
13704/// Register each compensation only after its forward step succeeds. Passing
13705/// the forward `Result` to [`Saga::finish`] runs compensations sequentially in
13706/// reverse registration order after any failure, including cooperative
13707/// cancellation. Each compensation is an ordinary durable activity, so replay,
13708/// duplicate delivery, and worker restart use existing history semantics.
13709pub struct Saga {
13710    ctx: WorkflowContext,
13711    compensations: Vec<SagaCompensation>,
13712}
13713
13714impl Saga {
13715    fn new(ctx: WorkflowContext) -> Self {
13716        Self {
13717            ctx,
13718            compensations: Vec::new(),
13719        }
13720    }
13721
13722    pub fn add_compensation<T: Serialize>(
13723        &mut self,
13724        activity_type: impl Into<String>,
13725        args: T,
13726    ) -> Result<&mut Self> {
13727        self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
13728    }
13729
13730    pub fn add_compensation_with_options<T: Serialize>(
13731        &mut self,
13732        activity_type: impl Into<String>,
13733        options: ActivityOptions,
13734        args: T,
13735    ) -> Result<&mut Self> {
13736        let activity_type = activity_type.into();
13737        if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
13738            return Err(Error::Codec(
13739                "saga compensation activity type must be non-empty without surrounding whitespace"
13740                    .to_string(),
13741            ));
13742        }
13743        options.validate().map_err(Error::InvalidActivityOptions)?;
13744        let arguments = AvroValue::from_serialize(&args)?;
13745        let registration_order = self.compensations.len() + 1;
13746        self.compensations.push(SagaCompensation {
13747            activity_type,
13748            options,
13749            arguments,
13750            registration_order,
13751        });
13752        Ok(self)
13753    }
13754
13755    /// Compensate `initiating_failure` and return the failure that must remain.
13756    pub async fn compensate(mut self, initiating_failure: Error) -> Error {
13757        let _shield = match self.ctx.cancellation_shield() {
13758            Ok(shield) => shield,
13759            Err(error) => return error,
13760        };
13761        while let Some(compensation) = self.compensations.pop() {
13762            if let Err(compensation_failure) = self
13763                .ctx
13764                .activity_with_options(
13765                    compensation.activity_type.clone(),
13766                    compensation.options,
13767                    compensation.arguments,
13768                )
13769                .await
13770            {
13771                if workflow_task_integrity_error(&compensation_failure) {
13772                    return compensation_failure;
13773                }
13774                return Error::SagaCompensationFailed(SagaCompensationFailure {
13775                    initiating_failure: Box::new(initiating_failure),
13776                    compensation_failure: Box::new(compensation_failure),
13777                    compensation_activity_type: compensation.activity_type,
13778                    compensation_registration_order: compensation.registration_order,
13779                });
13780            }
13781        }
13782        initiating_failure
13783    }
13784
13785    /// Return a successful forward value or compensate and preserve its failure.
13786    pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
13787        match outcome {
13788            Ok(value) => Ok(value),
13789            Err(error) => Err(self.compensate(error).await),
13790        }
13791    }
13792}
13793
13794pub struct ActivityCall {
13795    ctx: WorkflowContext,
13796    activity_type: String,
13797    options: ActivityOptions,
13798    worker_session: Option<WorkerSessionOptions>,
13799    args: Option<Result<AvroValue>>,
13800    scheduled: bool,
13801    local: bool,
13802    local_result: Option<Arc<Mutex<Option<ActivityOutcome>>>>,
13803    parallel_group_path: Vec<ParallelGroupMetadata>,
13804}
13805
13806impl ActivityCall {
13807    fn poll_avro_value(
13808        mut self: Pin<&mut Self>,
13809        _cx: &mut TaskContext<'_>,
13810    ) -> Poll<Result<AvroValue>> {
13811        let ctx = self.ctx.clone();
13812        let mut state = match ctx.state.lock() {
13813            Ok(state) => state,
13814            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
13815        };
13816
13817        if let Some(result) = &self.local_result {
13818            return match result.lock() {
13819                Ok(mut result) => result.take().map_or(Poll::Pending, |result| {
13820                    Poll::Ready(result.map_err(Error::ActivityFailed))
13821                }),
13822                Err(_) => Poll::Ready(Err(Error::WorkflowStatePoisoned)),
13823            };
13824        }
13825        if self.scheduled {
13826            return Poll::Pending;
13827        }
13828
13829        let options = match self.options.validate() {
13830            Ok(options) => options,
13831            Err(error) => {
13832                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
13833            }
13834        };
13835        if self.local {
13836            if self.worker_session.is_some() {
13837                return Poll::Ready(Err(Error::WorkerLoop(
13838                    "local activities cannot use worker-session routing".into(),
13839                )));
13840            }
13841            if let Err(error) = local_activity::validate(&options) {
13842                return Poll::Ready(Err(error));
13843            }
13844        }
13845        let session = match self
13846            .worker_session
13847            .as_ref()
13848            .map(WorkerSessionOptions::to_wire)
13849            .transpose()
13850        {
13851            Ok(value) => value,
13852            Err(error) => return Poll::Ready(Err(error)),
13853        };
13854        let session_queue = session
13855            .as_ref()
13856            .and_then(|session| session["queue"].as_str());
13857        let task_queue = options
13858            .task_queue
13859            .clone()
13860            .or_else(|| session_queue.map(str::to_owned))
13861            .unwrap_or_else(|| state.task_queue.clone());
13862        if session_queue.is_some_and(|queue| queue != task_queue) {
13863            return Poll::Ready(Err(Error::WorkerLoop(
13864                "worker-session queue must match the activity task queue".into(),
13865            )));
13866        }
13867        let current_recorded_options = RecordedActivityOptions {
13868            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
13869            execution_mode: RecordedSnapshotValue::Known(self.local.then(|| "local".to_string())),
13870            retry_policy: current_activity_retry_snapshot(&options),
13871            worker_session: session,
13872        };
13873
13874        let cursor = state.command_cursor;
13875        let call_kind = if self.local {
13876            CancellationCallKind::LocalActivity
13877        } else {
13878            CancellationCallKind::Activity
13879        };
13880        if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) {
13881            return Poll::Ready(Err(error));
13882        }
13883        let recorded = match state.cancellation_replay_command(cursor, call_kind) {
13884            Ok(recorded) => recorded,
13885            Err(error) => return Poll::Ready(Err(error)),
13886        };
13887        if let Some(recorded) = recorded {
13888            let sequence = recorded.sequence();
13889            match recorded {
13890                RecordedCommand::Activity {
13891                    activity_type,
13892                    cancellation_policy,
13893                    options: recorded_options,
13894                    outcome,
13895                    parallel_group_path,
13896                    ..
13897                } => {
13898                    let current_policy = options
13899                        .cancellation_policy
13900                        .unwrap_or(CancellationPolicy::TryCancel)
13901                        .as_str();
13902                    if cancellation_policy != current_policy {
13903                        return Poll::Ready(Err(Error::NonDeterministicReplay(
13904                            ReplayFailure::new(
13905                                "activity_cancellation_policy_changed",
13906                                Some(sequence),
13907                                Some(cancellation_policy),
13908                                Some(current_policy.to_string()),
13909                                "Activity cancellation policy changed during replay",
13910                            ),
13911                        )));
13912                    }
13913                    if let Err(error) = ensure_parallel_path_matches(
13914                        sequence,
13915                        parallel_group_path.as_deref(),
13916                        &self.parallel_group_path,
13917                    ) {
13918                        return Poll::Ready(Err(error));
13919                    }
13920                    if let Some(recorded_type) = activity_type {
13921                        if recorded_type != self.activity_type {
13922                            return Poll::Ready(Err(Error::NonDeterministicReplay(
13923                                ReplayFailure::new(
13924                                    "recorded_command_detail_mismatch",
13925                                    Some(sequence),
13926                                    Some(format!("activity:{recorded_type}")),
13927                                    Some(format!("activity:{}", self.activity_type)),
13928                                    "recorded activity type differs from the current workflow command",
13929                                ),
13930                            )));
13931                        }
13932                    }
13933                    if let Some(recorded_options) = recorded_options {
13934                        if recorded_options.worker_session
13935                            != current_recorded_options.worker_session
13936                        {
13937                            return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
13938                                "activity_worker_session_mismatch", Some(sequence),
13939                                Some(activity_options_description(&recorded_options)),
13940                                Some(activity_options_description(&current_recorded_options)),
13941                                "recorded worker-session routing differs from the current activity",
13942                            ))));
13943                        }
13944                        if !recorded_options
13945                            .task_queue
13946                            .matches_current(&current_recorded_options.task_queue)
13947                        {
13948                            return Poll::Ready(Err(Error::NonDeterministicReplay(
13949                                ReplayFailure::new(
13950                                    "activity_task_queue_mismatch",
13951                                    Some(sequence),
13952                                    Some(activity_options_description(&recorded_options)),
13953                                    Some(activity_options_description(&current_recorded_options)),
13954                                    "recorded activity task queue differs from the current workflow command",
13955                                ),
13956                            )));
13957                        }
13958                        if (self.local
13959                            && matches!(
13960                                recorded_options.execution_mode,
13961                                RecordedSnapshotValue::Unknown
13962                            ))
13963                            || !recorded_options
13964                                .execution_mode
13965                                .matches_current(&current_recorded_options.execution_mode)
13966                        {
13967                            return Poll::Ready(Err(Error::NonDeterministicReplay(
13968                                ReplayFailure::new(
13969                                    "activity_execution_mode_mismatch",
13970                                    Some(sequence),
13971                                    Some(activity_options_description(&recorded_options)),
13972                                    Some(activity_options_description(&current_recorded_options)),
13973                                    "recorded activity execution mode differs from the current workflow command",
13974                                ),
13975                            )));
13976                        }
13977                        if !recorded_options
13978                            .retry_policy
13979                            .matches_current(&current_recorded_options.retry_policy)
13980                        {
13981                            return Poll::Ready(Err(Error::NonDeterministicReplay(
13982                                ReplayFailure::new(
13983                                    "activity_retry_policy_mismatch",
13984                                    Some(sequence),
13985                                    Some(activity_options_description(&recorded_options)),
13986                                    Some(activity_options_description(&current_recorded_options)),
13987                                    "recorded activity retry policy differs from the current workflow command",
13988                                ),
13989                            )));
13990                        }
13991                    }
13992                    if let Err(error) = state.replay_cancellation_at(cursor, call_kind) {
13993                        return Poll::Ready(Err(error));
13994                    }
13995                    match state.prepare_scalar_cancellation(
13996                        cursor,
13997                        call_kind,
13998                        &self.parallel_group_path,
13999                    ) {
14000                        Ok(true) => {
14001                            self.scheduled = true;
14002                            return Poll::Pending;
14003                        }
14004                        Ok(false) => {}
14005                        Err(error) => return Poll::Ready(Err(error)),
14006                    }
14007                    state.command_cursor += 1;
14008                    if let Some(outcome) = outcome {
14009                        state.advance_cancellation_sequence(sequence, &self.parallel_group_path);
14010                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
14011                    }
14012                    state.matched_recorded_pending = true;
14013                    self.scheduled = true;
14014                    return Poll::Pending;
14015                }
14016                other => {
14017                    return Poll::Ready(Err(command_mismatch(
14018                        &other,
14019                        format!("activity:{}", self.activity_type),
14020                    )));
14021                }
14022            }
14023        }
14024
14025        if !self.scheduled {
14026            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
14027                Ok(args) => args,
14028                Err(error) => return Poll::Ready(Err(error)),
14029            };
14030            let arguments = normalize_avro_arguments(args);
14031            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
14032                Ok(envelope) => envelope,
14033                Err(error) => return Poll::Ready(Err(error)),
14034            };
14035
14036            let mut command = serde_json::Map::from_iter([
14037                (
14038                    "type".to_string(),
14039                    json!(if self.local {
14040                        "record_local_activity"
14041                    } else {
14042                        "schedule_activity"
14043                    }),
14044                ),
14045                (
14046                    "activity_type".to_string(),
14047                    json!(self.activity_type.clone()),
14048                ),
14049                ("queue".to_string(), json!(task_queue)),
14050                ("arguments".to_string(), envelope),
14051            ]);
14052            for (field, value) in [
14053                ("start_to_close_timeout", options.start_to_close_timeout),
14054                (
14055                    "schedule_to_start_timeout",
14056                    options.schedule_to_start_timeout,
14057                ),
14058                (
14059                    "schedule_to_close_timeout",
14060                    options.schedule_to_close_timeout,
14061                ),
14062                ("heartbeat_timeout", options.heartbeat_timeout),
14063            ] {
14064                if let Some(value) = value {
14065                    command.insert(field.to_string(), json!(value));
14066                }
14067            }
14068            if let Some(retry_policy) = &options.retry_policy {
14069                command.insert("retry_policy".to_string(), retry_policy.clone());
14070            }
14071            if let Some(policy) = options.cancellation_policy {
14072                command.insert("cancellation_policy".to_string(), json!(policy.as_str()));
14073            }
14074            if let Some(session) = current_recorded_options.worker_session {
14075                command.insert("worker_session".into(), session);
14076            }
14077            apply_parallel_group_path(&mut command, &self.parallel_group_path);
14078            ctx.apply_scope_membership(&mut command);
14079            match state.prepare_scalar_cancellation(cursor, call_kind, &self.parallel_group_path) {
14080                Ok(true) => {
14081                    self.scheduled = true;
14082                    return Poll::Pending;
14083                }
14084                Ok(false) => {}
14085                Err(error) => return Poll::Ready(Err(error)),
14086            }
14087            if self.local {
14088                command.remove("queue");
14089                command.insert("execution_mode".into(), json!("local"));
14090                let result = Arc::new(Mutex::new(None));
14091                let command_index = state.commands.len();
14092                state.local_activity_requests.push(local_activity::Request {
14093                    command_index,
14094                    options,
14095                    arguments,
14096                    result: result.clone(),
14097                });
14098                self.local_result = Some(result);
14099            }
14100            state.commands.push(Value::Object(command));
14101            self.scheduled = true;
14102        }
14103
14104        Poll::Pending
14105    }
14106}
14107
14108impl Future for ActivityCall {
14109    type Output = Result<Value>;
14110
14111    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
14112        match self.poll_avro_value(cx) {
14113            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
14114            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
14115            Poll::Pending => Poll::Pending,
14116        }
14117    }
14118}
14119
14120/// Future returned by [`WorkflowContext::sleep`].
14121pub struct TimerCall {
14122    ctx: WorkflowContext,
14123    delay_seconds: Option<u64>,
14124    scheduled: bool,
14125    matched_pending: bool,
14126    parallel_group_path: Vec<ParallelGroupMetadata>,
14127}
14128
14129impl Future for TimerCall {
14130    type Output = Result<()>;
14131
14132    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
14133        if self.matched_pending {
14134            return Poll::Pending;
14135        }
14136
14137        let ctx = self.ctx.clone();
14138        let Some(requested_delay) = self.delay_seconds else {
14139            return Poll::Ready(Err(Error::TimerDurationOverflow));
14140        };
14141        let mut state = match ctx.state.lock() {
14142            Ok(state) => state,
14143            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14144        };
14145
14146        let cursor = state.command_cursor;
14147        if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) {
14148            return Poll::Ready(Err(error));
14149        }
14150        let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Timer)
14151        {
14152            Ok(recorded) => recorded,
14153            Err(error) => return Poll::Ready(Err(error)),
14154        };
14155        if let Some(recorded) = recorded {
14156            match recorded {
14157                RecordedCommand::Timer {
14158                    sequence,
14159                    delay_seconds,
14160                    fired,
14161                    parallel_group_path,
14162                    ..
14163                } => {
14164                    if let Err(error) =
14165                        state.validate_scope_cleanup_timer(&ctx.cancellation_scope_id, sequence)
14166                    {
14167                        return Poll::Ready(Err(error));
14168                    }
14169                    if let Err(error) = ensure_parallel_path_matches(
14170                        sequence,
14171                        parallel_group_path.as_deref(),
14172                        &self.parallel_group_path,
14173                    ) {
14174                        return Poll::Ready(Err(error));
14175                    }
14176                    if delay_seconds != requested_delay {
14177                        return Poll::Ready(Err(Error::NonDeterministicReplay(
14178                            ReplayFailure::new(
14179                                "timer_delay_mismatch",
14180                                Some(sequence),
14181                                Some(format!("timer:{delay_seconds}s")),
14182                                Some(format!("timer:{requested_delay}s")),
14183                                "recorded timer delay differs from the current workflow command",
14184                            ),
14185                        )));
14186                    }
14187                    if let Err(error) =
14188                        state.replay_cancellation_at(cursor, CancellationCallKind::Timer)
14189                    {
14190                        return Poll::Ready(Err(error));
14191                    }
14192                    match state.prepare_scalar_cancellation(
14193                        cursor,
14194                        CancellationCallKind::Timer,
14195                        &self.parallel_group_path,
14196                    ) {
14197                        Ok(true) => {
14198                            self.scheduled = true;
14199                            self.matched_pending = true;
14200                            return Poll::Pending;
14201                        }
14202                        Ok(false) => {}
14203                        Err(error) => return Poll::Ready(Err(error)),
14204                    }
14205                    state.command_cursor += 1;
14206                    if fired {
14207                        state.advance_cancellation_sequence(sequence, &self.parallel_group_path);
14208                        return Poll::Ready(Ok(()));
14209                    }
14210                    state.matched_recorded_pending = true;
14211                    self.scheduled = true;
14212                    self.matched_pending = true;
14213                    return Poll::Pending;
14214                }
14215                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
14216            }
14217        }
14218
14219        if !self.scheduled {
14220            let mut command = serde_json::Map::from_iter([
14221                ("type".to_string(), json!("start_timer")),
14222                ("delay_seconds".to_string(), json!(requested_delay)),
14223            ]);
14224            match state.scope_cleanup_timer_proof(&ctx.cancellation_scope_id) {
14225                Ok(Some(proof)) => {
14226                    command.insert("cancellation_cleanup".into(), proof);
14227                }
14228                Ok(None) => {}
14229                Err(error) => return Poll::Ready(Err(error)),
14230            }
14231            apply_parallel_group_path(&mut command, &self.parallel_group_path);
14232            ctx.apply_scope_membership(&mut command);
14233            match state.prepare_scalar_cancellation(
14234                cursor,
14235                CancellationCallKind::Timer,
14236                &self.parallel_group_path,
14237            ) {
14238                Ok(true) => {
14239                    self.scheduled = true;
14240                    self.matched_pending = true;
14241                    return Poll::Pending;
14242                }
14243                Ok(false) => {}
14244                Err(error) => return Poll::Ready(Err(error)),
14245            }
14246            state.commands.push(Value::Object(command));
14247            self.scheduled = true;
14248        }
14249
14250        Poll::Pending
14251    }
14252}
14253
14254/// Future returned by [`WorkflowContext::wait_condition`].
14255pub struct ConditionWaitCall {
14256    ctx: WorkflowContext,
14257    options: ConditionWaitOptions,
14258    predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
14259    occurrence_id: Option<String>,
14260    opened_wait: bool,
14261    parallel_group_path: Vec<ParallelGroupMetadata>,
14262}
14263
14264impl Future for ConditionWaitCall {
14265    type Output = Result<ConditionWaitResult>;
14266
14267    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
14268        if self.opened_wait {
14269            return Poll::Pending;
14270        }
14271
14272        let options = match self.options.validate() {
14273            Ok(options) => options,
14274            Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
14275        };
14276        let ctx = self.ctx.clone();
14277        let occurrence_id = match self.occurrence_id.as_ref() {
14278            Some(occurrence_id) => occurrence_id.clone(),
14279            None => {
14280                let mut state = match ctx.state.lock() {
14281                    Ok(state) => state,
14282                    Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14283                };
14284                let ordinal = state.condition_wait_occurrence_counter;
14285                state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
14286                    Some(next) => next,
14287                    None => {
14288                        return Poll::Ready(Err(Error::WorkerLoop(
14289                            "condition wait occurrence counter overflowed".to_string(),
14290                        )))
14291                    }
14292                };
14293                let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
14294                drop(state);
14295                self.occurrence_id = Some(occurrence_id.clone());
14296                occurrence_id
14297            }
14298        };
14299
14300        let recorded_result = {
14301            let mut state = match ctx.state.lock() {
14302                Ok(state) => state,
14303                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14304            };
14305            let initial_cursor = state.command_cursor;
14306            if let Err(error) = ctx.validate_scope_membership(&mut state, initial_cursor) {
14307                return Poll::Ready(Err(error));
14308            }
14309            let recorded = match state
14310                .cancellation_replay_command(initial_cursor, CancellationCallKind::Condition)
14311            {
14312                Ok(recorded) => recorded,
14313                Err(error) => return Poll::Ready(Err(error)),
14314            };
14315            let Some(recorded) = recorded else {
14316                drop(state);
14317                return self.poll_new_condition(options);
14318            };
14319            if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
14320                return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
14321            }
14322
14323            let mut cursor = state.command_cursor;
14324            let mut result = None;
14325            let mut pending_delivery = false;
14326            loop {
14327                if cursor > initial_cursor
14328                    && matches!(
14329                        state.recorded_commands.get(cursor),
14330                        Some(RecordedCommand::CancellationBoundary { original: None, .. })
14331                    )
14332                {
14333                    break;
14334                }
14335                let recorded = match state
14336                    .cancellation_replay_command(cursor, CancellationCallKind::Condition)
14337                {
14338                    Ok(recorded) => recorded,
14339                    Err(error) => return Poll::Ready(Err(error)),
14340                };
14341                let Some(RecordedCommand::ConditionWait {
14342                    sequence,
14343                    occurrence_id: ref recorded_occurrence_id,
14344                    ref condition_key,
14345                    ref predicate_identity,
14346                    timeout_seconds,
14347                    result: recorded_result,
14348                    ref parallel_group_path,
14349                    ..
14350                }) = recorded
14351                else {
14352                    break;
14353                };
14354
14355                if cursor > initial_cursor && recorded_occurrence_id != &occurrence_id {
14356                    break;
14357                }
14358                if let Err(error) = ensure_parallel_path_matches(
14359                    sequence,
14360                    parallel_group_path.as_deref(),
14361                    &self.parallel_group_path,
14362                ) {
14363                    return Poll::Ready(Err(error));
14364                }
14365                if let Err(error) = validate_recorded_condition_wait(
14366                    sequence,
14367                    recorded_occurrence_id,
14368                    condition_key.as_deref(),
14369                    predicate_identity,
14370                    timeout_seconds,
14371                    &occurrence_id,
14372                    &options,
14373                ) {
14374                    return Poll::Ready(Err(error));
14375                }
14376                if result == Some(ConditionWaitResult::TimedOut) {
14377                    return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
14378                        "condition_wait_reopened_after_timeout",
14379                        Some(sequence),
14380                        Some("timed-out condition is terminal".to_string()),
14381                        Some("another physical wait-open".to_string()),
14382                        "condition history reopened one logical wait after its durable timeout",
14383                    ))));
14384                }
14385                if let Err(error) =
14386                    state.replay_cancellation_at(cursor, CancellationCallKind::Condition)
14387                {
14388                    return Poll::Ready(Err(error));
14389                }
14390                match state.prepare_scalar_cancellation(
14391                    cursor,
14392                    CancellationCallKind::Condition,
14393                    &self.parallel_group_path,
14394                ) {
14395                    Ok(pending) => pending_delivery |= pending,
14396                    Err(error) => return Poll::Ready(Err(error)),
14397                }
14398                result = recorded_result;
14399                if result.is_some() {
14400                    state.advance_cancellation_sequence(sequence, &self.parallel_group_path);
14401                }
14402                cursor += 1;
14403            }
14404            state.command_cursor = cursor;
14405            if pending_delivery {
14406                self.opened_wait = true;
14407                return Poll::Pending;
14408            }
14409            result
14410        };
14411
14412        if let Some(result) = recorded_result {
14413            return Poll::Ready(Ok(result));
14414        }
14415
14416        self.poll_open_condition(options, true)
14417    }
14418}
14419
14420impl ConditionWaitCall {
14421    fn poll_new_condition(
14422        self: Pin<&mut Self>,
14423        options: ValidatedConditionWaitOptions,
14424    ) -> Poll<Result<ConditionWaitResult>> {
14425        self.poll_open_condition(options, false)
14426    }
14427
14428    fn poll_open_condition(
14429        mut self: Pin<&mut Self>,
14430        options: ValidatedConditionWaitOptions,
14431        recorded_wait: bool,
14432    ) -> Poll<Result<ConditionWaitResult>> {
14433        {
14434            let ctx = self.ctx.clone();
14435            let mut state = match ctx.state.lock() {
14436                Ok(state) => state,
14437                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14438            };
14439            let cursor = state.command_cursor;
14440            match state.prepare_scalar_cancellation(
14441                cursor,
14442                CancellationCallKind::Condition,
14443                &self.parallel_group_path,
14444            ) {
14445                Ok(true) => {
14446                    self.opened_wait = true;
14447                    return Poll::Pending;
14448                }
14449                Ok(false) => {}
14450                Err(error) => return Poll::Ready(Err(error)),
14451            }
14452        }
14453        let selection_member = self
14454            .parallel_group_path
14455            .first()
14456            .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
14457        match (self.predicate)() {
14458            Ok(true) if !selection_member || recorded_wait => {
14459                return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
14460            }
14461            Ok(_) => {}
14462            Err(error) => return Poll::Ready(Err(error)),
14463        }
14464        if options.timeout_seconds == Some(0) && !selection_member {
14465            return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
14466        }
14467
14468        let ctx = self.ctx.clone();
14469        let mut state = match ctx.state.lock() {
14470            Ok(state) => state,
14471            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14472        };
14473        let mut command = serde_json::Map::from_iter([
14474            ("type".to_string(), json!("open_condition_wait")),
14475            (
14476                "condition_wait_occurrence_id".to_string(),
14477                json!(self.occurrence_id.as_deref().unwrap_or_default()),
14478            ),
14479            ("condition_key".to_string(), json!(options.condition_key)),
14480            (
14481                "condition_definition_fingerprint".to_string(),
14482                json!(options.predicate_identity),
14483            ),
14484        ]);
14485        if let Some(timeout_seconds) = options.timeout_seconds {
14486            command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
14487        }
14488        apply_parallel_group_path(&mut command, &self.parallel_group_path);
14489        ctx.apply_scope_membership(&mut command);
14490        state.commands.push(Value::Object(command));
14491        drop(state);
14492        self.opened_wait = true;
14493        Poll::Pending
14494    }
14495}
14496
14497fn validate_recorded_condition_wait(
14498    sequence: u64,
14499    recorded_occurrence_id: &str,
14500    recorded_key: Option<&str>,
14501    recorded_predicate_identity: &str,
14502    recorded_timeout_seconds: Option<u64>,
14503    current_occurrence_id: &str,
14504    current: &ValidatedConditionWaitOptions,
14505) -> Result<()> {
14506    if recorded_occurrence_id != current_occurrence_id {
14507        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
14508            "condition_wait_occurrence_mismatch",
14509            Some(sequence),
14510            Some(recorded_occurrence_id.to_string()),
14511            Some(current_occurrence_id.to_string()),
14512            "recorded condition occurrence differs from the current authored wait position",
14513        )));
14514    }
14515    if recorded_key != Some(current.condition_key.as_str()) {
14516        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
14517            "condition_wait_key_mismatch",
14518            Some(sequence),
14519            recorded_key.map(str::to_string),
14520            Some(current.condition_key.clone()),
14521            "recorded condition identity differs from the current workflow wait",
14522        )));
14523    }
14524    if recorded_predicate_identity != current.predicate_identity {
14525        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
14526            "condition_wait_predicate_mismatch",
14527            Some(sequence),
14528            Some(recorded_predicate_identity.to_string()),
14529            Some(current.predicate_identity.clone()),
14530            "recorded condition predicate behavior differs from current workflow code",
14531        )));
14532    }
14533    if recorded_timeout_seconds != current.timeout_seconds {
14534        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
14535            "condition_wait_timeout_mismatch",
14536            Some(sequence),
14537            recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
14538            current.timeout_seconds.map(|seconds| format!("{seconds}s")),
14539            "recorded condition timeout differs from the current workflow wait",
14540        )));
14541    }
14542    Ok(())
14543}
14544
14545/// Future returned by [`WorkflowContext::start_child_workflow`].
14546pub struct ChildWorkflowCall {
14547    ctx: WorkflowContext,
14548    workflow_type: String,
14549    options: ChildWorkflowOptions,
14550    args: Option<Result<AvroValue>>,
14551    scheduled: bool,
14552    matched_pending: bool,
14553    parallel_group_path: Vec<ParallelGroupMetadata>,
14554}
14555
14556impl ChildWorkflowCall {
14557    fn poll_avro_value(
14558        mut self: Pin<&mut Self>,
14559        _cx: &mut TaskContext<'_>,
14560    ) -> Poll<Result<ChildWorkflowAvroResult>> {
14561        if self.matched_pending {
14562            return Poll::Pending;
14563        }
14564
14565        let ctx = self.ctx.clone();
14566        let mut state = match ctx.state.lock() {
14567            Ok(state) => state,
14568            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14569        };
14570
14571        let cursor = state.command_cursor;
14572        if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) {
14573            return Poll::Ready(Err(error));
14574        }
14575        let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Child)
14576        {
14577            Ok(recorded) => recorded,
14578            Err(error) => return Poll::Ready(Err(error)),
14579        };
14580        if let Some(recorded) = recorded {
14581            let sequence = recorded.sequence();
14582            match recorded {
14583                RecordedCommand::ChildWorkflow {
14584                    workflow_type,
14585                    policies,
14586                    outcome,
14587                    parallel_group_path,
14588                    ..
14589                } => {
14590                    if let Err(error) = ensure_parallel_path_matches(
14591                        sequence,
14592                        parallel_group_path.as_deref(),
14593                        &self.parallel_group_path,
14594                    ) {
14595                        return Poll::Ready(Err(error));
14596                    }
14597                    if let Some(recorded_type) = workflow_type {
14598                        if recorded_type != self.workflow_type {
14599                            return Poll::Ready(Err(Error::NonDeterministicReplay(
14600                                ReplayFailure::new(
14601                                    "recorded_command_detail_mismatch",
14602                                    Some(sequence),
14603                                    Some(format!("child workflow:{recorded_type}")),
14604                                    Some(format!("child workflow:{}", self.workflow_type)),
14605                                    "recorded child workflow type differs from the current workflow command",
14606                                ),
14607                            )));
14608                        }
14609                    }
14610                    if let Err(error) =
14611                        ensure_child_policies_match(sequence, &policies, &self.options)
14612                    {
14613                        return Poll::Ready(Err(error));
14614                    }
14615                    if let Err(error) =
14616                        state.replay_cancellation_at(cursor, CancellationCallKind::Child)
14617                    {
14618                        return Poll::Ready(Err(error));
14619                    }
14620                    match state.prepare_scalar_cancellation(
14621                        cursor,
14622                        CancellationCallKind::Child,
14623                        &self.parallel_group_path,
14624                    ) {
14625                        Ok(true) => {
14626                            self.scheduled = true;
14627                            self.matched_pending = true;
14628                            return Poll::Pending;
14629                        }
14630                        Ok(false) => {}
14631                        Err(error) => return Poll::Ready(Err(error)),
14632                    }
14633                    state.command_cursor += 1;
14634                    if let Some(outcome) = outcome {
14635                        state.advance_cancellation_sequence(sequence, &self.parallel_group_path);
14636                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
14637                    }
14638                    state.matched_recorded_pending = true;
14639                    self.scheduled = true;
14640                    self.matched_pending = true;
14641                    return Poll::Pending;
14642                }
14643                other => {
14644                    return Poll::Ready(Err(command_mismatch(
14645                        &other,
14646                        format!("child workflow:{}", self.workflow_type),
14647                    )));
14648                }
14649            }
14650        }
14651
14652        if !self.scheduled {
14653            if self.options.task_queue.trim().is_empty() {
14654                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
14655                    "task_queue must not be empty".to_string(),
14656                )));
14657            }
14658            for (name, value) in [
14659                (
14660                    "execution_timeout_seconds",
14661                    self.options.execution_timeout_seconds,
14662                ),
14663                ("run_timeout_seconds", self.options.run_timeout_seconds),
14664            ] {
14665                if value == Some(0) {
14666                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
14667                        "{name} must be at least 1"
14668                    ))));
14669                }
14670            }
14671
14672            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
14673                Ok(args) => args,
14674                Err(error) => return Poll::Ready(Err(error)),
14675            };
14676            let arguments = match encode_typed_envelope(
14677                &normalize_avro_arguments(args),
14678                &state.payload_codec,
14679            ) {
14680                Ok(arguments) => arguments,
14681                Err(error) => return Poll::Ready(Err(error)),
14682            };
14683            let mut command = json!({
14684                "type": "start_child_workflow",
14685                "workflow_type": self.workflow_type,
14686                "queue": self.options.task_queue,
14687                "parent_close_policy": self.options.parent_close_policy.as_str(),
14688                "arguments": arguments,
14689            });
14690            let object = command
14691                .as_object_mut()
14692                .expect("child workflow command is always an object");
14693            if self.options.cancellation_policy != CancellationPolicy::Abandon {
14694                object.insert(
14695                    "cancellation_policy".to_string(),
14696                    json!(self.options.cancellation_policy.as_str()),
14697                );
14698            }
14699            if let Some(policy) = &self.options.retry_policy {
14700                let mut retry_policy = serde_json::Map::new();
14701                if let Some(max_attempts) = policy.max_attempts {
14702                    if max_attempts == 0 {
14703                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
14704                            "retry_policy.max_attempts must be at least 1".to_string(),
14705                        )));
14706                    }
14707                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
14708                }
14709                if !policy.backoff_seconds.is_empty() {
14710                    retry_policy
14711                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
14712                }
14713                if !policy.non_retryable_error_types.is_empty() {
14714                    retry_policy.insert(
14715                        "non_retryable_error_types".to_string(),
14716                        json!(policy.non_retryable_error_types),
14717                    );
14718                }
14719                if retry_policy.is_empty() {
14720                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
14721                        "retry_policy must configure at least one field".to_string(),
14722                    )));
14723                }
14724                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
14725            }
14726            if let Some(seconds) = self.options.execution_timeout_seconds {
14727                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
14728            }
14729            if let Some(seconds) = self.options.run_timeout_seconds {
14730                object.insert("run_timeout_seconds".to_string(), json!(seconds));
14731            }
14732            apply_parallel_group_path(object, &self.parallel_group_path);
14733            ctx.apply_scope_membership(object);
14734            match state.prepare_scalar_cancellation(
14735                cursor,
14736                CancellationCallKind::Child,
14737                &self.parallel_group_path,
14738            ) {
14739                Ok(true) => {
14740                    self.scheduled = true;
14741                    self.matched_pending = true;
14742                    return Poll::Pending;
14743                }
14744                Ok(false) => {}
14745                Err(error) => return Poll::Ready(Err(error)),
14746            }
14747            state.commands.push(command);
14748            self.scheduled = true;
14749        }
14750
14751        Poll::Pending
14752    }
14753}
14754
14755impl Future for ChildWorkflowCall {
14756    type Output = Result<ChildWorkflowResult>;
14757
14758    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
14759        match self.poll_avro_value(cx) {
14760            Poll::Ready(Ok(result)) => match result.result.into_json() {
14761                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
14762                    parent: result.parent,
14763                    child: result.child,
14764                    child_workflow_type: result.child_workflow_type,
14765                    result: projected,
14766                })),
14767                Err(error) => Poll::Ready(Err(error)),
14768            },
14769            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
14770            Poll::Pending => Poll::Pending,
14771        }
14772    }
14773}
14774
14775fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
14776    Error::NonDeterministicReplay(ReplayFailure::new(
14777        "recorded_command_mismatch",
14778        Some(recorded.sequence()),
14779        Some(recorded.shape().to_string()),
14780        Some(actual.into()),
14781        "current workflow command does not match the recorded durable command sequence",
14782    ))
14783}
14784
14785pub struct SignalCall {
14786    ctx: WorkflowContext,
14787    signal_name: String,
14788    runtime_reserved_allowed: bool,
14789    opened_wait: bool,
14790    matched_pending: bool,
14791    parallel_group_path: Vec<ParallelGroupMetadata>,
14792}
14793
14794impl SignalCall {
14795    fn poll_avro_value(
14796        mut self: Pin<&mut Self>,
14797        _cx: &mut TaskContext<'_>,
14798    ) -> Poll<Result<Vec<AvroValue>>> {
14799        if self.matched_pending {
14800            return Poll::Pending;
14801        }
14802        if !self.runtime_reserved_allowed {
14803            if let Err(error) = validate_user_signal_name(&self.signal_name) {
14804                return Poll::Ready(Err(error));
14805            }
14806        }
14807
14808        let ctx = self.ctx.clone();
14809        let mut state = match ctx.state.lock() {
14810            Ok(state) => state,
14811            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14812        };
14813
14814        let cursor = state.command_cursor;
14815        if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) {
14816            return Poll::Ready(Err(error));
14817        }
14818        let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Signal)
14819        {
14820            Ok(recorded) => recorded,
14821            Err(error) => return Poll::Ready(Err(error)),
14822        };
14823        if let Some(recorded) = recorded {
14824            match recorded {
14825                RecordedCommand::SignalWait {
14826                    sequence,
14827                    signal_name,
14828                    value,
14829                    parallel_group_path,
14830                } => {
14831                    if let Err(error) = ensure_parallel_path_matches(
14832                        sequence,
14833                        parallel_group_path.as_deref(),
14834                        &self.parallel_group_path,
14835                    ) {
14836                        return Poll::Ready(Err(error));
14837                    }
14838                    if signal_name != self.signal_name {
14839                        return Poll::Ready(Err(Error::NonDeterministicReplay(
14840                            ReplayFailure::new(
14841                                "recorded_command_detail_mismatch",
14842                                Some(sequence),
14843                                Some(format!("signal wait:{signal_name}")),
14844                                Some(format!("signal wait:{}", self.signal_name)),
14845                                "recorded signal name differs from the current workflow command",
14846                            ),
14847                        )));
14848                    }
14849
14850                    if let Err(error) =
14851                        state.replay_cancellation_at(cursor, CancellationCallKind::Signal)
14852                    {
14853                        return Poll::Ready(Err(error));
14854                    }
14855                    match state.prepare_scalar_cancellation(
14856                        cursor,
14857                        CancellationCallKind::Signal,
14858                        &self.parallel_group_path,
14859                    ) {
14860                        Ok(true) => {
14861                            self.opened_wait = true;
14862                            self.matched_pending = true;
14863                            return Poll::Pending;
14864                        }
14865                        Ok(false) => {}
14866                        Err(error) => return Poll::Ready(Err(error)),
14867                    }
14868                    state.command_cursor += 1;
14869                    if let Some(value) = value {
14870                        state.advance_cancellation_sequence(sequence, &self.parallel_group_path);
14871                        return Poll::Ready(Ok(value));
14872                    }
14873                    if state
14874                        .resume_signal
14875                        .as_ref()
14876                        .is_some_and(|signal| signal.signal_name == self.signal_name)
14877                    {
14878                        let signal = state
14879                            .resume_signal
14880                            .take()
14881                            .expect("matching resume signal is present");
14882                        state.unavailable_cancellation_boundary(&self.parallel_group_path);
14883                        return Poll::Ready(Ok(signal.arguments));
14884                    }
14885
14886                    state.matched_recorded_pending = true;
14887                    self.opened_wait = true;
14888                    self.matched_pending = true;
14889                    return Poll::Pending;
14890                }
14891                other => {
14892                    return Poll::Ready(Err(command_mismatch(
14893                        &other,
14894                        format!("signal wait:{}", self.signal_name),
14895                    )));
14896                }
14897            }
14898        }
14899
14900        match state.prepare_scalar_cancellation(
14901            cursor,
14902            CancellationCallKind::Signal,
14903            &self.parallel_group_path,
14904        ) {
14905            Ok(true) => {
14906                self.opened_wait = true;
14907                self.matched_pending = true;
14908                return Poll::Pending;
14909            }
14910            Ok(false) => {}
14911            Err(error) => return Poll::Ready(Err(error)),
14912        }
14913
14914        if state
14915            .resume_signal
14916            .as_ref()
14917            .is_some_and(|signal| signal.signal_name == self.signal_name)
14918        {
14919            let signal = state
14920                .resume_signal
14921                .take()
14922                .expect("matching resume signal is present");
14923            state.unavailable_cancellation_boundary(&self.parallel_group_path);
14924            return Poll::Ready(Ok(signal.arguments));
14925        }
14926
14927        if !self.opened_wait {
14928            let mut command = serde_json::Map::from_iter([
14929                ("type".to_string(), json!("open_signal_wait")),
14930                ("signal_name".to_string(), json!(self.signal_name)),
14931            ]);
14932            apply_parallel_group_path(&mut command, &self.parallel_group_path);
14933            ctx.apply_scope_membership(&mut command);
14934            state.commands.push(Value::Object(command));
14935            self.opened_wait = true;
14936        }
14937
14938        Poll::Pending
14939    }
14940}
14941
14942impl Future for SignalCall {
14943    type Output = Result<Vec<Value>>;
14944
14945    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
14946        match self.poll_avro_value(cx) {
14947            Poll::Ready(Ok(values)) => Poll::Ready(
14948                values
14949                    .into_iter()
14950                    .map(AvroValue::into_json)
14951                    .collect::<Result<Vec<_>>>(),
14952            ),
14953            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
14954            Poll::Pending => Poll::Pending,
14955        }
14956    }
14957}
14958
14959#[derive(Clone, Debug)]
14960pub struct ActivityContext {
14961    client: Client,
14962    pub task_id: String,
14963    pub activity_attempt_id: String,
14964    pub lease_owner: String,
14965    pub activity_type: String,
14966    pub attempt_number: u64,
14967    pub task_queue: String,
14968    pub worker_id: String,
14969    claim_guard: Option<cooperative_cancellation::ActivityClaimGuard>,
14970    local_heartbeats: Option<Arc<Mutex<local_activity::Heartbeats>>>,
14971    worker_session: Option<WorkerSession>,
14972}
14973
14974impl ActivityContext {
14975    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
14976        if let Some(heartbeats) = &self.local_heartbeats {
14977            return heartbeats
14978                .lock()
14979                .map_err(|_| Error::WorkflowStatePoisoned)?
14980                .record(details);
14981        }
14982        if let Some(guard) = &self.claim_guard {
14983            return guard.heartbeat(self, details).await;
14984        }
14985        if let Some(session) = &self.worker_session {
14986            if !session.active() {
14987                return Err(Error::ActivityExecutionAbandoned(
14988                    "worker-session lease or TTL is no longer active".into(),
14989                ));
14990            }
14991            let result = self
14992                .client
14993                .heartbeat_activity_task_value(
14994                    &self.task_id,
14995                    &self.activity_attempt_id,
14996                    &self.lease_owner,
14997                    details,
14998                )
14999                .await;
15000            return worker_session::settle_activity_heartbeat(
15001                session,
15002                result,
15003                &self.task_id,
15004                &self.activity_attempt_id,
15005                &self.lease_owner,
15006            );
15007        }
15008        self.client
15009            .heartbeat_activity_task(
15010                &self.task_id,
15011                &self.activity_attempt_id,
15012                &self.lease_owner,
15013                details,
15014            )
15015            .await
15016    }
15017}
15018
15019fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
15020    validate_payload_codec(codec)?;
15021    match value {
15022        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
15023            value, codec,
15024        )?)),
15025        None => Ok(AvroValue::Array(Vec::new())),
15026    }
15027}
15028
15029fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
15030    let Some(signal_name) = task
15031        .signal_name
15032        .as_deref()
15033        .filter(|value| !value.is_empty())
15034    else {
15035        return Ok(None);
15036    };
15037    let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
15038    let AvroValue::Array(arguments) = decoded else {
15039        unreachable!("normalize_avro_arguments always returns an array");
15040    };
15041
15042    Ok(Some(ResumeSignal {
15043        signal_name: signal_name.to_string(),
15044        arguments,
15045    }))
15046}
15047
15048fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
15049    validate_payload_codec(&task.payload_codec)?;
15050    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
15051    validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
15052    for event in &task.history_events {
15053        validate_history_event_payloads(event, &task.payload_codec)?;
15054    }
15055    Ok(())
15056}
15057
15058fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
15059    validate_payload_codec(&task.payload_codec)?;
15060    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
15061}
15062
15063fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
15064    validate_payload_codec(&task.payload_codec)?;
15065    validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
15066    validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
15067    for event in &task.history_events {
15068        validate_history_event_payloads(event, &task.payload_codec)?;
15069    }
15070
15071    let Some(export) = task.history_export.as_ref() else {
15072        return Ok(());
15073    };
15074    let export_codec = match export.get("payloads") {
15075        Some(payloads) => declared_payload_codec(payloads, "codec")?,
15076        None => None,
15077    }
15078    .unwrap_or(&task.payload_codec);
15079    validate_payload_codec(export_codec)?;
15080
15081    if let Some(events) = export.get("history_events").and_then(Value::as_array) {
15082        for event in events {
15083            let event_type = event
15084                .get("event_type")
15085                .or_else(|| event.get("type"))
15086                .and_then(Value::as_str)
15087                .unwrap_or_default();
15088            if let Some(payload) = event.get("payload") {
15089                validate_history_payloads(event_type, payload, export_codec)?;
15090            }
15091        }
15092    }
15093    for signal in export
15094        .get("signals")
15095        .and_then(Value::as_array)
15096        .into_iter()
15097        .flatten()
15098    {
15099        let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
15100        validate_payload_codec(codec)?;
15101        validate_optional_inbound_payload(signal.get("arguments"), codec)?;
15102    }
15103    for activity in export
15104        .get("activities")
15105        .and_then(Value::as_array)
15106        .into_iter()
15107        .flatten()
15108    {
15109        let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
15110        validate_payload_codec(codec)?;
15111        validate_optional_inbound_payload(activity.get("arguments"), codec)?;
15112        validate_optional_inbound_payload(activity.get("result"), codec)?;
15113    }
15114    Ok(())
15115}
15116
15117fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
15118    validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
15119}
15120
15121fn validate_history_payloads(
15122    event_type: &str,
15123    payload: &Value,
15124    fallback_codec: &str,
15125) -> Result<()> {
15126    let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
15127    validate_payload_codec(codec)?;
15128    for field in history_payload_fields(event_type) {
15129        validate_optional_inbound_payload(payload.get(*field), codec)?;
15130    }
15131    Ok(())
15132}
15133
15134const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
15135
15136fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
15137    match event_type {
15138        "ActivityCompleted" => &["result"],
15139        "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
15140        "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
15141        "UpdateCompleted" | "SideEffectRecorded" => &["result"],
15142        "ChildRunCompleted" => &["result", "output"],
15143        "WorkflowCompleted" => &["output"],
15144        "ServiceCallStarted"
15145        | "ServiceCallCompleted"
15146        | "ServiceCallFailed"
15147        | "ServiceCallCancelled" => &["request_payload", "response_payload"],
15148        _ => &[],
15149    }
15150}
15151
15152fn signal_history_payload(payload: &Value) -> Option<&Value> {
15153    SIGNAL_HISTORY_PAYLOAD_FIELDS
15154        .iter()
15155        .find_map(|field| payload.get(*field))
15156}
15157
15158fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
15159    match value.get(field) {
15160        None => Ok(None),
15161        Some(Value::String(codec)) => Ok(Some(codec)),
15162        Some(_) => Err(invalid_payload_envelope()),
15163    }
15164}
15165
15166fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
15167    validate_payload_codec(codec)?;
15168    if let Some(value) = value.filter(|value| !value.is_null()) {
15169        decode_wire_avro_value(value, codec)?;
15170    }
15171    Ok(())
15172}
15173
15174fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
15175    let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
15176        invalid_recorded_history(
15177            "parallel_group_metadata_invalid",
15178            sequence,
15179            "non-empty parallel_group_id",
15180            &payload.to_string(),
15181            "parallel-group history is missing its stable identity",
15182        )
15183    })?;
15184    let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
15185        invalid_recorded_history(
15186            "parallel_group_metadata_invalid",
15187            sequence,
15188            "activity, child, timer, signal, condition, or mixed group kind",
15189            &payload.to_string(),
15190            "parallel-group history is missing its group kind",
15191        )
15192    })?;
15193    if !matches!(
15194        kind.as_str(),
15195        "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
15196    ) {
15197        return Err(invalid_recorded_history(
15198            "parallel_group_metadata_invalid",
15199            sequence,
15200            "activity, child, timer, signal, condition, or mixed group kind",
15201            &kind,
15202            "parallel-group history contains an unsupported group kind",
15203        ));
15204    }
15205    let base_sequence = payload
15206        .get("parallel_group_base_sequence")
15207        .and_then(value_as_u64)
15208        .filter(|value| *value > 0)
15209        .ok_or_else(|| {
15210            invalid_recorded_history(
15211                "parallel_group_metadata_invalid",
15212                sequence,
15213                "positive parallel_group_base_sequence",
15214                &payload.to_string(),
15215                "parallel-group history contains an invalid base sequence",
15216            )
15217        })?;
15218    let size = payload
15219        .get("parallel_group_size")
15220        .and_then(value_as_u64)
15221        .and_then(|value| usize::try_from(value).ok())
15222        .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
15223        .ok_or_else(|| {
15224            invalid_recorded_history(
15225                "parallel_group_metadata_invalid",
15226                sequence,
15227                "bounded positive parallel_group_size",
15228                &payload.to_string(),
15229                "parallel-group history contains an invalid group size",
15230            )
15231        })?;
15232    let index = payload
15233        .get("parallel_group_index")
15234        .and_then(value_as_u64)
15235        .and_then(|value| usize::try_from(value).ok())
15236        .filter(|value| *value < size)
15237        .ok_or_else(|| {
15238            invalid_recorded_history(
15239                "parallel_group_metadata_invalid",
15240                sequence,
15241                "parallel_group_index within group bounds",
15242                &payload.to_string(),
15243                "parallel-group history contains an invalid member index",
15244            )
15245        })?;
15246    if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
15247        return Err(invalid_recorded_history(
15248            "parallel_group_metadata_invalid",
15249            sequence,
15250            "base sequence plus member index equals workflow sequence",
15251            &payload.to_string(),
15252            "parallel-group path does not preserve durable workflow position",
15253        ));
15254    }
15255    let mode = payload
15256        .get("parallel_group_mode")
15257        .and_then(Value::as_str)
15258        .unwrap_or("all");
15259    if !matches!(mode, "all" | "select") {
15260        return Err(invalid_recorded_history(
15261            "parallel_group_metadata_invalid",
15262            sequence,
15263            "parallel group mode all or select",
15264            mode,
15265            "parallel-group history contains an unsupported group mode",
15266        ));
15267    }
15268    let expected_id = if mode == "select" {
15269        format!("select-calls:{base_sequence}:{size}")
15270    } else {
15271        format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
15272    };
15273    if group_id != expected_id {
15274        return Err(invalid_recorded_history(
15275            "parallel_group_metadata_invalid",
15276            sequence,
15277            &expected_id,
15278            &group_id,
15279            "parallel-group history contains an incompatible stable group ID",
15280        ));
15281    }
15282    let selection_member_key = if mode == "select" {
15283        Some(selection_key_from_value(
15284            payload.get("selection_member_key"),
15285            sequence,
15286        )?)
15287    } else {
15288        None
15289    };
15290    let selection_member_index = if mode == "select" {
15291        Some(required_parallel_usize(
15292            payload,
15293            "selection_member_index",
15294            sequence,
15295        )?)
15296    } else {
15297        None
15298    };
15299    let selection_member_base_sequence = if mode == "select" {
15300        Some(
15301            payload
15302                .get("selection_member_base_sequence")
15303                .and_then(value_as_u64)
15304                .filter(|value| *value >= base_sequence)
15305                .ok_or_else(|| {
15306                    invalid_recorded_history(
15307                        "parallel_group_metadata_invalid",
15308                        sequence,
15309                        "selection member base within its group",
15310                        &payload.to_string(),
15311                        "selection history contains an invalid member base sequence",
15312                    )
15313                })?,
15314        )
15315    } else {
15316        None
15317    };
15318    let selection_member_size = if mode == "select" {
15319        let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
15320        if member_size == 0 {
15321            return Err(invalid_recorded_history(
15322                "parallel_group_metadata_invalid",
15323                sequence,
15324                "positive selection member size",
15325                &payload.to_string(),
15326                "selection history contains an invalid member size",
15327            ));
15328        }
15329        Some(member_size)
15330    } else {
15331        None
15332    };
15333    let selection_member_kind = if mode == "select" {
15334        let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
15335            invalid_recorded_history(
15336                "parallel_group_metadata_invalid",
15337                sequence,
15338                "selection member operation kind",
15339                &payload.to_string(),
15340                "selection history is missing its authored member kind",
15341            )
15342        })?;
15343        if !matches!(
15344            kind.as_str(),
15345            "activity" | "child" | "timer" | "signal" | "condition" | "group"
15346        ) {
15347            return Err(invalid_recorded_history(
15348                "parallel_group_metadata_invalid",
15349                sequence,
15350                "activity, child, timer, signal, condition, or group selection member kind",
15351                &kind,
15352                "selection history contains an unsupported member kind",
15353            ));
15354        }
15355        Some(kind)
15356    } else {
15357        None
15358    };
15359    if let (Some(member_base), Some(member_size)) =
15360        (selection_member_base_sequence, selection_member_size)
15361    {
15362        let member_end = member_base
15363            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
15364            .ok_or_else(|| {
15365                invalid_recorded_history(
15366                    "parallel_group_metadata_invalid",
15367                    sequence,
15368                    "bounded selection member range",
15369                    &payload.to_string(),
15370                    "selection member range overflowed",
15371                )
15372            })?;
15373        let group_end = base_sequence
15374            .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
15375            .unwrap_or(u64::MAX);
15376        if sequence < member_base || sequence >= member_end || member_end > group_end {
15377            return Err(invalid_recorded_history(
15378                "parallel_group_metadata_invalid",
15379                sequence,
15380                "workflow sequence within one bounded selection member",
15381                &payload.to_string(),
15382                "selection member range does not contain its durable leaf",
15383            ));
15384        }
15385    }
15386    Ok(ParallelGroupMetadata {
15387        parallel_group_id: group_id,
15388        parallel_group_kind: kind,
15389        parallel_group_base_sequence: base_sequence,
15390        parallel_group_size: size,
15391        parallel_group_index: index,
15392        parallel_group_mode: (mode == "select").then(|| "select".to_string()),
15393        selection_member_key,
15394        selection_member_index,
15395        selection_member_base_sequence,
15396        selection_member_size,
15397        selection_member_kind,
15398    })
15399}
15400
15401fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
15402    payload
15403        .get(field)
15404        .and_then(value_as_u64)
15405        .and_then(|value| usize::try_from(value).ok())
15406        .ok_or_else(|| {
15407            invalid_recorded_history(
15408                "parallel_group_metadata_invalid",
15409                sequence,
15410                &format!("non-negative integer {field}"),
15411                &payload.to_string(),
15412                "selection history contains invalid member metadata",
15413            )
15414        })
15415}
15416
15417fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
15418    match value {
15419        Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
15420        Some(value) => value_as_u64(value)
15421            .and_then(|value| usize::try_from(value).ok())
15422            .map(SelectionKey::Index)
15423            .ok_or_else(|| {
15424                invalid_recorded_history(
15425                    "selection_member_key_invalid",
15426                    sequence,
15427                    "non-empty string or non-negative integer member key",
15428                    &value.to_string(),
15429                    "selection history contains an invalid member key",
15430                )
15431            }),
15432        None => Err(invalid_recorded_history(
15433            "selection_member_key_missing",
15434            sequence,
15435            "selection_member_key",
15436            "<missing>",
15437            "selection history is missing its stable member key",
15438        )),
15439    }
15440}
15441
15442fn recorded_parallel_group_path(
15443    events: &[&HistoryEvent],
15444    sequence: u64,
15445) -> Result<Option<Vec<ParallelGroupMetadata>>> {
15446    let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
15447    for event in events {
15448        let payload = &event.payload;
15449        let has_metadata = payload.get("parallel_group_path").is_some()
15450            || payload.get("parallel_group_id").is_some()
15451            || payload.get("parallel_group_kind").is_some()
15452            || payload.get("parallel_group_base_sequence").is_some()
15453            || payload.get("parallel_group_size").is_some()
15454            || payload.get("parallel_group_index").is_some()
15455            || payload.get("parallel_group_mode").is_some()
15456            || payload.get("selection_member_key").is_some();
15457        if !has_metadata {
15458            continue;
15459        }
15460
15461        let top_level = recorded_parallel_group_entry(payload, sequence)?;
15462        let path = match payload.get("parallel_group_path") {
15463            None => vec![top_level.clone()],
15464            Some(Value::Array(entries)) if !entries.is_empty() => entries
15465                .iter()
15466                .map(|entry| recorded_parallel_group_entry(entry, sequence))
15467                .collect::<Result<Vec<_>>>()?,
15468            Some(value) => {
15469                return Err(invalid_recorded_history(
15470                    "parallel_group_metadata_invalid",
15471                    sequence,
15472                    "non-empty parallel_group_path list",
15473                    &value.to_string(),
15474                    "parallel-group history contains an invalid group path",
15475                ));
15476            }
15477        };
15478        if path.last() != Some(&top_level) {
15479            return Err(invalid_recorded_history(
15480                "parallel_group_metadata_invalid",
15481                sequence,
15482                &serde_json::to_string(&path.last()).unwrap_or_default(),
15483                &serde_json::to_string(&top_level).unwrap_or_default(),
15484                "parallel-group top-level fields do not match the innermost path entry",
15485            ));
15486        }
15487        if recorded.as_ref().is_some_and(|existing| existing != &path) {
15488            return Err(invalid_recorded_history(
15489                "parallel_group_history_conflict",
15490                sequence,
15491                &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
15492                &serde_json::to_string(&path).unwrap_or_default(),
15493                "parallel-group metadata changed between scheduling and resolution history",
15494            ));
15495        }
15496        recorded = Some(path);
15497    }
15498    Ok(recorded)
15499}
15500
15501fn recorded_commands(
15502    events: &[HistoryEvent],
15503    fallback_codec: &str,
15504    parent: WorkflowIdentity,
15505) -> Result<Vec<RecordedCommand>> {
15506    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
15507    let mut last_new_sequence = None;
15508
15509    for event in events {
15510        let is_activity = matches!(
15511            event.event_type.as_str(),
15512            "ActivityScheduled"
15513                | "ActivityStarted"
15514                | "ActivityHeartbeatRecorded"
15515                | "ActivityRetryScheduled"
15516                | "ActivityCompleted"
15517                | "ActivityFailed"
15518                | "ActivityCancelled"
15519                | "ActivityTimedOut"
15520        );
15521        let is_workflow_timer = matches!(
15522            event.event_type.as_str(),
15523            "TimerScheduled" | "TimerCancelled" | "TimerFired"
15524        ) && !is_internal_timer_event(event);
15525        let is_child_workflow = matches!(
15526            event.event_type.as_str(),
15527            "ChildWorkflowScheduled"
15528                | "ChildRunStarted"
15529                | "ChildRunCompleted"
15530                | "ChildRunFailed"
15531                | "ChildRunCancelled"
15532                | "ChildRunTerminated"
15533        );
15534        let is_signal_wait = is_recorded_signal_wait_event(event);
15535        let is_condition_wait = is_recorded_condition_wait_event(event);
15536        let is_search_attributes = event.event_type == "SearchAttributesUpserted";
15537        let is_side_effect = event.event_type == "SideEffectRecorded";
15538        let is_version_marker = event.event_type == "VersionMarkerRecorded";
15539        let is_memo = event.event_type == "MemoUpserted";
15540        if !is_activity
15541            && !is_workflow_timer
15542            && !is_child_workflow
15543            && !is_signal_wait
15544            && !is_condition_wait
15545            && !is_search_attributes
15546            && !is_side_effect
15547            && !is_version_marker
15548            && !is_memo
15549        {
15550            continue;
15551        }
15552
15553        let sequence = durable_event_sequence(event).ok_or_else(|| {
15554            Error::NonDeterministicReplay(ReplayFailure::new(
15555                "durable_command_sequence_missing",
15556                None,
15557                Some("positive workflow sequence".to_string()),
15558                Some(event.event_type.clone()),
15559                "durable command history event has no workflow sequence",
15560            ))
15561        })?;
15562        if sequence == 0 {
15563            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
15564                "durable_command_sequence_invalid",
15565                Some(sequence),
15566                Some("positive workflow sequence".to_string()),
15567                Some(sequence.to_string()),
15568                "durable command history uses an invalid workflow sequence",
15569            )));
15570        }
15571        if !events_by_sequence.contains_key(&sequence) {
15572            if let Some(previous) = last_new_sequence {
15573                if sequence < previous {
15574                    return Err(invalid_recorded_history(
15575                        "durable_command_sequence_mismatch",
15576                        sequence,
15577                        &format!("workflow sequence greater than {previous}"),
15578                        &sequence.to_string(),
15579                        "durable commands are not strictly ordered by their recorded workflow sequence",
15580                    ));
15581                }
15582            }
15583            last_new_sequence = Some(sequence);
15584        }
15585        events_by_sequence.entry(sequence).or_default().push(event);
15586    }
15587
15588    let commands: Vec<RecordedCommand> = events_by_sequence
15589        .into_iter()
15590        .map(|(sequence, sequence_events)| {
15591            let activity_events: Vec<_> = sequence_events
15592                .iter()
15593                .copied()
15594                .filter(|event| event.event_type.starts_with("Activity"))
15595                .collect();
15596            let timer_events: Vec<_> = sequence_events
15597                .iter()
15598                .copied()
15599                .filter(|event| event.event_type.starts_with("Timer"))
15600                .collect();
15601            let child_events: Vec<_> = sequence_events
15602                .iter()
15603                .copied()
15604                .filter(|event| {
15605                    event.event_type == "ChildWorkflowScheduled"
15606                        || event.event_type.starts_with("ChildRun")
15607                })
15608                .collect();
15609            let signal_wait_events: Vec<_> = sequence_events
15610                .iter()
15611                .copied()
15612                .filter(|event| is_recorded_signal_wait_event(event))
15613                .collect();
15614            let condition_wait_events: Vec<_> = sequence_events
15615                .iter()
15616                .copied()
15617                .filter(|event| is_recorded_condition_wait_event(event))
15618                .collect();
15619            let search_attribute_events: Vec<_> = sequence_events
15620                .iter()
15621                .copied()
15622                .filter(|event| event.event_type == "SearchAttributesUpserted")
15623                .collect();
15624            let side_effect_events: Vec<_> = sequence_events
15625                .iter()
15626                .copied()
15627                .filter(|event| event.event_type == "SideEffectRecorded")
15628                .collect();
15629            let version_marker_events: Vec<_> = sequence_events
15630                .iter()
15631                .copied()
15632                .filter(|event| event.event_type == "VersionMarkerRecorded")
15633                .collect();
15634            let memo_events: Vec<_> = sequence_events
15635                .iter()
15636                .copied()
15637                .filter(|event| event.event_type == "MemoUpserted")
15638                .collect();
15639
15640            let command_kind_count = usize::from(!activity_events.is_empty())
15641                + usize::from(!timer_events.is_empty())
15642                + usize::from(!child_events.is_empty())
15643                + usize::from(!signal_wait_events.is_empty())
15644                + usize::from(!condition_wait_events.is_empty())
15645                + usize::from(!search_attribute_events.is_empty())
15646                + usize::from(!side_effect_events.is_empty())
15647                + usize::from(!version_marker_events.is_empty())
15648                + usize::from(!memo_events.is_empty());
15649            if command_kind_count > 1 {
15650                let actual = [
15651                    (!activity_events.is_empty()).then_some("activity"),
15652                    (!timer_events.is_empty()).then_some("timer"),
15653                    (!child_events.is_empty()).then_some("child workflow"),
15654                    (!signal_wait_events.is_empty()).then_some("signal wait"),
15655                    (!condition_wait_events.is_empty()).then_some("condition wait"),
15656                    (!search_attribute_events.is_empty()).then_some("search-attribute update"),
15657                    (!side_effect_events.is_empty()).then_some("side effect"),
15658                    (!version_marker_events.is_empty()).then_some("version marker"),
15659                    (!memo_events.is_empty()).then_some("memo upsert"),
15660                ]
15661                .into_iter()
15662                .flatten()
15663                .collect::<Vec<_>>()
15664                .join(" and ");
15665                return Err(invalid_recorded_history(
15666                    "durable_command_sequence_collision",
15667                    sequence,
15668                    "one durable command kind",
15669                    &actual,
15670                    "one workflow sequence records more than one durable command kind",
15671                ));
15672            }
15673
15674            if !activity_events.is_empty() {
15675                let parallel_group_path =
15676                    recorded_parallel_group_path(&activity_events, sequence)?;
15677                let scheduled_count = activity_events
15678                    .iter()
15679                    .filter(|event| event.event_type == "ActivityScheduled")
15680                    .count();
15681                if scheduled_count > 1 {
15682                    return Err(invalid_recorded_history(
15683                        "duplicate_activity_schedule",
15684                        sequence,
15685                        "at most one ActivityScheduled event",
15686                        "multiple ActivityScheduled events",
15687                        "activity history schedules more than one command at one workflow sequence",
15688                    ));
15689                }
15690                let activity_type = activity_events.iter().find_map(|event| {
15691                    event
15692                        .payload
15693                        .get("activity_type")
15694                        .or_else(|| event.payload.get("activity_name"))
15695                        .and_then(Value::as_str)
15696                        .map(str::to_string)
15697                });
15698                if activity_events.iter().filter_map(|event| {
15699                    event
15700                        .payload
15701                        .get("activity_type")
15702                        .or_else(|| event.payload.get("activity_name"))
15703                        .and_then(Value::as_str)
15704                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
15705                    return Err(invalid_recorded_history(
15706                        "activity_identity_mismatch",
15707                        sequence,
15708                        activity_type.as_deref().unwrap_or("one activity identity"),
15709                        "conflicting activity identities",
15710                        "activity lifecycle events at one workflow sequence disagree on identity",
15711                    ));
15712                }
15713                let terminal: Vec<_> = activity_events
15714                    .iter()
15715                    .copied()
15716                    .filter(|event| {
15717                        matches!(
15718                            event.event_type.as_str(),
15719                            "ActivityCompleted"
15720                                | "ActivityFailed"
15721                                | "ActivityCancelled"
15722                                | "ActivityTimedOut"
15723                        )
15724                    })
15725                    .collect();
15726                let duplicate_delivery = terminal.first().is_some_and(|first| {
15727                    terminal.iter().all(|event| {
15728                        event.event_type == first.event_type && event.payload == first.payload
15729                    })
15730                });
15731                if terminal.len() > 1 && !duplicate_delivery {
15732                    return Err(invalid_recorded_history(
15733                        "duplicate_activity_terminal_event",
15734                        sequence,
15735                        "at most one terminal activity event",
15736                        "multiple terminal activity events",
15737                        "activity history settles one command more than once",
15738                    ));
15739                }
15740                let outcome = terminal
15741                    .first()
15742                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
15743                    .transpose()?;
15744                let execution_mode = recorded_activity_execution_mode(&activity_events, sequence)?;
15745                let activity_snapshot = activity_events
15746                    .iter()
15747                    .find(|event| event.event_type == "ActivityScheduled")
15748                    .and_then(|event| event.payload.get("activity"))
15749                    .and_then(Value::as_object);
15750                let options = Some(match activity_snapshot {
15751                    Some(activity) => RecordedActivityOptions {
15752                        task_queue: recorded_optional_string(activity, "queue"),
15753                        execution_mode,
15754                        retry_policy: recorded_activity_retry_snapshot(
15755                            activity.get("retry_policy"),
15756                        ),
15757                        worker_session: worker_session::recorded_session(&activity_events, sequence)?,
15758                    },
15759                    None => RecordedActivityOptions {
15760                        task_queue: RecordedSnapshotValue::Unknown,
15761                        execution_mode,
15762                        retry_policy: recorded_activity_retry_snapshot(None),
15763                        worker_session: worker_session::recorded_session(&activity_events, sequence)?,
15764                    },
15765                });
15766                return Ok(RecordedCommand::Activity {
15767                    sequence,
15768                    activity_type,
15769                    cancellation_policy: recorded_activity_cancellation_policy(&activity_events, sequence)?,
15770                    options,
15771                    outcome,
15772                    parallel_group_path,
15773                });
15774            }
15775
15776            if !child_events.is_empty() {
15777                let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
15778                let scheduled: Vec<_> = child_events
15779                    .iter()
15780                    .copied()
15781                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
15782                    .collect();
15783                if scheduled.len() != 1 {
15784                    return Err(invalid_recorded_history(
15785                        "child_workflow_schedule_missing_or_duplicate",
15786                        sequence,
15787                        "one ChildWorkflowScheduled event",
15788                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
15789                        "child workflow replay requires exactly one recorded schedule event",
15790                    ));
15791                }
15792                let policies = recorded_child_policies(&child_events, scheduled[0], sequence)?;
15793                let workflow_type = child_events.iter().find_map(|event| {
15794                    event
15795                        .payload
15796                        .get("child_workflow_type")
15797                        .or_else(|| event.payload.get("workflow_type"))
15798                        .and_then(Value::as_str)
15799                        .filter(|value| !value.is_empty())
15800                        .map(str::to_string)
15801                });
15802                if child_events
15803                    .iter()
15804                    .filter_map(|event| {
15805                        event
15806                            .payload
15807                            .get("child_workflow_type")
15808                            .or_else(|| event.payload.get("workflow_type"))
15809                            .and_then(Value::as_str)
15810                    })
15811                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
15812                {
15813                    return Err(invalid_recorded_history(
15814                        "child_workflow_identity_mismatch",
15815                        sequence,
15816                        workflow_type
15817                            .as_deref()
15818                            .unwrap_or("one child workflow type"),
15819                        "conflicting child workflow types",
15820                        "child workflow lifecycle events at one sequence disagree on type",
15821                    ));
15822                }
15823                let mut outcomes = child_workflow_outcomes(
15824                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
15825                    fallback_codec,
15826                    parent.clone(),
15827                )?;
15828                let terminal_events = child_events
15829                    .iter()
15830                    .copied()
15831                    .filter(|event| event.event_type.starts_with("ChildRun"))
15832                    .collect::<Vec<_>>();
15833                let duplicate_delivery = terminal_events.first().is_some_and(|first| {
15834                    terminal_events.iter().all(|event| {
15835                        event.event_type == first.event_type && event.payload == first.payload
15836                    })
15837                });
15838                if outcomes.len() > 1 && !duplicate_delivery {
15839                    return Err(invalid_recorded_history(
15840                        "duplicate_child_workflow_terminal_event",
15841                        sequence,
15842                        "at most one terminal child event",
15843                        "multiple terminal child events",
15844                        "child workflow history settles one command more than once",
15845                    ));
15846                }
15847                return Ok(RecordedCommand::ChildWorkflow {
15848                    sequence,
15849                    workflow_type,
15850                    policies,
15851                    outcome: outcomes.pop(),
15852                    parallel_group_path,
15853                });
15854            }
15855
15856            if !signal_wait_events.is_empty() {
15857                let opened: Vec<_> = signal_wait_events
15858                    .iter()
15859                    .copied()
15860                    .filter(|event| event.event_type == "SignalWaitOpened")
15861                    .collect();
15862                if opened.len() != 1 {
15863                    return Err(invalid_recorded_history(
15864                        "signal_wait_open_missing_or_duplicate",
15865                        sequence,
15866                        "one SignalWaitOpened event",
15867                        &format!("{} SignalWaitOpened events", opened.len()),
15868                        "signal replay requires exactly one canonical wait-open event",
15869                    ));
15870                }
15871
15872                let applied: Vec<_> = signal_wait_events
15873                    .iter()
15874                    .copied()
15875                    .filter(|event| event.event_type == "SignalApplied")
15876                    .collect();
15877                if applied.len() > 1 {
15878                    return Err(invalid_recorded_history(
15879                        "duplicate_signal_wait_apply",
15880                        sequence,
15881                        "at most one SignalApplied event",
15882                        "multiple SignalApplied events",
15883                        "signal history applies one durable wait more than once",
15884                    ));
15885                }
15886
15887                let signal_names = signal_wait_events
15888                    .iter()
15889                    .map(|event| required_signal_wait_name(event, sequence))
15890                    .collect::<Result<Vec<_>>>()?;
15891                let signal_name = signal_names
15892                    .first()
15893                    .expect("signal wait events are not empty")
15894                    .clone();
15895                if signal_names.iter().any(|candidate| candidate != &signal_name) {
15896                    return Err(invalid_recorded_history(
15897                        "signal_wait_identity_mismatch",
15898                        sequence,
15899                        &signal_name,
15900                        "conflicting signal names",
15901                        "signal wait lifecycle events at one workflow sequence disagree on identity",
15902                    ));
15903                }
15904                let value = applied
15905                    .first()
15906                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
15907                    .transpose()?;
15908                return Ok(RecordedCommand::SignalWait {
15909                    sequence,
15910                    signal_name,
15911                    value,
15912                    parallel_group_path: recorded_parallel_group_path(
15913                        &signal_wait_events,
15914                        sequence,
15915                    )?,
15916                });
15917            }
15918
15919            if !condition_wait_events.is_empty() {
15920                return recorded_condition_wait(
15921                    sequence,
15922                    &condition_wait_events,
15923                    events,
15924                );
15925            }
15926
15927            if !search_attribute_events.is_empty() {
15928                if search_attribute_events.len() != 1 {
15929                    return Err(invalid_recorded_history(
15930                        "duplicate_search_attribute_update",
15931                        sequence,
15932                        "one SearchAttributesUpserted event",
15933                        &format!(
15934                            "{} SearchAttributesUpserted events",
15935                            search_attribute_events.len()
15936                        ),
15937                        "search-attribute history records one workflow command more than once",
15938                    ));
15939                }
15940                let payload = &search_attribute_events[0].payload;
15941                let attributes = payload
15942                    .get("attributes")
15943                    .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
15944                    .cloned()
15945                    .ok_or_else(|| {
15946                        invalid_recorded_history(
15947                            "search_attribute_update_missing",
15948                            sequence,
15949                            "non-empty attributes object",
15950                            "missing or invalid attributes",
15951                            "search-attribute history is missing its recorded mutation",
15952                        )
15953                    })?;
15954                let attribute_types =
15955                    recorded_search_attribute_types(payload, &attributes, sequence)?;
15956                return Ok(RecordedCommand::SearchAttributes {
15957                    sequence,
15958                    attributes,
15959                    attribute_types,
15960                });
15961            }
15962
15963            if !side_effect_events.is_empty() {
15964                if side_effect_events.len() != 1 {
15965                    return Err(invalid_recorded_history(
15966                        "duplicate_side_effect_record",
15967                        sequence,
15968                        "one SideEffectRecorded event",
15969                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
15970                        "side-effect history records one workflow command more than once",
15971                    ));
15972                }
15973                let event = side_effect_events[0];
15974                let result = event.payload.get("result").ok_or_else(|| {
15975                    invalid_recorded_history(
15976                        "side_effect_result_missing",
15977                        sequence,
15978                        "recorded result payload",
15979                        "missing result",
15980                        "side-effect history is missing its recorded value",
15981                    )
15982                })?;
15983                let has_published_envelope = result.as_str().is_some()
15984                    || result.as_object().is_some_and(|envelope| {
15985                        envelope.get("codec").and_then(Value::as_str).is_some()
15986                            && envelope.get("blob").and_then(Value::as_str).is_some()
15987                    });
15988                if !has_published_envelope {
15989                    return Err(invalid_recorded_history(
15990                        "side_effect_payload_malformed",
15991                        sequence,
15992                        "payload blob or {codec, blob} envelope",
15993                        &result.to_string(),
15994                        "side-effect history result does not use a published payload envelope",
15995                    ));
15996                }
15997                let codec = event
15998                    .payload
15999                    .get("payload_codec")
16000                    .and_then(Value::as_str)
16001                    .unwrap_or(fallback_codec);
16002                let value = decode_wire_avro_value(result, codec).map_err(|error| {
16003                    if error.to_string().contains("unsupported_payload_codec") {
16004                        return error;
16005                    }
16006
16007                    invalid_recorded_history(
16008                        "side_effect_payload_incompatible",
16009                        sequence,
16010                        &format!("valid {codec} payload envelope"),
16011                        &error.to_string(),
16012                        "side-effect history payload cannot be decoded with its recorded codec",
16013                    )
16014                })?;
16015                return Ok(RecordedCommand::SideEffect { sequence, value });
16016            }
16017
16018            if !version_marker_events.is_empty() {
16019                if version_marker_events.len() != 1 {
16020                    return Err(invalid_recorded_history(
16021                        "duplicate_version_marker_record",
16022                        sequence,
16023                        "one VersionMarkerRecorded event",
16024                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
16025                        "version-marker history records one workflow command more than once",
16026                    ));
16027                }
16028                let payload = &version_marker_events[0].payload;
16029                let change_id = payload
16030                    .get("change_id")
16031                    .and_then(Value::as_str)
16032                    .filter(|value| !value.is_empty())
16033                    .map(str::to_string)
16034                    .ok_or_else(|| {
16035                        invalid_recorded_history(
16036                            "version_marker_field_missing",
16037                            sequence,
16038                            "non-empty change_id",
16039                            "missing or invalid change_id",
16040                            "version-marker history is missing its stable change ID",
16041                        )
16042                    })?;
16043                let version = required_version_i32(payload, "version", sequence)?;
16044                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
16045                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
16046                if min_supported > max_supported || version < min_supported || version > max_supported {
16047                    return Err(invalid_recorded_history(
16048                        "version_marker_history_range_invalid",
16049                        sequence,
16050                        "min_supported <= version <= max_supported",
16051                        &format!("{min_supported} <= {version} <= {max_supported}"),
16052                        "recorded version marker contains an internally incompatible range",
16053                    ));
16054                }
16055                return Ok(RecordedCommand::VersionMarker {
16056                    sequence,
16057                    change_id,
16058                    version,
16059                });
16060            }
16061
16062            if !memo_events.is_empty() {
16063                if memo_events.len() != 1 {
16064                    return Err(invalid_recorded_history(
16065                        "duplicate_memo_upsert_record",
16066                        sequence,
16067                        "one MemoUpserted event",
16068                        &format!("{} MemoUpserted events", memo_events.len()),
16069                        "memo history records one workflow update more than once",
16070                    ));
16071                }
16072                let payload = &memo_events[0].payload;
16073                let entries = payload.get("entries").cloned().ok_or_else(|| {
16074                    invalid_recorded_history(
16075                        "memo_entries_missing",
16076                        sequence,
16077                        "memo entries object",
16078                        "missing entries",
16079                        "MemoUpserted history is missing replay identity entries",
16080                    )
16081                })?;
16082                let entries = decode_memo_history_map(&entries, true).map_err(|error| {
16083                    invalid_recorded_history(
16084                        "memo_entries_invalid",
16085                        sequence,
16086                        "valid canonical memo entries",
16087                        &error.to_string(),
16088                        "MemoUpserted history contains invalid replay identity entries",
16089                    )
16090                })?;
16091                let merged = payload.get("merged").cloned().ok_or_else(|| {
16092                    invalid_recorded_history(
16093                        "memo_merged_projection_missing",
16094                        sequence,
16095                        "merged memo projection",
16096                        "missing merged",
16097                        "MemoUpserted history is missing its merged projection",
16098                    )
16099                })?;
16100                decode_memo_history_map(&merged, false).map_err(|error| {
16101                    invalid_recorded_history(
16102                        "memo_merged_projection_invalid",
16103                        sequence,
16104                        "valid merged memo projection",
16105                        &error.to_string(),
16106                        "MemoUpserted history contains an invalid merged projection",
16107                    )
16108                })?;
16109
16110                return Ok(RecordedCommand::Memo { sequence, entries });
16111            }
16112            let scheduled: Vec<_> = timer_events
16113                .iter()
16114                .copied()
16115                .filter(|event| event.event_type == "TimerScheduled")
16116                .collect();
16117            let fired: Vec<_> = timer_events
16118                .iter()
16119                .copied()
16120                .filter(|event| event.event_type == "TimerFired")
16121                .collect();
16122            if scheduled.len() != 1 {
16123                return Err(invalid_recorded_history(
16124                    "timer_schedule_missing_or_duplicate",
16125                    sequence,
16126                    "one TimerScheduled event",
16127                    &format!("{} TimerScheduled events", scheduled.len()),
16128                    "timer replay requires exactly one recorded schedule event",
16129                ));
16130            }
16131            if fired.len() > 1 {
16132                return Err(invalid_recorded_history(
16133                    "duplicate_timer_fire",
16134                    sequence,
16135                    "at most one TimerFired event",
16136                    "multiple TimerFired events",
16137                    "timer history contains more than one fire event for a workflow sequence",
16138                ));
16139            }
16140
16141            let scheduled = scheduled[0];
16142            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
16143            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
16144            if let Some(fired) = fired.first() {
16145                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
16146                if fired_timer_id != timer_id {
16147                    return Err(invalid_recorded_history(
16148                        "timer_identity_mismatch",
16149                        sequence,
16150                        &timer_id,
16151                        &fired_timer_id,
16152                        "TimerFired does not correspond to the recorded TimerScheduled event",
16153                    ));
16154                }
16155                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
16156                if fired_delay != delay_seconds {
16157                    return Err(invalid_recorded_history(
16158                        "timer_history_delay_mismatch",
16159                        sequence,
16160                        &delay_seconds.to_string(),
16161                        &fired_delay.to_string(),
16162                        "TimerScheduled and TimerFired record different delays",
16163                    ));
16164                }
16165            }
16166
16167            Ok(RecordedCommand::Timer {
16168                sequence,
16169                delay_seconds,
16170                fired: !fired.is_empty(),
16171                parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
16172            })
16173        })
16174        .collect::<Result<_>>()?;
16175
16176    let mut marker_decisions = HashMap::new();
16177    for command in &commands {
16178        if let RecordedCommand::VersionMarker {
16179            sequence,
16180            change_id,
16181            version,
16182        } = command
16183        {
16184            if let Some((first_version, first_sequence)) = marker_decisions.get(change_id) {
16185                if version != first_version {
16186                    return Err(invalid_recorded_history(
16187                        "version_marker_conflict",
16188                        *sequence,
16189                        &format!("version {first_version} for change ID {change_id:?} at sequence {first_sequence}"),
16190                        &format!("version {version} at sequence {sequence}"),
16191                        "workflow history contains conflicting decisions for one stable change ID",
16192                    ));
16193                }
16194            } else {
16195                marker_decisions.insert(change_id.clone(), (*version, *sequence));
16196            }
16197        }
16198    }
16199
16200    Ok(commands)
16201}
16202
16203fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
16204    payload
16205        .get(field)
16206        .and_then(Value::as_i64)
16207        .and_then(|value| i32::try_from(value).ok())
16208        .ok_or_else(|| {
16209            invalid_recorded_history(
16210                "version_marker_field_missing",
16211                sequence,
16212                &format!("integer {field}"),
16213                "missing or out-of-range integer",
16214                "version-marker history is missing a required integer field",
16215            )
16216        })
16217}
16218
16219fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
16220    event
16221        .payload
16222        .get("sequence")
16223        .or_else(|| event.payload.get("workflow_sequence"))
16224        .or_else(|| event.raw.get("sequence"))
16225        .or_else(|| event.raw.get("workflow_sequence"))
16226        .and_then(value_as_u64)
16227}
16228
16229fn is_internal_timer_event(event: &HistoryEvent) -> bool {
16230    matches!(
16231        event
16232            .payload
16233            .get("timer_kind")
16234            .or_else(|| event.raw.get("timer_kind"))
16235            .and_then(Value::as_str),
16236        Some("condition_timeout" | "signal_timeout")
16237    )
16238}
16239
16240fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
16241    matches!(
16242        event.event_type.as_str(),
16243        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
16244    )
16245}
16246
16247fn recorded_condition_wait(
16248    sequence: u64,
16249    condition_events: &[&HistoryEvent],
16250    all_events: &[HistoryEvent],
16251) -> Result<RecordedCommand> {
16252    let opened = condition_events
16253        .iter()
16254        .copied()
16255        .filter(|event| event.event_type == "ConditionWaitOpened")
16256        .collect::<Vec<_>>();
16257    if opened.len() != 1 {
16258        return Err(invalid_recorded_history(
16259            "condition_wait_open_missing_or_duplicate",
16260            sequence,
16261            "one ConditionWaitOpened event",
16262            &format!("{} ConditionWaitOpened events", opened.len()),
16263            "condition replay requires exactly one canonical wait-open event",
16264        ));
16265    }
16266    let terminal = condition_events
16267        .iter()
16268        .copied()
16269        .filter(|event| {
16270            matches!(
16271                event.event_type.as_str(),
16272                "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
16273            )
16274        })
16275        .collect::<Vec<_>>();
16276    if terminal.len() > 1 {
16277        return Err(invalid_recorded_history(
16278            "duplicate_condition_wait_terminal_event",
16279            sequence,
16280            "at most one condition terminal event",
16281            "multiple condition terminal events",
16282            "condition history settles one durable wait more than once",
16283        ));
16284    }
16285
16286    let opened = opened[0];
16287    let condition_wait_id = required_condition_wait_id(opened, sequence)?;
16288    let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
16289    for event in condition_events
16290        .iter()
16291        .copied()
16292        .filter(|event| !std::ptr::eq(*event, opened))
16293    {
16294        let event_wait_id = required_condition_wait_id(event, sequence)?;
16295        if event_wait_id != condition_wait_id {
16296            return Err(invalid_recorded_history(
16297                "condition_wait_id_mismatch",
16298                sequence,
16299                &condition_wait_id,
16300                &event_wait_id,
16301                "condition lifecycle events at one sequence disagree on wait identity",
16302            ));
16303        }
16304        let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
16305        if event_occurrence_id != occurrence_id {
16306            return Err(invalid_recorded_history(
16307                "condition_wait_occurrence_history_mismatch",
16308                sequence,
16309                &occurrence_id,
16310                &event_occurrence_id,
16311                "condition lifecycle events at one sequence disagree on authored occurrence identity",
16312            ));
16313        }
16314    }
16315
16316    let condition_key = optional_non_empty_history_string(opened, "condition_key");
16317    let predicate_identity = opened
16318        .payload
16319        .get("condition_definition_fingerprint")
16320        .and_then(Value::as_str)
16321        .filter(|value| !value.is_empty())
16322        .map(str::to_string)
16323        .ok_or_else(|| {
16324            invalid_recorded_history(
16325                "condition_wait_predicate_fingerprint_missing",
16326                sequence,
16327                "non-empty condition_definition_fingerprint",
16328                &opened.event_type,
16329                "canonical condition history is missing its predicate identity",
16330            )
16331        })?;
16332    let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
16333    for event in condition_events
16334        .iter()
16335        .copied()
16336        .filter(|event| !std::ptr::eq(*event, opened))
16337    {
16338        for (field, opened_value) in [
16339            ("condition_key", condition_key.as_deref()),
16340            (
16341                "condition_definition_fingerprint",
16342                Some(predicate_identity.as_str()),
16343            ),
16344        ] {
16345            if let Some(value) = optional_non_empty_history_string(event, field) {
16346                if opened_value.is_some_and(|opened_value| opened_value != value) {
16347                    return Err(invalid_recorded_history(
16348                        "condition_wait_definition_history_mismatch",
16349                        sequence,
16350                        opened_value.unwrap_or_default(),
16351                        &value,
16352                        "condition lifecycle events disagree on the recorded definition",
16353                    ));
16354                }
16355            }
16356        }
16357        if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
16358            if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
16359                return Err(invalid_recorded_history(
16360                    "condition_wait_definition_history_mismatch",
16361                    sequence,
16362                    &format!("{}s", timeout_seconds.unwrap_or_default()),
16363                    &format!("{event_timeout}s"),
16364                    "condition lifecycle events disagree on the recorded timeout",
16365                ));
16366            }
16367        }
16368    }
16369
16370    let timeout_timer_events = all_events
16371        .iter()
16372        .filter(|event| {
16373            matches!(
16374                event.event_type.as_str(),
16375                "TimerScheduled" | "TimerCancelled" | "TimerFired"
16376            ) && event.payload.get("timer_kind").and_then(Value::as_str)
16377                == Some("condition_timeout")
16378                && event
16379                    .payload
16380                    .get("condition_wait_id")
16381                    .and_then(Value::as_str)
16382                    == Some(condition_wait_id.as_str())
16383        })
16384        .collect::<Vec<_>>();
16385    let scheduled = timeout_timer_events
16386        .iter()
16387        .copied()
16388        .filter(|event| event.event_type == "TimerScheduled")
16389        .collect::<Vec<_>>();
16390    let fired = timeout_timer_events
16391        .iter()
16392        .copied()
16393        .filter(|event| event.event_type == "TimerFired")
16394        .collect::<Vec<_>>();
16395    if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
16396        return Err(invalid_recorded_history(
16397            "condition_wait_timeout_history_invalid",
16398            sequence,
16399            "one timeout schedule and at most one fire",
16400            &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
16401            "condition timeout history has a missing or duplicate lifecycle event",
16402        ));
16403    }
16404    if let Some(scheduled) = scheduled.first() {
16405        let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
16406        let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
16407        if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
16408            return Err(invalid_recorded_history(
16409                "condition_wait_timeout_delay_mismatch",
16410                sequence,
16411                &format!("{}s", timeout_seconds.unwrap_or_default()),
16412                &format!("{delay_seconds}s"),
16413                "condition timeout timer differs from the wait definition",
16414            ));
16415        }
16416        if let Some(fired) = fired.first() {
16417            let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
16418            let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
16419            if fired_timer_id != timer_id || fired_delay != delay_seconds {
16420                return Err(invalid_recorded_history(
16421                    "condition_wait_timeout_identity_mismatch",
16422                    sequence,
16423                    &format!("{timer_id}:{delay_seconds}s"),
16424                    &format!("{fired_timer_id}:{fired_delay}s"),
16425                    "condition timeout fire does not match its durable schedule",
16426                ));
16427            }
16428        }
16429    }
16430
16431    let result = terminal.first().map(|event| {
16432        if event.event_type == "ConditionWaitTimedOut" {
16433            ConditionWaitResult::TimedOut
16434        } else {
16435            ConditionWaitResult::Satisfied
16436        }
16437    });
16438    let result = if !fired.is_empty() {
16439        if result == Some(ConditionWaitResult::Satisfied) {
16440            return Err(invalid_recorded_history(
16441                "condition_wait_terminal_conflict",
16442                sequence,
16443                "one satisfied or timed-out outcome",
16444                "satisfied event and fired timeout",
16445                "condition history records conflicting terminal outcomes",
16446            ));
16447        }
16448        Some(ConditionWaitResult::TimedOut)
16449    } else {
16450        result
16451    };
16452
16453    Ok(RecordedCommand::ConditionWait {
16454        sequence,
16455        occurrence_id,
16456        condition_key,
16457        predicate_identity,
16458        timeout_seconds,
16459        result,
16460        parallel_group_path: recorded_condition_group_path(condition_events, all_events, sequence)?,
16461    })
16462}
16463
16464fn recorded_condition_group_path(
16465    condition_events: &[&HistoryEvent],
16466    all_events: &[HistoryEvent],
16467    sequence: u64,
16468) -> Result<Option<Vec<ParallelGroupMetadata>>> {
16469    let opened = condition_events
16470        .iter()
16471        .copied()
16472        .find(|event| event.event_type == "ConditionWaitOpened")
16473        .expect("canonical condition opener is validated first");
16474    let occurrence = required_condition_wait_occurrence_id(opened, sequence)?;
16475    let previous_events = all_events
16476        .iter()
16477        .take_while(|event| !std::ptr::eq(*event, opened))
16478        .collect::<Vec<_>>();
16479    let originals = previous_events
16480        .iter()
16481        .copied()
16482        .filter(|event| event.event_type == "ConditionWaitOpened")
16483        .filter(|event| {
16484            event
16485                .payload
16486                .get("condition_wait_occurrence_id")
16487                .and_then(Value::as_str)
16488                == Some(occurrence.as_str())
16489        })
16490        .collect::<Vec<_>>();
16491    let Some(original) = originals.first().copied() else {
16492        return recorded_parallel_group_path(condition_events, sequence);
16493    };
16494    let authored_sequence = required_history_u64(original, "sequence", sequence)?;
16495    if authored_sequence == sequence {
16496        return recorded_parallel_group_path(condition_events, sequence);
16497    }
16498    let original_path = recorded_parallel_group_path(&[original], authored_sequence)?;
16499    if original_path.is_none() {
16500        return recorded_parallel_group_path(condition_events, sequence);
16501    }
16502    let reopened_path = recorded_parallel_group_path(condition_events, authored_sequence)?;
16503    let identity_matches = [
16504        "condition_key",
16505        "condition_definition_fingerprint",
16506        "timeout_seconds",
16507    ]
16508    .iter()
16509    .all(|field| opened.payload.get(*field) == original.payload.get(*field));
16510    let previous = originals
16511        .iter()
16512        .copied()
16513        .filter(|event| durable_event_sequence(event).is_some_and(|physical| physical < sequence))
16514        .max_by_key(|event| durable_event_sequence(event));
16515    let previously_satisfied = previous.is_some_and(|previous| {
16516        previous_events.iter().any(|event| {
16517            event.event_type == "ConditionWaitSatisfied"
16518                && event.payload.get("condition_wait_id")
16519                    == previous.payload.get("condition_wait_id")
16520                && event.payload.get("condition_wait_occurrence_id")
16521                    == previous.payload.get("condition_wait_occurrence_id")
16522                && durable_event_sequence(event) == durable_event_sequence(previous)
16523        })
16524    });
16525    if !identity_matches || reopened_path != original_path || !previously_satisfied {
16526        return Err(invalid_recorded_history(
16527            "condition_wait_group_reopen_mismatch",
16528            sequence,
16529            "same authored condition and group path after a satisfied physical wait",
16530            &opened.payload.to_string(),
16531            "grouped condition reopen has no matching recorded predecessor",
16532        ));
16533    }
16534    Ok(reopened_path)
16535}
16536
16537fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
16538    event
16539        .payload
16540        .get("condition_wait_occurrence_id")
16541        .and_then(Value::as_str)
16542        .filter(|value| !value.is_empty())
16543        .map(str::to_string)
16544        .ok_or_else(|| {
16545            invalid_recorded_history(
16546                "condition_wait_occurrence_id_missing",
16547                sequence,
16548                "non-empty condition_wait_occurrence_id",
16549                &event.event_type,
16550                "condition history is missing authored occurrence identity",
16551            )
16552        })
16553}
16554
16555fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
16556    event
16557        .payload
16558        .get("condition_wait_id")
16559        .and_then(Value::as_str)
16560        .filter(|value| !value.is_empty())
16561        .map(str::to_string)
16562        .ok_or_else(|| {
16563            invalid_recorded_history(
16564                "condition_wait_id_missing",
16565                sequence,
16566                "non-empty condition_wait_id",
16567                &event.event_type,
16568                "canonical condition history is missing its durable wait identity",
16569            )
16570        })
16571}
16572
16573fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
16574    event
16575        .payload
16576        .get(field)
16577        .and_then(Value::as_str)
16578        .filter(|value| !value.is_empty())
16579        .map(str::to_string)
16580}
16581
16582fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
16583    match event.payload.get(field) {
16584        None | Some(Value::Null) => Ok(None),
16585        Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
16586            invalid_recorded_history(
16587                "condition_wait_definition_invalid",
16588                sequence,
16589                &format!("non-negative integer {field}"),
16590                &value.to_string(),
16591                "condition history contains an invalid numeric definition field",
16592            )
16593        }),
16594    }
16595}
16596
16597fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
16598    event
16599        .payload
16600        .get("signal_name")
16601        .or_else(|| event.raw.get("signal_name"))
16602        .and_then(Value::as_str)
16603        .filter(|value| !value.is_empty())
16604        .map(str::to_string)
16605        .ok_or_else(|| {
16606            invalid_recorded_history(
16607                "signal_wait_name_missing",
16608                sequence,
16609                "non-empty signal_name",
16610                &event.event_type,
16611                "canonical signal-wait history is missing its signal identity",
16612            )
16613        })
16614}
16615
16616fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
16617    // Applying a buffered control-plane signal has no authored call sequence.
16618    // Its outer sequence orders history events and must not become a wait.
16619    if event.event_type == "SignalApplied"
16620        && event
16621            .payload
16622            .get("signal_wait_id")
16623            .and_then(Value::as_str)
16624            .is_some_and(|id| id.starts_with("signal-command:"))
16625        && event.payload.get("sequence").is_none()
16626        && event.payload.get("workflow_sequence").is_none()
16627    {
16628        return false;
16629    }
16630    matches!(
16631        event.event_type.as_str(),
16632        "SignalWaitOpened" | "SignalApplied"
16633    )
16634}
16635
16636fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
16637    event
16638        .payload
16639        .get(field)
16640        .and_then(Value::as_str)
16641        .filter(|value| !value.is_empty())
16642        .map(str::to_string)
16643        .ok_or_else(|| {
16644            invalid_recorded_history(
16645                "timer_history_field_missing",
16646                sequence,
16647                field,
16648                &event.event_type,
16649                "timer history is missing a required identity field",
16650            )
16651        })
16652}
16653
16654fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
16655    event
16656        .payload
16657        .get(field)
16658        .and_then(value_as_u64)
16659        .ok_or_else(|| {
16660            invalid_recorded_history(
16661                "timer_history_field_missing",
16662                sequence,
16663                field,
16664                &event.event_type,
16665                "timer history is missing a required numeric field",
16666            )
16667        })
16668}
16669
16670fn recorded_search_attribute_types(
16671    payload: &Value,
16672    attributes: &Value,
16673    sequence: u64,
16674) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
16675    let Some(raw_types) = payload.get("attribute_types") else {
16676        // This is the explicit compatibility rule for histories recorded
16677        // before typed identity was persisted. Values still constrain replay;
16678        // the unknown type snapshot does not assert a typed match.
16679        return Ok(RecordedSnapshotValue::Unknown);
16680    };
16681    let Some(raw_types) = raw_types.as_object() else {
16682        return Err(invalid_recorded_history(
16683            "search_attribute_types_malformed",
16684            sequence,
16685            "canonical attribute type map",
16686            &raw_types.to_string(),
16687            "search-attribute history contains malformed type identity",
16688        ));
16689    };
16690    let attribute_keys = attributes
16691        .as_object()
16692        .expect("recorded search attributes were validated as an object");
16693    let mut types = BTreeMap::new();
16694    for (key, value) in raw_types {
16695        let Some(attribute_type) = value.as_str() else {
16696            return Err(invalid_recorded_history(
16697                "search_attribute_types_malformed",
16698                sequence,
16699                "canonical string type name",
16700                &value.to_string(),
16701                "search-attribute history contains a non-string type identity",
16702            ));
16703        };
16704        if !attribute_keys.contains_key(key)
16705            || !matches!(
16706                attribute_type,
16707                "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
16708            )
16709        {
16710            return Err(invalid_recorded_history(
16711                "search_attribute_types_malformed",
16712                sequence,
16713                "canonical types for keys present in attributes",
16714                &format!("{key}:{attribute_type}"),
16715                "search-attribute history contains unsupported or orphaned type identity",
16716            ));
16717        }
16718        types.insert(key.clone(), attribute_type.to_string());
16719    }
16720    Ok(RecordedSnapshotValue::Known(types))
16721}
16722
16723fn invalid_recorded_history(
16724    reason: &str,
16725    sequence: u64,
16726    expected: &str,
16727    actual: &str,
16728    message: &str,
16729) -> Error {
16730    Error::NonDeterministicReplay(ReplayFailure::new(
16731        reason,
16732        Some(sequence),
16733        Some(expected.to_string()),
16734        Some(actual.to_string()),
16735        message,
16736    ))
16737}
16738
16739type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
16740
16741fn activity_outcome(
16742    event: &HistoryEvent,
16743    fallback_codec: &str,
16744    recorded_activity_type: Option<String>,
16745) -> Result<ActivityOutcome> {
16746    if event.event_type == "ActivityCompleted" {
16747        let codec = event
16748            .payload
16749            .get("payload_codec")
16750            .and_then(Value::as_str)
16751            .unwrap_or(fallback_codec);
16752        return Ok(Ok(decode_wire_avro_value(
16753            event.payload.get("result").unwrap_or(&Value::Null),
16754            codec,
16755        )?));
16756    }
16757
16758    let payload = &event.payload;
16759    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
16760        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
16761        "ActivityCancelled" => (
16762            ActivityFailureKind::Cancelled,
16763            "cancelled",
16764            "activity was cancelled",
16765        ),
16766        "ActivityTimedOut" => (
16767            ActivityFailureKind::TimedOut,
16768            "timeout",
16769            "activity timed out",
16770        ),
16771        _ => unreachable!("activity_outcome is called only for terminal activity events"),
16772    };
16773    let exception = payload
16774        .get("exception")
16775        .filter(|value| !value.is_null())
16776        .cloned();
16777    let failure_category = payload_string(payload, "failure_category");
16778    let timeout_kind = payload_string(payload, "timeout_kind");
16779    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
16780        ActivityFailureKind::Failed => failure_category
16781            .clone()
16782            .unwrap_or_else(|| fallback_reason.to_string()),
16783        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
16784        ActivityFailureKind::TimedOut => timeout_kind
16785            .clone()
16786            .unwrap_or_else(|| fallback_reason.to_string()),
16787    });
16788    let message = payload_string(payload, "message")
16789        .or_else(|| {
16790            exception
16791                .as_ref()
16792                .and_then(|value| payload_string(value, "message"))
16793        })
16794        .unwrap_or_else(|| fallback_message.to_string());
16795
16796    Ok(Err(ActivityFailure {
16797        kind,
16798        reason,
16799        message,
16800        activity_execution_id: payload_string(payload, "activity_execution_id"),
16801        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
16802        activity_type: payload_string(payload, "activity_type")
16803            .or_else(|| payload_string(payload, "activity_name"))
16804            .or(recorded_activity_type),
16805        activity_class: payload_string(payload, "activity_class"),
16806        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
16807        failure_id: payload_string(payload, "failure_id"),
16808        failure_category,
16809        timeout_kind,
16810        non_retryable: payload
16811            .get("non_retryable")
16812            .and_then(Value::as_bool)
16813            .unwrap_or(false),
16814        exception_type: payload_string(payload, "exception_type").or_else(|| {
16815            exception
16816                .as_ref()
16817                .and_then(|value| payload_string(value, "type"))
16818        }),
16819        exception_class: payload_string(payload, "exception_class").or_else(|| {
16820            exception
16821                .as_ref()
16822                .and_then(|value| payload_string(value, "class"))
16823        }),
16824        code: payload
16825            .get("code")
16826            .filter(|value| !value.is_null())
16827            .cloned(),
16828        exception,
16829    }))
16830}
16831
16832type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
16833
16834fn child_workflow_outcomes(
16835    events: &[HistoryEvent],
16836    fallback_codec: &str,
16837    parent: WorkflowIdentity,
16838) -> Result<Vec<ChildWorkflowOutcome>> {
16839    let mut outcomes = Vec::new();
16840
16841    for event in events {
16842        let kind = match event.event_type.as_str() {
16843            "ChildRunCompleted" => None,
16844            "ChildRunFailed" => Some((
16845                ChildWorkflowFailureKind::Failed,
16846                "child_workflow",
16847                "child workflow failed",
16848            )),
16849            "ChildRunCancelled" => Some((
16850                ChildWorkflowFailureKind::Cancelled,
16851                "cancelled",
16852                "child workflow was cancelled",
16853            )),
16854            "ChildRunTerminated" => Some((
16855                ChildWorkflowFailureKind::Terminated,
16856                "terminated",
16857                "child workflow was terminated",
16858            )),
16859            _ => continue,
16860        };
16861        let payload = &event.payload;
16862        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
16863        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
16864        let child_workflow_type = payload_string(payload, "child_workflow_type");
16865
16866        if let Some((kind, reason, fallback_message)) = kind {
16867            let exception = payload
16868                .get("exception")
16869                .filter(|value| !value.is_null())
16870                .cloned();
16871            let message = payload_string(payload, "message")
16872                .or_else(|| {
16873                    exception
16874                        .as_ref()
16875                        .and_then(|value| payload_string(value, "message"))
16876                })
16877                .unwrap_or_else(|| fallback_message.to_string());
16878            let exception_type = payload_string(payload, "exception_type").or_else(|| {
16879                exception
16880                    .as_ref()
16881                    .and_then(|value| payload_string(value, "type"))
16882            });
16883            let exception_class = payload_string(payload, "exception_class").or_else(|| {
16884                exception
16885                    .as_ref()
16886                    .and_then(|value| payload_string(value, "class"))
16887            });
16888            outcomes.push(Err(ChildWorkflowFailure {
16889                kind,
16890                reason: reason.to_string(),
16891                message,
16892                parent_workflow_id: parent.workflow_id.clone(),
16893                parent_workflow_run_id: parent.run_id.clone(),
16894                child_workflow_id,
16895                child_workflow_run_id,
16896                child_workflow_type,
16897                failure_id: payload_string(payload, "failure_id"),
16898                failure_category: payload_string(payload, "failure_category"),
16899                exception_type,
16900                exception_class,
16901                non_retryable: payload
16902                    .get("non_retryable")
16903                    .and_then(Value::as_bool)
16904                    .unwrap_or(false),
16905                code: payload
16906                    .get("code")
16907                    .filter(|value| !value.is_null())
16908                    .cloned(),
16909                exception,
16910            }));
16911            continue;
16912        }
16913
16914        let codec = payload
16915            .get("payload_codec")
16916            .and_then(Value::as_str)
16917            .unwrap_or(fallback_codec);
16918        let result = payload
16919            .get("result")
16920            .or_else(|| payload.get("output"))
16921            .unwrap_or(&Value::Null);
16922        outcomes.push(Ok(ChildWorkflowAvroResult {
16923            parent: parent.clone(),
16924            child: WorkflowIdentity {
16925                workflow_id: child_workflow_id,
16926                run_id: child_workflow_run_id,
16927            },
16928            child_workflow_type,
16929            result: decode_wire_avro_value(result, codec)?,
16930        }));
16931    }
16932
16933    Ok(outcomes)
16934}
16935
16936fn payload_string(payload: &Value, key: &str) -> Option<String> {
16937    payload
16938        .get(key)
16939        .and_then(Value::as_str)
16940        .filter(|value| !value.is_empty())
16941        .map(str::to_string)
16942}
16943
16944fn recorded_activity_failure_boundary(
16945    ctx: &WorkflowContext,
16946    error: &Error,
16947) -> Option<(u64, String)> {
16948    let Error::ActivityFailed(failure) = error else {
16949        return None;
16950    };
16951    if failure.kind != ActivityFailureKind::Failed {
16952        return None;
16953    }
16954    let activity_execution_id = failure
16955        .activity_execution_id
16956        .as_deref()
16957        .filter(|value| !value.is_empty())?;
16958    let state = ctx.state.lock().ok()?;
16959    let mut matches = state.recorded_commands.iter().filter_map(|command| {
16960        let RecordedCommand::Activity {
16961            sequence,
16962            outcome: Some(Err(recorded_failure)),
16963            ..
16964        } = command
16965        else {
16966            return None;
16967        };
16968        (*sequence > 0 && recorded_failure == failure)
16969            .then_some((*sequence, activity_execution_id.to_string()))
16970    });
16971    let boundary = matches.next()?;
16972    matches.next().is_none().then_some(boundary)
16973}
16974
16975fn workflow_failure_command(
16976    error: &Error,
16977    failed_activity_boundary: Option<(u64, String)>,
16978) -> Value {
16979    let (exception_type, exception_class, properties) = match error {
16980        Error::ActivityFailed(failure) => (
16981            match failure.kind {
16982                ActivityFailureKind::Failed => "ActivityFailed",
16983                ActivityFailureKind::Cancelled => "ActivityCancelled",
16984                ActivityFailureKind::TimedOut => "ActivityTimedOut",
16985            },
16986            "durable_workflow::ActivityFailure",
16987            json!({
16988                "reason": failure.reason,
16989                "activity_execution_id": failure.activity_execution_id,
16990                "activity_attempt_id": failure.activity_attempt_id,
16991                "activity_type": failure.activity_type,
16992                "activity_class": failure.activity_class,
16993                "attempt_number": failure.attempt_number,
16994                "failure_id": failure.failure_id,
16995                "failure_category": failure.failure_category,
16996                "timeout_kind": failure.timeout_kind,
16997                "activity_non_retryable": failure.non_retryable,
16998                "activity_exception_type": failure.exception_type,
16999                "activity_exception_class": failure.exception_class,
17000                "activity_code": failure.code,
17001                "activity_exception": failure.exception,
17002            }),
17003        ),
17004        Error::ChildWorkflowFailed(failure) => (
17005            match failure.kind {
17006                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
17007                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
17008                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
17009            },
17010            "durable_workflow::ChildWorkflowFailure",
17011            json!({
17012                "reason": failure.reason,
17013                "parent_workflow_id": failure.parent_workflow_id,
17014                "parent_workflow_run_id": failure.parent_workflow_run_id,
17015                "child_workflow_id": failure.child_workflow_id,
17016                "child_workflow_run_id": failure.child_workflow_run_id,
17017                "child_workflow_type": failure.child_workflow_type,
17018                "failure_id": failure.failure_id,
17019                "failure_category": failure.failure_category,
17020                "child_exception_type": failure.exception_type,
17021                "child_exception_class": failure.exception_class,
17022                "child_non_retryable": failure.non_retryable,
17023                "child_code": failure.code,
17024                "child_exception": failure.exception,
17025            }),
17026        ),
17027        Error::ParallelFailed(failure) => (
17028            "ParallelFailed",
17029            "durable_workflow::ParallelFailure",
17030            json!({
17031                "parallel_group_id": failure.group_id,
17032                "parallel_member_path": failure.member_path,
17033                "parallel_group_path": failure.group_path,
17034                "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
17035                "cause_type": workflow_error_type(&failure.cause),
17036                "cause_message": failure.cause.to_string(),
17037            }),
17038        ),
17039        Error::SagaCompensationFailed(failure) => (
17040            "SagaCompensationFailed",
17041            "durable_workflow::SagaCompensationFailure",
17042            json!({
17043                "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
17044                "initiating_failure_message": failure.initiating_failure.to_string(),
17045                "compensation_activity_type": failure.compensation_activity_type,
17046                "compensation_registration_order": failure.compensation_registration_order,
17047                "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
17048                "compensation_failure_message": failure.compensation_failure.to_string(),
17049            }),
17050        ),
17051        Error::WorkflowCancellationRequested(_) => (
17052            "WorkflowCancellationRequested",
17053            "durable_workflow::WorkflowCancellationRequested",
17054            json!({"reason": "cancelled"}),
17055        ),
17056        Error::CooperativeCancellationRequested(cancellation) => (
17057            "WorkflowCancellationRequested",
17058            "durable_workflow::CooperativeCancellationRequested",
17059            json!({"reason": "cancelled", "request_id": cancellation.request.request_id,
17060                "cleanup_deadline_at": cancellation.request.cleanup_deadline_at}),
17061        ),
17062        Error::CancellationScopeRequested(cancellation) => (
17063            "WorkflowCancellationRequested",
17064            "durable_workflow::CancellationScopeRequested",
17065            json!({"reason":"scope_cancelled", "request_id":cancellation.context.request_id(),
17066                "cleanup_deadline_at":cancellation.context.deadline().to_rfc3339(),
17067                "cancellation":cancellation.context.to_value()}),
17068        ),
17069        Error::NonDeterministicReplay(_) => (
17070            "NonDeterministicReplay",
17071            "durable_workflow::Error",
17072            Value::Null,
17073        ),
17074        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
17075    };
17076    let non_retryable = match error {
17077        Error::ActivityFailed(failure) => failure.non_retryable,
17078        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
17079        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
17080        Error::SagaCompensationFailed(failure) => {
17081            workflow_error_non_retryable(&failure.compensation_failure)
17082        }
17083        Error::WorkflowCancellationRequested(_)
17084        | Error::CooperativeCancellationRequested(_)
17085        | Error::CancellationScopeRequested(_) => true,
17086        Error::NonDeterministicReplay(_) => true,
17087        _ => false,
17088    };
17089
17090    let mut command = json!({
17091        "type": "fail_workflow",
17092        "message": error.to_string(),
17093        "exception_type": exception_type,
17094        "exception_class": exception_class,
17095        "non_retryable": non_retryable,
17096        "exception": {
17097            "type": exception_type,
17098            "class": exception_class,
17099            "message": error.to_string(),
17100            "properties": properties,
17101        }
17102    });
17103
17104    if let Some((sequence, activity_execution_id)) = failed_activity_boundary {
17105        command["failed_step_sequence"] = json!(sequence);
17106        command["failed_activity_execution_id"] = json!(activity_execution_id);
17107    }
17108
17109    command
17110}
17111
17112fn workflow_error_type(error: &Error) -> &'static str {
17113    match error {
17114        Error::ActivityFailed(failure) => match failure.kind {
17115            ActivityFailureKind::Failed => "ActivityFailed",
17116            ActivityFailureKind::Cancelled => "ActivityCancelled",
17117            ActivityFailureKind::TimedOut => "ActivityTimedOut",
17118        },
17119        Error::ChildWorkflowFailed(failure) => match failure.kind {
17120            ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
17121            ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
17122            ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
17123        },
17124        Error::ParallelFailed(_) => "ParallelFailed",
17125        Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
17126        Error::WorkflowCancellationRequested(_)
17127        | Error::CooperativeCancellationRequested(_)
17128        | Error::CancellationScopeRequested(_) => "WorkflowCancellationRequested",
17129        Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
17130        _ => "RustWorkflowError",
17131    }
17132}
17133
17134fn workflow_error_non_retryable(error: &Error) -> bool {
17135    match error {
17136        Error::ActivityFailed(failure) => failure.non_retryable,
17137        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
17138        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
17139        Error::SagaCompensationFailed(failure) => {
17140            workflow_error_non_retryable(&failure.compensation_failure)
17141        }
17142        Error::WorkflowCancellationRequested(_)
17143        | Error::CooperativeCancellationRequested(_)
17144        | Error::CancellationScopeRequested(_)
17145        | Error::NonDeterministicReplay(_) => true,
17146        _ => false,
17147    }
17148}
17149
17150fn workflow_task_integrity_error(error: &Error) -> bool {
17151    matches!(
17152        error,
17153        Error::NonDeterministicReplay(_)
17154            | Error::Protocol(_)
17155            | Error::CancellationScopeExecutionUnavailable
17156            | Error::MissingWorkflowCommandIdentity
17157            | Error::WorkflowStatePoisoned
17158    )
17159}
17160
17161fn decode_signal_event_arguments(
17162    event: &HistoryEvent,
17163    fallback_codec: &str,
17164) -> Result<Vec<AvroValue>> {
17165    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
17166    validate_payload_codec(codec)?;
17167    let raw = signal_history_payload(&event.payload);
17168    let decoded = match raw.filter(|value| !value.is_null()) {
17169        Some(value) => decode_wire_avro_value(value, codec)?,
17170        None => AvroValue::Array(Vec::new()),
17171    };
17172    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
17173        unreachable!("normalize_avro_arguments always returns an array");
17174    };
17175    Ok(arguments)
17176}
17177
17178fn decode_update_event_arguments(
17179    event: &HistoryEvent,
17180    fallback_codec: &str,
17181) -> Result<Vec<AvroValue>> {
17182    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
17183    validate_payload_codec(codec)?;
17184    let decoded = match event
17185        .payload
17186        .get("arguments")
17187        .filter(|value| !value.is_null())
17188    {
17189        Some(value) => decode_wire_avro_value(value, codec)?,
17190        None => AvroValue::Array(Vec::new()),
17191    };
17192    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
17193        unreachable!("normalize_avro_arguments always returns an array");
17194    };
17195    Ok(arguments)
17196}
17197
17198fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
17199    let Some(export_events) = task
17200        .history_export
17201        .as_ref()
17202        .and_then(|export| export.get("history_events"))
17203        .and_then(Value::as_array)
17204    else {
17205        return Ok(());
17206    };
17207
17208    if export_events.len() > task.history_events.len() {
17209        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
17210    }
17211
17212    Ok(())
17213}
17214
17215fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
17216    let Some(export) = task.history_export.as_ref() else {
17217        return Ok(());
17218    };
17219    let signals = export
17220        .get("signals")
17221        .and_then(Value::as_array)
17222        .cloned()
17223        .unwrap_or_default();
17224    let activities = export
17225        .get("activities")
17226        .and_then(Value::as_array)
17227        .cloned()
17228        .unwrap_or_default();
17229    let export_codec = export
17230        .get("payloads")
17231        .and_then(|payloads| payloads.get("codec"))
17232        .and_then(Value::as_str)
17233        .unwrap_or(&task.payload_codec)
17234        .to_string();
17235    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
17236
17237    for event in &mut task.history_events {
17238        if event.event_type == "ActivityCompleted" {
17239            let sequence = event
17240                .payload
17241                .get("sequence")
17242                .or_else(|| event.payload.get("workflow_sequence"))
17243                .and_then(value_as_u64);
17244            let Some(activity) = sequence.and_then(|sequence| {
17245                activities.iter().find(|activity| {
17246                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
17247                })
17248            }) else {
17249                continue;
17250            };
17251            let Some(payload) = event.payload.as_object_mut() else {
17252                continue;
17253            };
17254            if missing_payload(payload.get("result")) {
17255                if let Some(result) = activity
17256                    .get("result")
17257                    .filter(|value| !missing_payload(Some(value)))
17258                {
17259                    payload.insert("result".to_string(), result.clone());
17260                }
17261            }
17262            for field in ["payload_codec", "activity_type"] {
17263                if payload
17264                    .get(field)
17265                    .and_then(Value::as_str)
17266                    .unwrap_or_default()
17267                    .is_empty()
17268                {
17269                    if let Some(value) = activity.get(field) {
17270                        payload.insert(field.to_string(), value.clone());
17271                    }
17272                }
17273            }
17274            continue;
17275        }
17276
17277        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
17278            continue;
17279        }
17280        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
17281        let command_id = event
17282            .payload
17283            .get("workflow_command_id")
17284            .or_else(|| event.raw.get("workflow_command_id"))
17285            .and_then(Value::as_str);
17286        let signal_name = event
17287            .payload
17288            .get("signal_name")
17289            .and_then(Value::as_str)
17290            .unwrap_or_default()
17291            .to_string();
17292        let matched = signals
17293            .iter()
17294            .find(|signal| {
17295                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
17296            })
17297            .or_else(|| {
17298                signals.iter().find(|signal| {
17299                    command_id.is_some()
17300                        && signal.get("command_id").and_then(Value::as_str) == command_id
17301                })
17302            })
17303            .or_else(|| {
17304                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
17305                let signal = signals
17306                    .iter()
17307                    .filter(|signal| {
17308                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
17309                    })
17310                    .nth(*offset);
17311                if signal.is_some() {
17312                    *offset += 1;
17313                }
17314                signal
17315            });
17316        let Some(signal) = matched else {
17317            continue;
17318        };
17319        let signal_codec = signal
17320            .get("payload_codec")
17321            .and_then(Value::as_str)
17322            .unwrap_or(&export_codec);
17323        let Some(payload) = event.payload.as_object_mut() else {
17324            continue;
17325        };
17326        if missing_payload(payload.get("arguments")) {
17327            if let Some(arguments) = signal
17328                .get("arguments")
17329                .filter(|value| !missing_payload(Some(value)))
17330            {
17331                let envelope = match arguments {
17332                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
17333                    other => other.clone(),
17334                };
17335                payload.insert("arguments".to_string(), envelope);
17336            }
17337        }
17338        if payload
17339            .get("payload_codec")
17340            .and_then(Value::as_str)
17341            .unwrap_or_default()
17342            .is_empty()
17343        {
17344            payload.insert("payload_codec".to_string(), json!(signal_codec));
17345        }
17346    }
17347
17348    Ok(())
17349}
17350
17351fn missing_payload(value: Option<&Value>) -> bool {
17352    match value {
17353        None | Some(Value::Null) => true,
17354        Some(Value::String(value)) => value.is_empty(),
17355        Some(_) => false,
17356    }
17357}
17358
17359fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
17360    history_signal_events(
17361        &task.history_events,
17362        task.history_export.as_ref(),
17363        &task.payload_codec,
17364    )
17365}
17366
17367fn history_signal_events(
17368    history_events: &[HistoryEvent],
17369    history_export: Option<&Value>,
17370    payload_codec: &str,
17371) -> Result<Vec<QuerySignal>> {
17372    let export_signals = history_export
17373        .and_then(|export| export.get("signals"))
17374        .and_then(Value::as_array)
17375        .cloned()
17376        .unwrap_or_default();
17377    let export_codec = history_export
17378        .and_then(|export| export.get("payloads"))
17379        .and_then(|payloads| payloads.get("codec"))
17380        .and_then(Value::as_str)
17381        .unwrap_or(payload_codec);
17382    let mut name_offsets: HashMap<String, usize> = HashMap::new();
17383    let mut signals: Vec<QuerySignal> = Vec::new();
17384    let mut signal_indices = HashMap::new();
17385    let mut command_indices = HashMap::new();
17386
17387    for event in history_events {
17388        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
17389            continue;
17390        }
17391
17392        let name = event
17393            .payload
17394            .get("signal_name")
17395            .and_then(Value::as_str)
17396            .unwrap_or_default();
17397        if name.is_empty() {
17398            continue;
17399        }
17400        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
17401        let command_id = event
17402            .payload
17403            .get("workflow_command_id")
17404            .or_else(|| event.raw.get("workflow_command_id"))
17405            .and_then(Value::as_str);
17406        let matched_export = export_signals
17407            .iter()
17408            .find(|candidate| {
17409                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
17410            })
17411            .or_else(|| {
17412                export_signals.iter().find(|candidate| {
17413                    command_id.is_some()
17414                        && candidate.get("command_id").and_then(Value::as_str) == command_id
17415                })
17416            })
17417            .or_else(|| {
17418                let offset = name_offsets.entry(name.to_string()).or_default();
17419                let candidate = export_signals
17420                    .iter()
17421                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
17422                    .nth(*offset);
17423                if candidate.is_some() {
17424                    *offset += 1;
17425                }
17426                candidate
17427            });
17428        let codec = event
17429            .payload
17430            .get("payload_codec")
17431            .and_then(Value::as_str)
17432            .or_else(|| {
17433                matched_export
17434                    .and_then(|signal| signal.get("payload_codec"))
17435                    .and_then(Value::as_str)
17436            })
17437            .unwrap_or(export_codec);
17438        let raw_arguments = signal_history_payload(&event.payload)
17439            .filter(|value| !value.is_null())
17440            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
17441        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
17442        let workflow_sequence = event
17443            .payload
17444            .get("workflow_sequence")
17445            .and_then(value_as_u64)
17446            .or_else(|| {
17447                matched_export
17448                    .and_then(|signal| signal.get("workflow_sequence"))
17449                    .and_then(value_as_u64)
17450            });
17451
17452        let id = signal_id.or_else(|| {
17453            matched_export
17454                .and_then(|signal| signal.get("id"))
17455                .and_then(Value::as_str)
17456        });
17457        let command_id = command_id.or_else(|| {
17458            matched_export
17459                .and_then(|signal| signal.get("command_id"))
17460                .and_then(Value::as_str)
17461        });
17462        let duplicate = id
17463            .and_then(|id| signal_indices.get(id))
17464            .or_else(|| command_id.and_then(|id| command_indices.get(id)))
17465            .copied();
17466        let index = duplicate.unwrap_or(signals.len());
17467        if let Some(index) = duplicate {
17468            let original = &mut signals[index];
17469            let same_arguments = if event.event_type == "SignalApplied" {
17470                if let Some(value) = event.payload.get("value") {
17471                    let expected = match original.avro_arguments.as_slice() {
17472                        [] => AvroValue::Boolean(true),
17473                        [value] => value.clone(),
17474                        values => AvroValue::Array(values.to_vec()),
17475                    };
17476                    let applied = decode_wire_avro_value(value, codec)?;
17477                    applied == expected
17478                        || applied == AvroValue::Array(original.avro_arguments.clone())
17479                } else {
17480                    original.avro_arguments == avro_arguments
17481                }
17482            } else {
17483                original.avro_arguments == avro_arguments
17484            };
17485            if original.name != name
17486                || !same_arguments
17487                || matches!((original.id.as_deref(), id), (Some(first), Some(next)) if first != next)
17488            {
17489                return Err(Error::Codec("conflicting committed signal identity".into()));
17490            }
17491            if original.id.is_none() {
17492                original.id = id.map(str::to_string);
17493            }
17494            original.workflow_sequence = original.workflow_sequence.or(workflow_sequence);
17495        } else {
17496            signals.push(QuerySignal {
17497                id: id.map(str::to_string),
17498                name: name.to_string(),
17499                arguments,
17500                avro_arguments,
17501                workflow_sequence,
17502            });
17503        }
17504        if let Some(id) = id {
17505            signal_indices.insert(id.to_string(), index);
17506        }
17507        if let Some(id) = command_id {
17508            command_indices.insert(id.to_string(), index);
17509        }
17510    }
17511
17512    if signals.is_empty() {
17513        for signal in export_signals {
17514            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
17515                continue;
17516            }
17517            let Some(name) = signal.get("name").and_then(Value::as_str) else {
17518                continue;
17519            };
17520            let codec = signal
17521                .get("payload_codec")
17522                .and_then(Value::as_str)
17523                .unwrap_or(export_codec);
17524            let (arguments, avro_arguments) =
17525                decode_query_signal_arguments(signal.get("arguments"), codec)?;
17526            signals.push(QuerySignal {
17527                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
17528                name: name.to_string(),
17529                arguments,
17530                avro_arguments,
17531                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
17532            });
17533        }
17534        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
17535    }
17536
17537    Ok(signals)
17538}
17539
17540fn decode_query_signal_arguments(
17541    raw: Option<&Value>,
17542    codec: &str,
17543) -> Result<(Vec<Value>, Vec<AvroValue>)> {
17544    validate_payload_codec(codec)?;
17545    let decoded = match raw.filter(|value| !value.is_null()) {
17546        Some(value) => decode_wire_avro_value(value, codec)?,
17547        None => AvroValue::Array(Vec::new()),
17548    };
17549    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
17550        unreachable!("normalize_avro_arguments always returns an array");
17551    };
17552    let arguments = avro_arguments
17553        .iter()
17554        .cloned()
17555        .map(AvroValue::into_json)
17556        .collect::<Result<Vec<_>>>()?;
17557    Ok((arguments, avro_arguments))
17558}
17559
17560fn value_as_u64(value: &Value) -> Option<u64> {
17561    value
17562        .as_u64()
17563        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
17564}
17565
17566#[cfg(test)]
17567mod tests {
17568    use super::*;
17569    mod activity_cancellation_policies;
17570    mod cancellation_scope_admission;
17571    mod cancellation_scope_authoring;
17572    mod cancellation_scope_history;
17573    mod cancellation_scope_opening;
17574    mod cancellation_scope_replay;
17575    mod child_workflow_policies;
17576    mod cooperative_cancellation;
17577    mod local_activity;
17578    mod runtime_payloads;
17579    mod runtime_uploads;
17580    mod worker_session;
17581    use std::{
17582        fs,
17583        io::{Read, Write},
17584        net::{SocketAddr, TcpListener, TcpStream},
17585        process::Command as ProcessCommand,
17586        sync::atomic::AtomicUsize,
17587        thread,
17588    };
17589
17590    #[derive(Clone, Copy, Debug)]
17591    enum InvalidTaskPayloadCodec {
17592        Missing,
17593        Null,
17594        NonString,
17595    }
17596
17597    impl InvalidTaskPayloadCodec {
17598        fn label(self) -> &'static str {
17599            match self {
17600                Self::Missing => "missing",
17601                Self::Null => "null",
17602                Self::NonString => "non-string",
17603            }
17604        }
17605
17606        fn apply(self, task: &mut Value) {
17607            let task = task.as_object_mut().expect("task fixture object");
17608            match self {
17609                Self::Missing => {
17610                    task.remove("payload_codec");
17611                }
17612                Self::Null => {
17613                    task.insert("payload_codec".to_string(), Value::Null);
17614                }
17615                Self::NonString => {
17616                    task.insert("payload_codec".to_string(), json!(42));
17617                }
17618            }
17619        }
17620    }
17621
17622    fn fixture_envelope(value: Value) -> Value {
17623        encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
17624    }
17625
17626    fn fixture_blob(value: Value) -> String {
17627        encode_payload(&value, DEFAULT_CODEC)
17628            .expect("encode Avro test fixture")
17629            .blob
17630    }
17631
17632    #[test]
17633    fn client_builder_rejects_the_sdk_owned_api_suffix() {
17634        for base_url in [
17635            "http://127.0.0.1:8080/api",
17636            "http://localhost:8080/api/",
17637            "https://runtime.example.test/namespaces/orders/api",
17638        ] {
17639            let error = Client::builder(base_url)
17640                .build()
17641                .expect_err("SDK-owned /api suffix must be rejected during build");
17642
17643            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
17644            assert!(
17645                error.to_string().contains("SDK appends /api automatically"),
17646                "the validation error must explain how to fix the endpoint"
17647            );
17648        }
17649    }
17650
17651    #[test]
17652    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
17653        for (base_url, expected) in [
17654            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
17655            (
17656                "http://localhost:8080/durable-workflow/",
17657                "http://localhost:8080/durable-workflow",
17658            ),
17659            (
17660                "https://runtime.example.test/namespaces/orders",
17661                "https://runtime.example.test/namespaces/orders",
17662            ),
17663            (
17664                "https://runtime.example.test/gateway/api/namespaces/orders",
17665                "https://runtime.example.test/gateway/api/namespaces/orders",
17666            ),
17667            (
17668                "https://api.example.test/runtime/orders/",
17669                "https://api.example.test/runtime/orders",
17670            ),
17671        ] {
17672            let client = Client::builder(base_url)
17673                .build()
17674                .expect("Server and Cloud runtime base URL must remain valid");
17675
17676            assert_eq!(client.base_url, expected);
17677        }
17678    }
17679
17680    #[test]
17681    fn workflow_completion_uses_the_additive_command_protocol_floor() {
17682        assert_eq!(
17683            workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
17684            WORKER_PROTOCOL_VERSION
17685        );
17686        assert_eq!(
17687            workflow_completion_protocol_version(&[json!({
17688                "type": "upsert_search_attributes",
17689                "attributes": {"OrderStatus": "waiting"},
17690            })]),
17691            SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
17692        );
17693        assert_eq!(
17694            workflow_completion_protocol_version(&[json!({
17695                "type": "upsert_search_attributes",
17696                "attributes": {"OrderStatus": "waiting"},
17697                "attribute_types": {"OrderStatus": "keyword"},
17698            })]),
17699            TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
17700        );
17701        assert_eq!(
17702            workflow_completion_protocol_version(&[
17703                json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
17704                json!({"type": "open_condition_wait", "condition_key": "ready"}),
17705            ]),
17706            MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
17707        );
17708        assert_eq!(
17709            workflow_completion_protocol_version(&[
17710                json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
17711                json!({"type": "open_condition_wait", "condition_key": "ready"}),
17712            ]),
17713            CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
17714        );
17715        assert_eq!(
17716            workflow_completion_protocol_version(&[json!({
17717                "type": "open_condition_wait",
17718                "condition_wait_occurrence_id": "rust:condition-wait:0",
17719                "condition_key": "ready",
17720            })]),
17721            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
17722        );
17723        assert_eq!(
17724            workflow_completion_protocol_version_with_message_streams(
17725                &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
17726                true,
17727            ),
17728            MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
17729        );
17730        assert_eq!(
17731            workflow_completion_protocol_version_with_message_streams(
17732                &[json!({
17733                    "type": "open_condition_wait",
17734                    "condition_wait_occurrence_id": "rust:condition-wait:0",
17735                    "condition_key": "ready",
17736                })],
17737                true,
17738            ),
17739            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
17740        );
17741    }
17742
17743    #[test]
17744    fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
17745        let manifest = portable_worker_affinity_capability_manifest();
17746
17747        for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
17748            assert_eq!(manifest[capability]["supported"], json!(false));
17749            assert_eq!(
17750                manifest[capability]["minimum_protocol_version"],
17751                json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
17752            );
17753            assert!(manifest[capability]["reason"]
17754                .as_str()
17755                .is_some_and(|reason| !reason.is_empty()));
17756        }
17757    }
17758
17759    fn typed_fidelity_probe() -> AvroValue {
17760        AvroValue::Map(BTreeMap::from([
17761            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
17762            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
17763            (
17764                "numeric".to_string(),
17765                AvroValue::Map(BTreeMap::from([
17766                    ("0".to_string(), AvroValue::String("zero".to_string())),
17767                    ("1".to_string(), AvroValue::String("one".to_string())),
17768                ])),
17769            ),
17770            (
17771                "nested".to_string(),
17772                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
17773                    "enabled".to_string(),
17774                    AvroValue::Boolean(true),
17775                )]))]),
17776            ),
17777            (
17778                "projection_collisions".to_string(),
17779                AvroValue::Array(projection_collision_probe()),
17780            ),
17781        ]))
17782    }
17783
17784    fn projection_collision_probe() -> Vec<AvroValue> {
17785        vec![
17786            AvroValue::Map(BTreeMap::from([
17787                ("$type".to_string(), AvroValue::String("bytes".to_string())),
17788                (
17789                    "base64".to_string(),
17790                    AvroValue::String("ordinary user text".to_string()),
17791                ),
17792            ])),
17793            AvroValue::Map(BTreeMap::from([
17794                ("$type".to_string(), AvroValue::String("map".to_string())),
17795                (
17796                    "entries".to_string(),
17797                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
17798                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
17799                        (
17800                            "value".to_string(),
17801                            AvroValue::String("user map".to_string()),
17802                        ),
17803                    ]))]),
17804                ),
17805            ])),
17806        ]
17807    }
17808
17809    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
17810    struct TypedContract {
17811        nested: TypedNested,
17812        mode: TypedMode,
17813        optional: Option<String>,
17814        absent: Option<String>,
17815        items: Vec<i64>,
17816        labels: BTreeMap<String, String>,
17817        bytes: serde_bytes::ByteBuf,
17818        signed: i64,
17819        finite: f64,
17820    }
17821
17822    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
17823    struct TypedNested {
17824        enabled: bool,
17825    }
17826
17827    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
17828    enum TypedMode {
17829        Detailed { label: String },
17830    }
17831
17832    fn typed_contract() -> TypedContract {
17833        TypedContract {
17834            nested: TypedNested { enabled: true },
17835            mode: TypedMode::Detailed {
17836                label: "compiler-checked".to_string(),
17837            },
17838            optional: Some("present".to_string()),
17839            absent: None,
17840            items: vec![i64::MIN, 0, i64::MAX],
17841            labels: BTreeMap::from([
17842                ("language".to_string(), "rust".to_string()),
17843                ("wire".to_string(), "avro".to_string()),
17844            ]),
17845            bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
17846            signed: -9_223_372_036_854_775_000,
17847            finite: 12.5,
17848        }
17849    }
17850
17851    #[derive(Clone, Debug, Default, PartialEq)]
17852    struct ReplayCounterState {
17853        loaded: Option<String>,
17854        count: i64,
17855        finished: bool,
17856    }
17857
17858    fn replay_counter_worker() -> Worker {
17859        let client = Client::new("http://127.0.0.1:8080").expect("client");
17860        let mut worker = Worker::new(client, "rust-workers");
17861        worker.register_replayed_workflow(
17862            "replay-counter",
17863            ReplayCounterState::default,
17864            |ctx, _input, state| async move {
17865                let loaded = ctx.activity("load-counter", json!([])).await?;
17866                state.update(|current| {
17867                    current.loaded = loaded.as_str().map(str::to_string);
17868                })?;
17869                for _ in 0..2 {
17870                    let signal = ctx.wait_signal("increment").await?;
17871                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
17872                    state.update(|current| current.count += amount)?;
17873                }
17874                state.update(|current| current.finished = true)?;
17875                state.read(|current| Ok(json!(current.count)))?
17876            },
17877        );
17878        worker.register_replayed_query::<ReplayCounterState, _, _>(
17879            "replay-counter",
17880            "current",
17881            |_ctx, state, _args| async move {
17882                Ok(json!({
17883                    "loaded": state.loaded,
17884                    "count": state.count,
17885                    "finished": state.finished,
17886                }))
17887            },
17888        );
17889        worker.register_replayed_query::<ReplayCounterState, _, _>(
17890            "replay-counter",
17891            "detached-mutation",
17892            |_ctx, state, _args| async move {
17893                let mut detached = (*state).clone();
17894                detached.count = 999;
17895                Ok(json!(detached.count))
17896            },
17897        );
17898        worker.register_replayed_query::<ReplayCounterState, _, _>(
17899            "replay-counter",
17900            "failed-mutation",
17901            |_ctx, state, _args| async move {
17902                let mut detached = (*state).clone();
17903                detached.count = 999;
17904                Err(Error::WorkerLoop("query refused".to_string()))
17905            },
17906        );
17907        worker
17908    }
17909
17910    fn replay_counter_query(
17911        query_name: &str,
17912        history_events: Value,
17913        run_status: &str,
17914    ) -> QueryTask {
17915        let arguments = fixture_envelope(json!([]));
17916        serde_json::from_value(json!({
17917            "query_task_id": format!("query-{query_name}"),
17918            "workflow_type": "replay-counter",
17919            "query_name": query_name,
17920            "payload_codec": DEFAULT_CODEC,
17921            "workflow_arguments": arguments.clone(),
17922            "query_arguments": arguments,
17923            "history_events": history_events,
17924            "run_status": run_status,
17925        }))
17926        .expect("query task")
17927    }
17928
17929    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
17930        workflow_context_with_codec(history, DEFAULT_CODEC)
17931    }
17932
17933    fn workflow_context_with_codec(
17934        history: Vec<HistoryEvent>,
17935        payload_codec: &str,
17936    ) -> WorkflowContext {
17937        WorkflowContext {
17938            cancellation_scope_id: "root".into(),
17939            state: Arc::new(Mutex::new(
17940                WorkflowState::new_with_identity(
17941                    history,
17942                    None,
17943                    None,
17944                    "rust-workers".to_string(),
17945                    payload_codec.to_string(),
17946                    None,
17947                )
17948                .expect("valid workflow history"),
17949            )),
17950        }
17951    }
17952
17953    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
17954        HistoryEvent {
17955            event_type: event_type.to_string(),
17956            payload,
17957            raw: HashMap::new(),
17958        }
17959    }
17960
17961    fn parallel_path_entry(
17962        kind: &str,
17963        base: u64,
17964        size: usize,
17965        index: usize,
17966    ) -> ParallelGroupMetadata {
17967        parallel_group_entry(base, size, index, kind)
17968    }
17969
17970    fn parallel_history_event(
17971        event_type: &str,
17972        sequence: u64,
17973        identity_field: &str,
17974        identity: &str,
17975        path: Vec<ParallelGroupMetadata>,
17976        result: Option<Value>,
17977    ) -> HistoryEvent {
17978        let mut payload = serde_json::Map::from_iter([
17979            ("sequence".to_string(), json!(sequence)),
17980            (identity_field.to_string(), json!(identity)),
17981        ]);
17982        let inner = path.last().expect("parallel history path");
17983        apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
17984        payload.insert("parallel_group_path".to_string(), json!(path));
17985        if let Some(result) = result {
17986            let field = if event_type == "ChildRunCompleted" {
17987                "result"
17988            } else {
17989                "result"
17990            };
17991            payload.insert(field.to_string(), fixture_envelope(result));
17992            payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
17993        }
17994        history_event(event_type, Value::Object(payload))
17995    }
17996
17997    fn nested_parallel_operations() -> Vec<ParallelOperation> {
17998        vec![
17999            ParallelOperation::activity("first", json!([])),
18000            ParallelOperation::group(vec![
18001                ParallelOperation::child_workflow(
18002                    "second",
18003                    ChildWorkflowOptions::new("child-workers"),
18004                    json!([]),
18005                ),
18006                ParallelOperation::activity("third", json!([])),
18007            ]),
18008        ]
18009    }
18010
18011    fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
18012        let outer = [
18013            parallel_path_entry("mixed", 1, 3, 0),
18014            parallel_path_entry("mixed", 1, 3, 1),
18015            parallel_path_entry("mixed", 1, 3, 2),
18016        ];
18017        [
18018            vec![outer[0].clone()],
18019            vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
18020            vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
18021        ]
18022    }
18023
18024    #[test]
18025    fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
18026        let ctx = workflow_context(Vec::new());
18027        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
18028        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18029
18030        assert!(matches!(
18031            call.as_mut().poll(&mut task_context),
18032            Poll::Pending
18033        ));
18034        let commands = ctx.take_commands().expect("parallel commands");
18035        assert_eq!(
18036            commands
18037                .iter()
18038                .map(|command| command["type"].as_str().unwrap_or_default())
18039                .collect::<Vec<_>>(),
18040            [
18041                "schedule_activity",
18042                "start_child_workflow",
18043                "schedule_activity"
18044            ]
18045        );
18046        let paths = nested_parallel_paths();
18047        for (command, path) in commands.iter().zip(paths) {
18048            assert_eq!(command["parallel_group_path"], json!(path));
18049            assert_eq!(
18050                command["parallel_group_id"],
18051                json!(path.last().expect("inner group").parallel_group_id)
18052            );
18053        }
18054    }
18055
18056    fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
18057        let paths = nested_parallel_paths();
18058        let third = parallel_history_event(
18059            "ActivityCompleted",
18060            3,
18061            "activity_type",
18062            "third",
18063            paths[2].clone(),
18064            Some(json!("three")),
18065        );
18066        vec![
18067            parallel_history_event(
18068                "ActivityCompleted",
18069                1,
18070                "activity_type",
18071                "first",
18072                paths[0].clone(),
18073                Some(json!("one")),
18074            ),
18075            parallel_history_event(
18076                "ChildWorkflowScheduled",
18077                2,
18078                "child_workflow_type",
18079                "second",
18080                paths[1].clone(),
18081                None,
18082            ),
18083            parallel_history_event(
18084                "ChildRunCompleted",
18085                2,
18086                "child_workflow_type",
18087                "second",
18088                paths[1].clone(),
18089                Some(json!("two")),
18090            ),
18091            third.clone(),
18092            third,
18093        ]
18094    }
18095
18096    #[test]
18097    fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
18098        for _restart_or_completed_replay in 0..2 {
18099            let ctx = workflow_context(completed_nested_parallel_history());
18100            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
18101            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18102            let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
18103                panic!("completed nested parallel history must replay");
18104            };
18105            assert_eq!(
18106                results,
18107                vec![
18108                    ParallelResult::Activity(json!("one")),
18109                    ParallelResult::Group(vec![
18110                        ParallelResult::ChildWorkflow(ChildWorkflowResult {
18111                            parent: WorkflowIdentity {
18112                                workflow_id: None,
18113                                run_id: None,
18114                            },
18115                            child: WorkflowIdentity {
18116                                workflow_id: None,
18117                                run_id: None,
18118                            },
18119                            child_workflow_type: Some("second".to_string()),
18120                            result: json!("two"),
18121                        }),
18122                        ParallelResult::Activity(json!("three")),
18123                    ]),
18124                ]
18125            );
18126            assert!(ctx.take_commands().expect("commands").is_empty());
18127            ctx.ensure_history_consumed().expect("history consumed");
18128        }
18129    }
18130
18131    #[test]
18132    fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
18133        let paths = nested_parallel_paths();
18134        let history = vec![
18135            parallel_history_event(
18136                "ActivityCompleted",
18137                1,
18138                "activity_type",
18139                "first",
18140                paths[0].clone(),
18141                Some(json!("one")),
18142            ),
18143            parallel_history_event(
18144                "ChildWorkflowScheduled",
18145                2,
18146                "child_workflow_type",
18147                "second",
18148                paths[1].clone(),
18149                None,
18150            ),
18151            parallel_history_event(
18152                "ChildRunFailed",
18153                2,
18154                "child_workflow_type",
18155                "second",
18156                paths[1].clone(),
18157                None,
18158            ),
18159            parallel_history_event(
18160                "ActivityCompleted",
18161                3,
18162                "activity_type",
18163                "third",
18164                paths[2].clone(),
18165                Some(json!("late")),
18166            ),
18167        ];
18168        let ctx = workflow_context(history);
18169        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
18170        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18171        let outcome = call.as_mut().poll(&mut task_context);
18172        let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
18173            panic!("one failed child must return a typed partial failure: {outcome:?}");
18174        };
18175        assert_eq!(failure.member_path, [1, 0]);
18176        assert_eq!(failure.group_id, "parallel-calls:1:3");
18177        assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
18178        assert_eq!(
18179            failure
18180                .completed
18181                .iter()
18182                .map(|completion| completion.member_path.clone())
18183                .collect::<Vec<_>>(),
18184            [vec![0], vec![1, 1]]
18185        );
18186    }
18187
18188    #[test]
18189    fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
18190        let paths = nested_parallel_paths();
18191        let history = vec![
18192            parallel_history_event(
18193                "ActivityScheduled",
18194                1,
18195                "activity_type",
18196                "first",
18197                paths[0].clone(),
18198                None,
18199            ),
18200            parallel_history_event(
18201                "ChildWorkflowScheduled",
18202                2,
18203                "child_workflow_type",
18204                "second",
18205                paths[1].clone(),
18206                None,
18207            ),
18208            parallel_history_event(
18209                "ActivityScheduled",
18210                3,
18211                "activity_type",
18212                "third",
18213                paths[2].clone(),
18214                None,
18215            ),
18216        ];
18217        for _restart in 0..2 {
18218            let ctx = workflow_context(history.clone());
18219            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
18220            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18221            let outcome = call.as_mut().poll(&mut task_context);
18222            assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
18223            assert!(ctx.take_commands().expect("commands").is_empty());
18224        }
18225    }
18226
18227    fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
18228        vec![selection_group_entry(
18229            1,
18230            2,
18231            index,
18232            "activity",
18233            &SelectionMemberMetadata {
18234                key: SelectionKey::Name(key.to_string()),
18235                index,
18236                base_sequence: index as u64 + 1,
18237                size: 1,
18238                kind: "activity".to_string(),
18239            },
18240        )]
18241    }
18242
18243    fn selection_activity_event(
18244        event_type: &str,
18245        index: usize,
18246        key: &str,
18247        result: Option<Value>,
18248    ) -> HistoryEvent {
18249        let sequence = index as u64 + 1;
18250        let mut event = parallel_history_event(
18251            event_type,
18252            sequence,
18253            "activity_type",
18254            &format!("{key}-activity"),
18255            selection_path(index, key),
18256            result,
18257        );
18258        event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
18259        event.raw.insert(
18260            "id".to_string(),
18261            json!(if event_type == "ActivityCompleted" {
18262                format!("event-{key}")
18263            } else {
18264                format!("{event_type}-{key}")
18265            }),
18266        );
18267        event
18268    }
18269
18270    fn selection_winner_marker() -> HistoryEvent {
18271        history_event(
18272            "SelectionResolved",
18273            json!({
18274                "selection_group_id": "select-calls:1:2",
18275                "selection_group_base_sequence": 1,
18276                "selection_group_size": 2,
18277                "member_key": "fast",
18278                "member_index": 1,
18279                "member_base_sequence": 2,
18280                "member_size": 1,
18281                "operation_kind": "activity",
18282                "operation_identity": "activity-fast",
18283                "outcome": "completed",
18284                "resolution_event_id": "event-fast",
18285                "resolution_event_type": "ActivityCompleted",
18286            }),
18287        )
18288    }
18289
18290    fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
18291        ctx.select_keyed(vec![
18292            (
18293                "slow",
18294                ParallelOperation::activity_with_options(
18295                    "slow-activity",
18296                    ActivityOptions::new().task_queue("default"),
18297                    json!([]),
18298                ),
18299            ),
18300            (
18301                "fast",
18302                ParallelOperation::activity_with_options(
18303                    "fast-activity",
18304                    ActivityOptions::new().task_queue("default"),
18305                    json!([]),
18306                ),
18307            ),
18308        ])
18309    }
18310
18311    fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
18312        let ctx = workflow_context(history);
18313        let mut call = Box::pin(keyed_activity_selection(&ctx));
18314        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18315        let selected = match call.as_mut().poll(&mut task_context) {
18316            Poll::Ready(Ok(selected)) => selected,
18317            Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
18318            Poll::Pending => panic!("persisted selection winner must replay without pending"),
18319        };
18320        assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
18321        assert_eq!(
18322            selected.value,
18323            Some(ParallelResult::Activity(json!("winner-value")))
18324        );
18325        let slow = selected
18326            .handle(&SelectionKey::Name("slow".to_string()))
18327            .expect("slow handle")
18328            .clone();
18329        let mut await_slow = Box::pin(slow.await_result());
18330        assert!(matches!(
18331            await_slow.as_mut().poll(&mut task_context),
18332            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
18333        ));
18334        assert!(ctx.take_commands().expect("commands").is_empty());
18335    }
18336
18337    const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
18338
18339    fn canonical_selection_history() -> Vec<HistoryEvent> {
18340        const FIXTURE: &[u8] =
18341            include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
18342        assert_eq!(
18343            format!("{:x}", Sha256::digest(FIXTURE)),
18344            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
18345        );
18346        let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
18347
18348        serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
18349    }
18350
18351    #[test]
18352    fn selection_fresh_process_entrypoint() {
18353        let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
18354            return;
18355        };
18356        let persisted = fs::read(path).expect("persisted selection history");
18357        assert_eq!(
18358            format!("{:x}", Sha256::digest(&persisted)),
18359            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
18360        );
18361        let fixture: Value =
18362            serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
18363        let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
18364            .expect("valid persisted selection history");
18365
18366        assert_persisted_selection_replay(history);
18367    }
18368
18369    #[test]
18370    fn selection_starts_every_member_with_stable_keys_and_group_identity() {
18371        let ctx = workflow_context(Vec::new());
18372        let mut call = Box::pin(keyed_activity_selection(&ctx));
18373        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18374
18375        assert!(matches!(
18376            call.as_mut().poll(&mut task_context),
18377            Poll::Pending
18378        ));
18379        let commands = ctx.take_commands().expect("selection commands");
18380        assert_eq!(commands.len(), 2);
18381        assert_eq!(commands[0]["selection_member_key"], json!("slow"));
18382        assert_eq!(commands[1]["selection_member_key"], json!("fast"));
18383        assert!(commands.iter().all(|command| {
18384            command["parallel_group_id"] == json!("select-calls:1:2")
18385                && command["parallel_group_mode"] == json!("select")
18386        }));
18387    }
18388
18389    #[test]
18390    fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
18391        let ctx = workflow_context(Vec::new());
18392        let mut invalid = Box::pin(ctx.select_keyed(vec![(
18393            "",
18394            ParallelOperation::activity("invalid", json!([])),
18395        )]));
18396        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18397        assert!(matches!(
18398            invalid.as_mut().poll(&mut task_context),
18399            Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
18400                reason: "selection_key_invalid",
18401                ..
18402            })))
18403        ));
18404
18405        for invalid_key in [json!(""), json!(-1)] {
18406            let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
18407            event.payload["selection_member_key"] = invalid_key.clone();
18408            event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
18409            assert!(matches!(
18410                WorkflowState::new_with_identity(
18411                    vec![event],
18412                    None,
18413                    None,
18414                    "rust-workers".to_string(),
18415                    DEFAULT_CODEC.to_string(),
18416                    None,
18417                ),
18418                Err(Error::NonDeterministicReplay(_))
18419            ));
18420        }
18421    }
18422
18423    #[test]
18424    fn selection_preserves_valid_named_and_numeric_keys() {
18425        let ctx = workflow_context(Vec::new());
18426        let mut selection = Box::pin(ctx.select_keyed(vec![
18427            (
18428                SelectionKey::Index(0),
18429                ParallelOperation::activity("numeric", json!([])),
18430            ),
18431            (
18432                SelectionKey::Name("named".to_string()),
18433                ParallelOperation::timer(Duration::from_secs(1)),
18434            ),
18435        ]));
18436        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18437
18438        assert!(matches!(
18439            selection.as_mut().poll(&mut task_context),
18440            Poll::Pending
18441        ));
18442        let commands = ctx.take_commands().expect("selection commands");
18443        assert_eq!(commands[0]["selection_member_key"], json!(0));
18444        assert_eq!(commands[1]["selection_member_key"], json!("named"));
18445    }
18446
18447    #[test]
18448    fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
18449        let history = canonical_selection_history();
18450        assert_persisted_selection_replay(history.clone());
18451
18452        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
18453            .join("tests/fixtures/durable_selection_runtime_history.json");
18454        let output =
18455            ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
18456                .args([
18457                    "--exact",
18458                    "tests::selection_fresh_process_entrypoint",
18459                    "--nocapture",
18460                ])
18461                .env(SELECTION_COLD_REPLAY_HISTORY, &path)
18462                .output()
18463                .expect("run fresh selection replay process");
18464
18465        assert!(
18466            output.status.success(),
18467            "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
18468            String::from_utf8_lossy(&output.stdout),
18469            String::from_utf8_lossy(&output.stderr),
18470        );
18471    }
18472
18473    #[test]
18474    fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
18475        let mut history = canonical_selection_history();
18476        history.retain(|event| event.event_type != "SelectionResolved");
18477        let ctx = workflow_context(history);
18478        let mut selection = Box::pin(keyed_activity_selection(&ctx));
18479        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18480
18481        assert!(matches!(
18482            selection.as_mut().poll(&mut task_context),
18483            Poll::Pending
18484        ));
18485        assert!(ctx.take_commands().expect("commands").is_empty());
18486        assert!(
18487            ctx.matched_recorded_pending()
18488                .expect("selection pending state"),
18489            "terminal member history must keep the workflow durably pending until SelectionResolved commits"
18490        );
18491    }
18492
18493    #[test]
18494    fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
18495        for (terminal_event, predicate_satisfied, timeout_seconds) in [
18496            ("ConditionWaitSatisfied", true, None),
18497            ("ConditionWaitTimedOut", false, Some(0)),
18498        ] {
18499            let member = SelectionMemberMetadata {
18500                key: SelectionKey::Name("condition".to_string()),
18501                index: 0,
18502                base_sequence: 1,
18503                size: 1,
18504                kind: "condition".to_string(),
18505            };
18506            let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
18507            let mut payload = json!({
18508                "sequence": 1,
18509                "condition_wait_id": "condition-1",
18510                "condition_wait_occurrence_id": "rust:condition-wait:0",
18511                "condition_key": "ready",
18512                "condition_definition_fingerprint": "sha256:ready-v1",
18513                "parallel_group_path": path,
18514            });
18515            payload
18516                .as_object_mut()
18517                .expect("condition history payload")
18518                .extend(
18519                    serde_json::to_value(&path[0])
18520                        .expect("condition selection metadata")
18521                        .as_object()
18522                        .expect("condition selection metadata object")
18523                        .clone(),
18524                );
18525            if let Some(timeout_seconds) = timeout_seconds {
18526                payload["timeout_seconds"] = json!(timeout_seconds);
18527            }
18528            let history = vec![
18529                history_event("ConditionWaitOpened", payload.clone()),
18530                history_event(terminal_event, payload),
18531            ];
18532            let ctx = workflow_context(history);
18533            let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
18534            if timeout_seconds.is_some() {
18535                options = options.timeout(Duration::ZERO);
18536            }
18537            let mut selection = Box::pin(ctx.select_keyed(vec![(
18538                "condition",
18539                ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
18540            )]));
18541            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18542
18543            assert!(matches!(
18544                selection.as_mut().poll(&mut task_context),
18545                Poll::Pending
18546            ));
18547            assert!(ctx.take_commands().expect("commands").is_empty());
18548            assert!(
18549                ctx.matched_recorded_pending()
18550                    .expect("condition selection pending state"),
18551                "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
18552            );
18553        }
18554    }
18555
18556    #[test]
18557    fn selection_immediate_condition_members_open_a_durable_wait() {
18558        for predicate_satisfied in [true, false] {
18559            let ctx = workflow_context(Vec::new());
18560            let mut selection = Box::pin(ctx.select_keyed(vec![(
18561                "condition",
18562                ParallelOperation::condition(
18563                    ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
18564                    move || Ok(predicate_satisfied),
18565                ),
18566            )]));
18567            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18568
18569            assert!(matches!(
18570                selection.as_mut().poll(&mut task_context),
18571                Poll::Pending
18572            ));
18573            let commands = ctx.take_commands().expect("condition selection command");
18574            assert_eq!(commands.len(), 1);
18575            assert_eq!(commands[0]["type"], json!("open_condition_wait"));
18576            assert_eq!(commands[0]["timeout_seconds"], json!(0));
18577            assert_eq!(
18578                commands[0]["parallel_group_path"][0]["parallel_group_mode"],
18579                json!("select")
18580            );
18581        }
18582    }
18583
18584    #[test]
18585    fn selection_loser_cancellation_is_explicit_and_idempotent() {
18586        let history = vec![
18587            selection_activity_event("ActivityScheduled", 0, "slow", None),
18588            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
18589            selection_winner_marker(),
18590        ];
18591        let ctx = workflow_context(history.clone());
18592        let mut call = Box::pin(keyed_activity_selection(&ctx));
18593        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18594        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
18595            panic!("winner must replay");
18596        };
18597        let slow = selected
18598            .handle(&SelectionKey::Name("slow".to_string()))
18599            .expect("slow handle")
18600            .clone();
18601        let mut cancel = Box::pin(slow.cancel());
18602        assert!(matches!(
18603            cancel.as_mut().poll(&mut task_context),
18604            Poll::Pending
18605        ));
18606        assert!(matches!(
18607            cancel.as_mut().poll(&mut task_context),
18608            Poll::Pending
18609        ));
18610        let commands = ctx.take_commands().expect("cancel command");
18611        assert_eq!(commands.len(), 1);
18612        assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
18613        assert_eq!(commands[0]["member_key"], json!("slow"));
18614
18615        let mut cancelled_history = history;
18616        cancelled_history.push(history_event(
18617            "SelectionOperationCancelled",
18618            json!({
18619                "selection_group_id": "select-calls:1:2",
18620                "member_key": "slow",
18621                "member_index": 0,
18622                "member_base_sequence": 1,
18623                "member_size": 1,
18624                "operation_kind": "activity",
18625                "operation_identity": "activity-slow",
18626                "cancelled_at": "2026-08-27T00:00:00Z",
18627            }),
18628        ));
18629        let replayed = workflow_context(cancelled_history);
18630        let mut call = Box::pin(keyed_activity_selection(&replayed));
18631        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
18632            panic!("winner must replay after cancellation");
18633        };
18634        let slow = selected
18635            .handle(&SelectionKey::Name("slow".to_string()))
18636            .expect("slow handle")
18637            .clone();
18638        let mut cancel = Box::pin(slow.cancel());
18639        assert!(matches!(
18640            cancel.as_mut().poll(&mut task_context),
18641            Poll::Ready(Ok(()))
18642        ));
18643        assert!(replayed.take_commands().expect("commands").is_empty());
18644    }
18645
18646    #[test]
18647    fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
18648        let base_history = vec![
18649            selection_activity_event("ActivityScheduled", 0, "slow", None),
18650            selection_activity_event("ActivityScheduled", 1, "fast", None),
18651            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
18652            selection_winner_marker(),
18653        ];
18654        for (field, corrupt) in [
18655            ("member_key", json!("fast")),
18656            ("member_index", json!(1)),
18657            ("member_base_sequence", json!(3)),
18658            ("member_size", json!(2)),
18659            ("operation_kind", json!("timer")),
18660            ("operation_identity", json!("forged")),
18661        ] {
18662            let mut cancellation = json!({
18663                "selection_group_id": "select-calls:1:2",
18664                "member_key": "slow",
18665                "member_index": 0,
18666                "member_base_sequence": 1,
18667                "member_size": 1,
18668                "operation_kind": "activity",
18669                "operation_identity": "activity-slow",
18670            });
18671            cancellation[field] = corrupt;
18672            let mut history = base_history.clone();
18673            history.push(history_event("SelectionOperationCancelled", cancellation));
18674            let ctx = workflow_context(history);
18675            let mut selection = Box::pin(keyed_activity_selection(&ctx));
18676            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18677
18678            assert!(matches!(
18679                selection.as_mut().poll(&mut task_context),
18680                Poll::Ready(Err(Error::NonDeterministicReplay(_)))
18681            ));
18682        }
18683    }
18684
18685    #[test]
18686    fn selection_child_identity_prefers_the_durable_run_id() {
18687        let ctx = workflow_context(vec![history_event(
18688            "ChildWorkflowScheduled",
18689            json!({
18690                "sequence": 1,
18691                "child_workflow_type": "child",
18692                "child_workflow_instance_id": "child-instance",
18693                "child_workflow_run_id": "child-run",
18694            }),
18695        )]);
18696        let state = ctx.state.lock().expect("workflow state");
18697
18698        assert_eq!(
18699            selection_operation_identity(&state, "child", 1, 1),
18700            "child-run"
18701        );
18702    }
18703
18704    #[test]
18705    fn selection_activity_identity_requires_canonical_execution_id() {
18706        let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
18707        let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
18708        let mut fast_completed =
18709            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
18710        for event in [&mut fast_open, &mut fast_completed] {
18711            event
18712                .payload
18713                .as_object_mut()
18714                .expect("activity payload")
18715                .remove("activity_execution_id");
18716            event.payload["activity_id"] = json!("forged-activity-id");
18717        }
18718        let mut marker = selection_winner_marker();
18719        marker.payload["operation_identity"] = json!("forged-activity-id");
18720        let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
18721        let mut selection = Box::pin(keyed_activity_selection(&ctx));
18722        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18723
18724        assert!(matches!(
18725            selection.as_mut().poll(&mut task_context),
18726            Poll::Ready(Err(Error::NonDeterministicReplay(_)))
18727        ));
18728    }
18729
18730    #[test]
18731    fn selection_completion_before_cancellation_remains_awaitable() {
18732        let history = vec![
18733            selection_activity_event("ActivityScheduled", 0, "slow", None),
18734            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
18735            selection_winner_marker(),
18736            selection_activity_event(
18737                "ActivityCompleted",
18738                0,
18739                "slow",
18740                Some(json!("completed-first")),
18741            ),
18742        ];
18743        let ctx = workflow_context(history);
18744        let mut selection = Box::pin(keyed_activity_selection(&ctx));
18745        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18746        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
18747            panic!("winner must replay");
18748        };
18749        let slow = selected
18750            .handle(&SelectionKey::Name("slow".to_string()))
18751            .expect("slow handle")
18752            .clone();
18753        let mut cancel = Box::pin(slow.cancel());
18754        assert!(matches!(
18755            cancel.as_mut().poll(&mut task_context),
18756            Poll::Ready(Ok(()))
18757        ));
18758        let mut await_slow = Box::pin(slow.await_result());
18759        assert!(matches!(
18760            await_slow.as_mut().poll(&mut task_context),
18761            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
18762        ));
18763        let commands = ctx.take_commands().expect("commands");
18764        assert!(commands.is_empty());
18765    }
18766
18767    #[test]
18768    fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
18769        let nested_member = SelectionMemberMetadata {
18770            key: SelectionKey::Name("nested".to_string()),
18771            index: 0,
18772            base_sequence: 1,
18773            size: 2,
18774            kind: "group".to_string(),
18775        };
18776        let deadline_member = SelectionMemberMetadata {
18777            key: SelectionKey::Name("deadline".to_string()),
18778            index: 1,
18779            base_sequence: 3,
18780            size: 1,
18781            kind: "timer".to_string(),
18782        };
18783        let nested_paths = [
18784            vec![
18785                selection_group_entry(1, 3, 0, "mixed", &nested_member),
18786                parallel_group_entry(1, 2, 0, "activity"),
18787            ],
18788            vec![
18789                selection_group_entry(1, 3, 1, "mixed", &nested_member),
18790                parallel_group_entry(1, 2, 1, "activity"),
18791            ],
18792        ];
18793        let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
18794        let mut timer_fired = parallel_history_event(
18795            "TimerFired",
18796            3,
18797            "timer_id",
18798            "timer-3",
18799            deadline_path.clone(),
18800            None,
18801        );
18802        timer_fired.payload["delay_seconds"] = json!(0);
18803        timer_fired
18804            .raw
18805            .insert("id".to_string(), json!("timer-fired"));
18806        let mut timer_scheduled = parallel_history_event(
18807            "TimerScheduled",
18808            3,
18809            "timer_id",
18810            "timer-3",
18811            deadline_path,
18812            None,
18813        );
18814        timer_scheduled.payload["delay_seconds"] = json!(0);
18815        let history = vec![
18816            parallel_history_event(
18817                "ActivityScheduled",
18818                1,
18819                "activity_type",
18820                "nested-first",
18821                nested_paths[0].clone(),
18822                None,
18823            ),
18824            parallel_history_event(
18825                "ActivityScheduled",
18826                2,
18827                "activity_type",
18828                "nested-second",
18829                nested_paths[1].clone(),
18830                None,
18831            ),
18832            timer_scheduled,
18833            timer_fired,
18834            history_event(
18835                "SelectionResolved",
18836                json!({
18837                    "selection_group_id": "select-calls:1:3",
18838                    "selection_group_base_sequence": 1,
18839                    "selection_group_size": 3,
18840                    "member_key": "deadline",
18841                    "member_index": 1,
18842                    "member_base_sequence": 3,
18843                    "member_size": 1,
18844                    "operation_kind": "timer",
18845                    "operation_identity": "timer-3",
18846                    "outcome": "completed",
18847                    "resolution_event_id": "timer-fired",
18848                    "resolution_event_type": "TimerFired",
18849                }),
18850            ),
18851            parallel_history_event(
18852                "ActivityFailed",
18853                2,
18854                "activity_type",
18855                "nested-second",
18856                nested_paths[1].clone(),
18857                None,
18858            ),
18859        ];
18860        let ctx = workflow_context(history);
18861        let mut selection = Box::pin(ctx.select_keyed(vec![
18862            (
18863                "nested",
18864                ParallelOperation::group(vec![
18865                    ParallelOperation::activity("nested-first", json!([])),
18866                    ParallelOperation::activity("nested-second", json!([])),
18867                ]),
18868            ),
18869            ("deadline", ParallelOperation::timer(Duration::ZERO)),
18870        ]));
18871        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18872        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
18873            panic!("deadline winner must replay");
18874        };
18875        let nested = selected
18876            .handle(&SelectionKey::Name("nested".to_string()))
18877            .expect("nested handle")
18878            .clone();
18879        let mut cancel = Box::pin(nested.cancel());
18880        assert!(matches!(
18881            cancel.as_mut().poll(&mut task_context),
18882            Poll::Ready(Ok(()))
18883        ));
18884        let mut await_nested = Box::pin(nested.await_result());
18885
18886        assert!(matches!(
18887            await_nested.as_mut().poll(&mut task_context),
18888            Poll::Ready(Err(Error::ActivityFailed(_)))
18889        ));
18890        assert!(ctx.take_commands().expect("commands").is_empty());
18891    }
18892
18893    #[test]
18894    fn selection_supports_child_timer_signal_condition_and_nested_groups() {
18895        let ctx = workflow_context(Vec::new());
18896        let mut call = Box::pin(ctx.select(vec![
18897            ParallelOperation::child_workflow(
18898                "child",
18899                ChildWorkflowOptions::new("children"),
18900                json!([]),
18901            ),
18902            ParallelOperation::timer(Duration::from_secs(30)),
18903            ParallelOperation::signal("approval"),
18904            ParallelOperation::condition(
18905                ConditionWaitOptions::new("ready", "sha256:ready"),
18906                || Ok(false),
18907            ),
18908            ParallelOperation::group(vec![
18909                ParallelOperation::activity("nested-one", json!([])),
18910                ParallelOperation::activity("nested-two", json!([])),
18911            ]),
18912        ]));
18913        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18914        assert!(matches!(
18915            call.as_mut().poll(&mut task_context),
18916            Poll::Pending
18917        ));
18918        let commands = ctx.take_commands().expect("selection commands");
18919        assert_eq!(
18920            commands
18921                .iter()
18922                .map(|command| command["type"].as_str().unwrap_or_default())
18923                .collect::<Vec<_>>(),
18924            [
18925                "start_child_workflow",
18926                "start_timer",
18927                "open_signal_wait",
18928                "open_condition_wait",
18929                "schedule_activity",
18930                "schedule_activity",
18931            ]
18932        );
18933        assert!(commands.iter().all(|command| {
18934            command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
18935        }));
18936        assert_eq!(
18937            commands[4]["parallel_group_path"].as_array().map(Vec::len),
18938            Some(2)
18939        );
18940        assert_eq!(
18941            commands[4]["parallel_group_path"][0]["selection_member_kind"],
18942            json!("group")
18943        );
18944        assert_eq!(
18945            commands[5]["parallel_group_path"][0]["selection_member_kind"],
18946            json!("group")
18947        );
18948
18949        let one_leaf_ctx = workflow_context(Vec::new());
18950        let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
18951            ParallelOperation::activity("nested-only", json!([])),
18952        ])]));
18953        assert!(matches!(
18954            one_leaf.as_mut().poll(&mut task_context),
18955            Poll::Pending
18956        ));
18957        let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
18958        assert_eq!(one_leaf_commands.len(), 1);
18959        assert_eq!(
18960            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
18961            json!("group")
18962        );
18963        assert_eq!(
18964            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
18965            json!(1)
18966        );
18967    }
18968
18969    async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
18970        let mut saga = ctx.saga();
18971        let outcome = async {
18972            let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
18973            saga.add_compensation("trip.cancel-flight", json!([flight]))?;
18974            let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
18975            saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
18976            ctx.activity("trip.charge", json!([])).await?;
18977            Ok(json!({"status": "booked"}))
18978        }
18979        .await;
18980        saga.finish(outcome).await
18981    }
18982
18983    fn saga_activity(
18984        event_type: &str,
18985        sequence: u64,
18986        activity_type: &str,
18987        result: Option<Value>,
18988    ) -> HistoryEvent {
18989        let mut payload = json!({
18990            "sequence": sequence,
18991            "activity_type": activity_type,
18992            "message": format!("{activity_type} failed"),
18993            "exception_type": "PlannedFailure",
18994            "non_retryable": true,
18995        });
18996        if let Some(result) = result {
18997            payload["result"] = fixture_envelope(result);
18998        }
18999        history_event(event_type, payload)
19000    }
19001
19002    #[test]
19003    fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
19004        let completed_hotel_compensation = saga_activity(
19005            "ActivityCompleted",
19006            4,
19007            "trip.cancel-hotel",
19008            Some(Value::Null),
19009        );
19010        let history = vec![
19011            saga_activity(
19012                "ActivityCompleted",
19013                1,
19014                "trip.reserve-flight",
19015                Some(json!("flight-1")),
19016            ),
19017            saga_activity(
19018                "ActivityCompleted",
19019                2,
19020                "trip.reserve-hotel",
19021                Some(json!("hotel-1")),
19022            ),
19023            saga_activity("ActivityFailed", 3, "trip.charge", None),
19024            completed_hotel_compensation.clone(),
19025            completed_hotel_compensation,
19026        ];
19027
19028        for _restart in 0..2 {
19029            let ctx = workflow_context(history.clone());
19030            let mut future = Box::pin(trip_saga(ctx.clone()));
19031            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19032            assert!(matches!(
19033                future.as_mut().poll(&mut task_context),
19034                Poll::Pending
19035            ));
19036            let commands = ctx.take_commands().expect("compensation command");
19037            assert_eq!(commands.len(), 1);
19038            assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
19039        }
19040    }
19041
19042    #[test]
19043    fn saga_compensation_failure_preserves_both_typed_failures() {
19044        let history = vec![
19045            saga_activity(
19046                "ActivityCompleted",
19047                1,
19048                "trip.reserve-flight",
19049                Some(json!("flight-1")),
19050            ),
19051            saga_activity(
19052                "ActivityCompleted",
19053                2,
19054                "trip.reserve-hotel",
19055                Some(json!("hotel-1")),
19056            ),
19057            saga_activity("ActivityFailed", 3, "trip.charge", None),
19058            saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
19059        ];
19060        let ctx = workflow_context(history);
19061        let mut future = Box::pin(trip_saga(ctx));
19062        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19063        let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
19064            future.as_mut().poll(&mut task_context)
19065        else {
19066            panic!("compensation failure must remain structured");
19067        };
19068        assert!(matches!(
19069            *failure.initiating_failure,
19070            Error::ActivityFailed(_)
19071        ));
19072        assert!(matches!(
19073            *failure.compensation_failure,
19074            Error::ActivityFailed(_)
19075        ));
19076        assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
19077        assert_eq!(failure.compensation_registration_order, 2);
19078    }
19079
19080    #[test]
19081    fn saga_compensates_cooperative_cancellation() {
19082        let ctx = workflow_context(vec![saga_activity(
19083            "ActivityCompleted",
19084            1,
19085            "trip.reserve-flight",
19086            Some(json!("flight-1")),
19087        )]);
19088        ctx.state.lock().expect("state").cancel_requested = true;
19089        let run = {
19090            let ctx = ctx.clone();
19091            async move {
19092                let mut saga = ctx.saga();
19093                let outcome = async {
19094                    let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
19095                    saga.add_compensation("trip.cancel-flight", json!([flight]))?;
19096                    ctx.throw_if_cancellation_requested()?;
19097                    Ok(json!("unexpected"))
19098                }
19099                .await;
19100                saga.finish(outcome).await
19101            }
19102        };
19103        let mut future = Box::pin(run);
19104        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19105        assert!(matches!(
19106            future.as_mut().poll(&mut task_context),
19107            Poll::Pending
19108        ));
19109        let commands = ctx.take_commands().expect("cancellation compensation");
19110        assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
19111    }
19112
19113    fn workflow_task(
19114        workflow_type: &str,
19115        history_events: Vec<HistoryEvent>,
19116        payload_codec: &str,
19117    ) -> WorkflowTask {
19118        WorkflowTask {
19119            task_id: format!("wft-{workflow_type}"),
19120            workflow_command_id: None,
19121            workflow_id: Some(format!("wf-{workflow_type}")),
19122            run_id: Some(format!("run-{workflow_type}")),
19123            workflow_type: workflow_type.to_string(),
19124            cancel_requested: false,
19125            payload_codec: payload_codec.to_string(),
19126            arguments: Some(
19127                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
19128            ),
19129            total_history_events: Some(history_events.len() as u64),
19130            history_size_bytes: None,
19131            continue_as_new_recommended: None,
19132            history_budget_pressure: None,
19133            history_events,
19134            next_history_page_token: None,
19135            workflow_task_attempt: 1,
19136            workflow_signal_id: None,
19137            signal_name: None,
19138            signal_arguments: None,
19139            workflow_update_id: None,
19140            update_name: None,
19141            lease_owner: Some("rust-worker".to_string()),
19142        }
19143    }
19144
19145    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
19146    struct SideEffectProbe {
19147        request_id: String,
19148        attempt: u32,
19149    }
19150
19151    #[test]
19152    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
19153        let calls = AtomicUsize::new(0);
19154        let ctx = workflow_context(Vec::new());
19155        let value = ctx
19156            .side_effect(|| {
19157                calls.fetch_add(1, Ordering::SeqCst);
19158                SideEffectProbe {
19159                    request_id: "request-42".to_string(),
19160                    attempt: 3,
19161                }
19162            })
19163            .expect("first side effect");
19164        assert_eq!(value.attempt, 3);
19165        assert_eq!(calls.load(Ordering::SeqCst), 1);
19166        let commands = ctx.take_commands().expect("commands");
19167        assert_eq!(commands.len(), 1);
19168        assert_eq!(commands[0]["type"], "record_side_effect");
19169        assert_eq!(
19170            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
19171            serde_json::to_value(&value).expect("value")
19172        );
19173
19174        let replay = workflow_context(vec![history_event(
19175            "SideEffectRecorded",
19176            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
19177        )]);
19178        let replayed: SideEffectProbe = replay
19179            .side_effect(|| {
19180                calls.fetch_add(1, Ordering::SeqCst);
19181                panic!("committed side-effect callbacks must not run during replay")
19182            })
19183            .expect("replayed side effect");
19184        assert_eq!(replayed, value);
19185        assert_eq!(calls.load(Ordering::SeqCst), 1);
19186        assert!(replay.take_commands().expect("commands").is_empty());
19187        replay.ensure_history_consumed().expect("history consumed");
19188    }
19189
19190    #[test]
19191    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
19192        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
19193        let value = ctx
19194            .side_effect(|| SideEffectProbe {
19195                request_id: "avro-request".to_string(),
19196                attempt: 1,
19197            })
19198            .expect("Avro side effect");
19199        let uuid = ctx.uuid_v4().expect("deterministic UUID");
19200        let commands = ctx.take_commands().expect("commands");
19201        assert_eq!(commands.len(), 2);
19202        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
19203        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
19204        assert_eq!(
19205            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
19206            serde_json::to_value(&value).expect("value")
19207        );
19208
19209        let replay = workflow_context_with_codec(
19210            vec![
19211                history_event(
19212                    "SideEffectRecorded",
19213                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
19214                ),
19215                history_event(
19216                    "SideEffectRecorded",
19217                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
19218                ),
19219            ],
19220            DEFAULT_CODEC,
19221        );
19222        let replayed: SideEffectProbe = replay
19223            .side_effect(|| panic!("Avro callback must not run"))
19224            .expect("replayed Avro value");
19225        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
19226        assert_eq!(replayed, value);
19227        assert_eq!(replayed_uuid, uuid);
19228        assert!(replay.take_commands().expect("commands").is_empty());
19229    }
19230
19231    #[test]
19232    fn typed_side_effect_replay_preserves_bytes_and_maps() {
19233        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
19234        let value = ctx
19235            .side_effect_avro_value(typed_fidelity_probe)
19236            .expect("typed side effect");
19237        let commands = ctx.take_commands().expect("side-effect command");
19238        assert_eq!(
19239            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
19240                .expect("recorded side effect"),
19241            value
19242        );
19243
19244        let replay = workflow_context_with_codec(
19245            vec![history_event(
19246                "SideEffectRecorded",
19247                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
19248            )],
19249            DEFAULT_CODEC,
19250        );
19251        assert_eq!(
19252            replay
19253                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
19254                .expect("replayed typed side effect"),
19255            value
19256        );
19257    }
19258
19259    #[test]
19260    fn ordered_side_effects_share_the_durable_command_stream() {
19261        let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
19262        let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
19263        let ctx = workflow_context(vec![
19264            history_event(
19265                "SideEffectRecorded",
19266                json!({"sequence": 1, "result": first}),
19267            ),
19268            history_event(
19269                "SideEffectRecorded",
19270                json!({"sequence": 2, "result": second}),
19271            ),
19272        ]);
19273        let first: String = ctx
19274            .side_effect(|| panic!("first callback must not run"))
19275            .expect("first replay");
19276        let second: i32 = ctx
19277            .side_effect(|| panic!("second callback must not run"))
19278            .expect("second replay");
19279        assert_eq!(first, "first");
19280        assert_eq!(second, 29);
19281        ctx.ensure_history_consumed().expect("ordered history");
19282
19283        let reordered = workflow_context(vec![history_event(
19284            "VersionMarkerRecorded",
19285            json!({
19286                "sequence": 1,
19287                "change_id": "before-side-effect",
19288                "version": 1,
19289                "min_supported": 1,
19290                "max_supported": 1,
19291            }),
19292        )]);
19293        let error = reordered
19294            .side_effect(|| "new".to_string())
19295            .expect_err("command reordering must fail");
19296        assert!(matches!(
19297            error,
19298            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19299                if reason == "recorded_command_mismatch"
19300        ));
19301    }
19302
19303    #[test]
19304    fn legacy_patch_preserves_pending_and_completed_activity_history() {
19305        for completed in [false, true] {
19306            let mut history = vec![history_event(
19307                "ActivityScheduled",
19308                json!({"sequence": 1, "activity_type": "old"}),
19309            )];
19310            if completed {
19311                history.push(history_event(
19312                    "ActivityCompleted",
19313                    json!({
19314                        "sequence": 1,
19315                        "activity_type": "old",
19316                        "result": encode_value_envelope(&json!(41), DEFAULT_CODEC).unwrap(),
19317                    }),
19318                ));
19319            }
19320            for _cold_replay in 0..2 {
19321                let ctx = workflow_context(history.clone());
19322                assert!(!ctx.patched("added-step").expect("legacy patch decision"));
19323                assert!(!ctx.patched("added-step").expect("same frozen decision"));
19324                ctx.deprecate_patch("added-step")
19325                    .expect("retained boundary");
19326                let mut activity = Box::pin(ctx.activity("old", json!([])));
19327                let mut context = TaskContext::from_waker(noop_waker_ref());
19328                match activity.as_mut().poll(&mut context) {
19329                    Poll::Ready(result) => {
19330                        assert!(completed);
19331                        assert_eq!(result.expect("recorded result"), json!(41));
19332                        ctx.ensure_history_consumed().expect("old command retained");
19333                    }
19334                    Poll::Pending => assert!(!completed),
19335                }
19336                assert!(ctx.take_commands().expect("commands").is_empty());
19337            }
19338        }
19339    }
19340
19341    #[test]
19342    fn legacy_patch_preserves_an_unmarked_completed_workflow() {
19343        let ctx = workflow_context(vec![history_event("WorkflowCompleted", json!({}))]);
19344        assert!(!ctx.patched("added-step").expect("legacy completed run"));
19345        ctx.deprecate_patch("added-step")
19346            .expect("same legacy decision");
19347        assert!(ctx.take_commands().expect("commands").is_empty());
19348        ctx.ensure_history_consumed().expect("completed history");
19349    }
19350
19351    #[test]
19352    fn legacy_patch_rejects_a_range_without_the_original_branch() {
19353        let ctx = workflow_context(vec![history_event(
19354            "ActivityScheduled",
19355            json!({"sequence": 1, "activity_type": "old"}),
19356        )]);
19357        let error = ctx
19358            .get_version("added-step", 0, 1)
19359            .expect_err("legacy is unsupported");
19360        assert!(matches!(
19361            error,
19362            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19363                if reason == "version_marker_incompatible_range"
19364        ));
19365        assert!(!ctx
19366            .patched("supported-step")
19367            .expect("failed check did not consume history"));
19368        assert!(ctx.take_commands().expect("commands").is_empty());
19369    }
19370
19371    fn retained_patch_marker(sequence: u64, version: i32, time: &str) -> HistoryEvent {
19372        let mut event = history_event(
19373            "VersionMarkerRecorded",
19374            json!({
19375                "sequence": sequence,
19376                "change_id": "retained-patch",
19377                "version": version,
19378                "min_supported": -1,
19379                "max_supported": 1,
19380            }),
19381        );
19382        event.raw.insert("timestamp".into(), json!(time));
19383        event
19384    }
19385
19386    #[test]
19387    fn retained_patch_markers_preserve_each_original_boundary() {
19388        for completed in [false, true] {
19389            let mut history = vec![
19390                retained_patch_marker(1, 1, "2026-10-10T21:47:29.120756Z"),
19391                retained_patch_marker(2, 1, "2026-10-10T21:47:29.123604Z"),
19392                history_event(
19393                    "ActivityScheduled",
19394                    json!({"sequence": 3, "activity_type": "old"}),
19395                ),
19396            ];
19397            if completed {
19398                history.push(history_event(
19399                    "ActivityCompleted",
19400                    json!({
19401                        "sequence": 3,
19402                        "activity_type": "old",
19403                        "result": fixture_envelope(json!(41)),
19404                    }),
19405                ));
19406            }
19407            for _cold_replay in 0..2 {
19408                let ctx = workflow_context(history.clone());
19409                assert!(ctx.patched("retained-patch").unwrap());
19410                assert_eq!(ctx.state.lock().unwrap().command_cursor, 1);
19411                assert!(ctx.patched("retained-patch").unwrap());
19412                assert_eq!(ctx.state.lock().unwrap().command_cursor, 2);
19413                ctx.deprecate_patch("retained-patch").unwrap();
19414                assert_eq!(ctx.state.lock().unwrap().command_cursor, 2);
19415                let mut activity = Box::pin(ctx.activity("old", json!([])));
19416                let mut context = TaskContext::from_waker(noop_waker_ref());
19417                match activity.as_mut().poll(&mut context) {
19418                    Poll::Ready(result) => {
19419                        assert!(completed);
19420                        assert_eq!(result.unwrap(), json!(41));
19421                        ctx.ensure_history_consumed().unwrap();
19422                    }
19423                    Poll::Pending => assert!(!completed),
19424                }
19425                assert!(ctx.take_commands().unwrap().is_empty());
19426                let state = ctx.state.lock().unwrap();
19427                for (actual, original) in state.history_events.iter().zip(&history) {
19428                    assert_eq!(actual.payload, original.payload);
19429                    assert_eq!(actual.raw, original.raw);
19430                }
19431            }
19432        }
19433    }
19434
19435    #[test]
19436    fn retained_patch_aliases_do_not_skip_intervening_operations() {
19437        let ctx = workflow_context(vec![
19438            retained_patch_marker(1, 1, "2026-10-10T21:47:29.120756Z"),
19439            history_event(
19440                "SideEffectRecorded",
19441                json!({
19442                    "sequence": 2, "result": fixture_envelope(json!("between")),
19443                }),
19444            ),
19445            retained_patch_marker(3, 1, "2026-10-10T21:47:29.123604Z"),
19446            history_event(
19447                "VersionMarkerRecorded",
19448                json!({
19449                    "sequence": 4, "change_id": "other-patch", "version": 1,
19450                    "min_supported": -1, "max_supported": 1,
19451                }),
19452            ),
19453        ]);
19454        assert!(ctx.patched("retained-patch").unwrap());
19455        assert!(ctx.patched("retained-patch").unwrap());
19456        assert_eq!(ctx.state.lock().unwrap().command_cursor, 1);
19457        assert_eq!(
19458            ctx.side_effect(|| "must not execute".to_string()).unwrap(),
19459            "between"
19460        );
19461        ctx.deprecate_patch("retained-patch").unwrap();
19462        assert_eq!(ctx.state.lock().unwrap().command_cursor, 3);
19463        ctx.deprecate_patch("retained-patch").unwrap();
19464        assert_eq!(ctx.state.lock().unwrap().command_cursor, 3);
19465        assert!(ctx.patched("other-patch").unwrap());
19466        ctx.ensure_history_consumed().unwrap();
19467        assert!(ctx.take_commands().unwrap().is_empty());
19468    }
19469
19470    #[test]
19471    fn retained_patch_alias_range_validation_leaves_the_cursor_unchanged() {
19472        let mut alias = retained_patch_marker(2, 1, "2026-10-10T21:47:29.123604Z");
19473        alias.payload["max_supported"] = json!(3);
19474        let ctx = workflow_context(vec![
19475            retained_patch_marker(1, 1, "2026-10-10T21:47:29.120756Z"),
19476            alias,
19477        ]);
19478        assert_eq!(ctx.get_version("retained-patch", -1, 2).unwrap(), 1);
19479        let error = ctx.get_version("retained-patch", 2, 3).unwrap_err();
19480        assert!(matches!(error,
19481            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19482                if reason == "version_marker_incompatible_range"));
19483        assert_eq!(ctx.state.lock().unwrap().command_cursor, 1);
19484        assert_eq!(ctx.get_version("retained-patch", -1, 3).unwrap(), 1);
19485        ctx.ensure_history_consumed().unwrap();
19486        assert!(ctx.take_commands().unwrap().is_empty());
19487    }
19488
19489    #[test]
19490    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
19491        let ctx = workflow_context(Vec::new());
19492        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
19493        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
19494        assert!(ctx.patched("new-search").expect("patch"));
19495        ctx.deprecate_patch("new-search").expect("deprecate patch");
19496        let commands = ctx.take_commands().expect("commands");
19497        assert_eq!(commands.len(), 2);
19498        assert_eq!(commands[0]["type"], "record_version_marker");
19499        assert_eq!(commands[0]["version"], 2);
19500        assert_eq!(commands[1]["change_id"], "new-search");
19501
19502        let replay = workflow_context(vec![history_event(
19503            "VersionMarkerRecorded",
19504            json!({
19505                "sequence": 1,
19506                "change_id": "checkout-v2",
19507                "version": 2,
19508                "min_supported": 1,
19509                "max_supported": 2,
19510            }),
19511        )]);
19512        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
19513        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
19514        assert!(replay.take_commands().expect("commands").is_empty());
19515        replay.ensure_history_consumed().expect("history consumed");
19516    }
19517
19518    #[test]
19519    fn version_markers_reject_incompatible_or_malformed_history() {
19520        let incompatible = workflow_context(vec![history_event(
19521            "VersionMarkerRecorded",
19522            json!({
19523                "sequence": 1,
19524                "change_id": "checkout-v2",
19525                "version": 1,
19526                "min_supported": 1,
19527                "max_supported": 2,
19528            }),
19529        )]);
19530        let error = incompatible
19531            .get_version("checkout-v2", 2, 3)
19532            .expect_err("old version is unsupported");
19533        assert!(matches!(
19534            error,
19535            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19536                if reason == "version_marker_incompatible_range"
19537        ));
19538
19539        for (history, reason) in [
19540            (
19541                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
19542                "side_effect_result_missing",
19543            ),
19544            (
19545                vec![history_event(
19546                    "SideEffectRecorded",
19547                    json!({
19548                        "sequence": 1,
19549                        "result": {"codec": "avro", "blob": "not-base64"},
19550                    }),
19551                )],
19552                "side_effect_payload_incompatible",
19553            ),
19554            (
19555                vec![history_event(
19556                    "SideEffectRecorded",
19557                    json!({"sequence": 1, "result": {"unwrapped": true}}),
19558                )],
19559                "side_effect_payload_malformed",
19560            ),
19561            (
19562                vec![history_event(
19563                    "VersionMarkerRecorded",
19564                    json!({
19565                        "sequence": 1,
19566                        "change_id": "change",
19567                        "version": 1,
19568                        "min_supported": 2,
19569                        "max_supported": 1,
19570                    }),
19571                )],
19572                "version_marker_history_range_invalid",
19573            ),
19574        ] {
19575            let error = WorkflowState::new(
19576                history,
19577                "rust-workers".to_string(),
19578                DEFAULT_CODEC.to_string(),
19579                None,
19580            )
19581            .expect_err("malformed history must fail");
19582            assert!(matches!(
19583                error,
19584                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
19585                    if actual == reason
19586            ));
19587        }
19588    }
19589
19590    #[test]
19591    fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
19592        let history = vec![history_event(
19593            "SearchAttributesUpserted",
19594            json!({
19595                "sequence": 1,
19596                "attributes": {"customer_tier": "gold"},
19597                "attribute_types": {"customer_tier": "keyword"},
19598                "merged": {"customer_tier": "gold"}
19599            }),
19600        )];
19601
19602        let matching = workflow_context(history.clone());
19603        matching
19604            .upsert_search_attributes(
19605                SearchAttributeUpdate::new()
19606                    .keyword("customer_tier", "gold")
19607                    .expect("keyword update"),
19608            )
19609            .expect("matching typed update must replay");
19610        matching
19611            .ensure_history_consumed()
19612            .expect("history consumed");
19613
19614        let changed_type = workflow_context(history.clone());
19615        let error = changed_type
19616            .upsert_search_attributes(
19617                SearchAttributeUpdate::new()
19618                    .string("customer_tier", "gold")
19619                    .expect("string update"),
19620            )
19621            .expect_err("same JSON value with a different declaration must be nondeterministic");
19622        let Error::NonDeterministicReplay(failure) = error else {
19623            panic!("typed identity drift must be a replay failure");
19624        };
19625        assert_eq!(failure.reason, "search_attribute_type_mismatch");
19626        assert_eq!(failure.sequence, Some(1));
19627
19628        let changed_value = workflow_context(history);
19629        let error = changed_value
19630            .upsert_search_attributes(
19631                SearchAttributeUpdate::new()
19632                    .keyword("customer_tier", "platinum")
19633                    .expect("keyword update"),
19634            )
19635            .expect_err("changed values must be nondeterministic");
19636        let Error::NonDeterministicReplay(failure) = error else {
19637            panic!("value drift must be a replay failure");
19638        };
19639        assert_eq!(failure.reason, "search_attribute_value_mismatch");
19640    }
19641
19642    #[test]
19643    fn legacy_search_attribute_history_keeps_type_identity_unknown() {
19644        let history = vec![history_event(
19645            "SearchAttributesUpserted",
19646            json!({
19647                "sequence": 1,
19648                "attributes": {"customer_tier": "gold"},
19649                "merged": {"customer_tier": "gold"}
19650            }),
19651        )];
19652
19653        for update in [
19654            SearchAttributeUpdate::new()
19655                .keyword("customer_tier", "gold")
19656                .expect("keyword update"),
19657            SearchAttributeUpdate::new()
19658                .string("customer_tier", "gold")
19659                .expect("string update"),
19660        ] {
19661            let restarted = workflow_context(history.clone());
19662            restarted
19663                .upsert_search_attributes(update)
19664                .expect("legacy history constrains values but has unknown type identity");
19665            restarted
19666                .ensure_history_consumed()
19667                .expect("history consumed");
19668        }
19669    }
19670
19671    #[test]
19672    fn search_attribute_command_emits_canonical_types() {
19673        let ctx = workflow_context(Vec::new());
19674        ctx.upsert_search_attributes(
19675            SearchAttributeUpdate::new()
19676                .keyword("customer_tier", "gold")
19677                .expect("keyword update")
19678                .int("attempts", 3)
19679                .expect("int update")
19680                .delete("obsolete")
19681                .expect("delete update"),
19682        )
19683        .expect("valid search attributes");
19684
19685        assert_eq!(
19686            ctx.take_commands().expect("commands"),
19687            vec![json!({
19688                "type": "upsert_search_attributes",
19689                "attributes": {
19690                    "attempts": 3,
19691                    "customer_tier": "gold",
19692                    "obsolete": null
19693                },
19694                "attribute_types": {
19695                    "attempts": "int",
19696                    "customer_tier": "keyword"
19697                }
19698            })]
19699        );
19700    }
19701
19702    #[test]
19703    fn duplicate_side_effects_and_conflicting_version_markers_are_rejected() {
19704        let duplicate_side_effect = WorkflowState::new(
19705            vec![
19706                history_event(
19707                    "SideEffectRecorded",
19708                    json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
19709                ),
19710                history_event(
19711                    "SideEffectRecorded",
19712                    json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
19713                ),
19714            ],
19715            "rust-workers".to_string(),
19716            DEFAULT_CODEC.to_string(),
19717            None,
19718        )
19719        .expect_err("duplicate side effect");
19720        assert!(matches!(
19721            duplicate_side_effect,
19722            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19723                if reason == "duplicate_side_effect_record"
19724        ));
19725
19726        let marker = |sequence, version| {
19727            history_event(
19728                "VersionMarkerRecorded",
19729                json!({
19730                    "sequence": sequence,
19731                    "change_id": "same-change",
19732                    "version": version,
19733                    "min_supported": -1,
19734                    "max_supported": 1,
19735                }),
19736            )
19737        };
19738        let duplicate_marker = WorkflowState::new(
19739            vec![marker(1, 1), marker(3, -1)],
19740            "rust-workers".to_string(),
19741            DEFAULT_CODEC.to_string(),
19742            None,
19743        )
19744        .expect_err("conflicting marker");
19745        assert!(matches!(
19746            duplicate_marker,
19747            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19748                if reason == "version_marker_conflict"
19749        ));
19750    }
19751
19752    #[test]
19753    fn retained_patch_aliases_reject_duplicate_physical_records_and_invalid_ranges() {
19754        let marker = retained_patch_marker(1, 1, "2026-10-10T21:47:29.120756Z");
19755        let mut invalid_alias = retained_patch_marker(2, 1, "2026-10-10T21:47:29.123604Z");
19756        invalid_alias.payload["max_supported"] = json!(0);
19757        for (history, expected_reason) in [
19758            (
19759                vec![marker.clone(), marker.clone()],
19760                "duplicate_version_marker_record",
19761            ),
19762            (
19763                vec![marker, invalid_alias],
19764                "version_marker_history_range_invalid",
19765            ),
19766        ] {
19767            let error =
19768                WorkflowState::new(history, "rust-workers".into(), DEFAULT_CODEC.into(), None)
19769                    .expect_err("invalid historical alias");
19770            assert!(matches!(error,
19771                Error::NonDeterministicReplay(ReplayFailure { reason, .. })
19772                    if reason == expected_reason));
19773        }
19774    }
19775
19776    #[test]
19777    fn retained_patch_aliases_require_the_original_call_boundary() {
19778        let ctx = workflow_context(vec![
19779            retained_patch_marker(1, 1, "2026-10-10T21:47:29.120756Z"),
19780            retained_patch_marker(2, 1, "2026-10-10T21:47:29.123604Z"),
19781        ]);
19782        assert!(ctx.patched("retained-patch").unwrap());
19783        assert!(ctx.ensure_history_consumed().is_err());
19784        assert_eq!(ctx.state.lock().unwrap().command_cursor, 1);
19785        ctx.deprecate_patch("retained-patch").unwrap();
19786        ctx.ensure_history_consumed().unwrap();
19787        assert!(ctx.take_commands().unwrap().is_empty());
19788    }
19789
19790    #[test]
19791    fn retained_patch_aliases_do_not_replace_an_earlier_legacy_decision() {
19792        let ctx = workflow_context(vec![
19793            history_event(
19794                "SideEffectRecorded",
19795                json!({
19796                    "sequence": 1, "result": fixture_envelope(json!("old-operation")),
19797                }),
19798            ),
19799            retained_patch_marker(2, 1, "2026-10-10T21:47:29.123604Z"),
19800        ]);
19801        assert!(!ctx.patched("retained-patch").unwrap());
19802        assert_eq!(
19803            ctx.side_effect(|| "must not execute".to_string()).unwrap(),
19804            "old-operation"
19805        );
19806        let error = ctx.patched("retained-patch").unwrap_err();
19807        assert!(matches!(error,
19808            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19809                if reason == "version_marker_conflict"));
19810        assert_eq!(ctx.state.lock().unwrap().command_cursor, 1);
19811        assert!(ctx.take_commands().unwrap().is_empty());
19812    }
19813
19814    #[test]
19815    fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
19816        let mut state = WorkflowState::new(
19817            Vec::new(),
19818            "rust-workers".to_string(),
19819            DEFAULT_CODEC.to_string(),
19820            None,
19821        )
19822        .expect("workflow state");
19823        state.workflow_command_identity = "command-7".to_string();
19824        let context = WorkflowContext {
19825            state: Arc::new(Mutex::new(state)),
19826            cancellation_scope_id: "root".into(),
19827        };
19828        let item =
19829            WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
19830
19831        context
19832            .append_workflow_stream("output", &[item], Some(10))
19833            .expect("append command");
19834        context
19835            .error_workflow_stream("output", "producer failed", None)
19836            .expect("error command");
19837        let commands = context.take_commands().expect("commands");
19838
19839        assert_eq!(commands[0]["type"], "record_side_effect");
19840        assert_eq!(
19841            commands[0]["workflow_stream"]["command_identity"],
19842            "command-7"
19843        );
19844        assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
19845        assert_eq!(
19846            commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
19847            "dw-stream:command-7:0:0"
19848        );
19849        assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
19850
19851        let recorded = history_event(
19852            "SideEffectRecorded",
19853            json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
19854        );
19855        let mut replay_state = WorkflowState::new(
19856            vec![recorded],
19857            "rust-workers".to_string(),
19858            DEFAULT_CODEC.to_string(),
19859            None,
19860        )
19861        .expect("replay state");
19862        replay_state.workflow_command_identity = "command-7".to_string();
19863        let replay_context = WorkflowContext {
19864            state: Arc::new(Mutex::new(replay_state)),
19865            cancellation_scope_id: "root".into(),
19866        };
19867        replay_context
19868            .append_workflow_stream(
19869                "output",
19870                &[WorkflowStreamAppendItem::from_reference(
19871                    "s3://bucket/item.avro",
19872                )],
19873                Some(10),
19874            )
19875            .expect("replayed append");
19876        assert!(replay_context
19877            .take_commands()
19878            .expect("replayed commands")
19879            .is_empty());
19880    }
19881
19882    #[test]
19883    fn workflow_stream_authoring_requires_server_durable_command_identity() {
19884        let context = workflow_context(Vec::new());
19885        let error = context
19886            .append_workflow_stream(
19887                "output",
19888                &[WorkflowStreamAppendItem::from_reference(
19889                    "s3://bucket/item.avro",
19890                )],
19891                None,
19892            )
19893            .expect_err("stream append without durable command identity must fail closed");
19894
19895        assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
19896        assert!(context.take_commands().expect("commands").is_empty());
19897    }
19898
19899    #[test]
19900    fn worker_stream_authoring_uses_task_identity_and_replays_without_output() {
19901        let client = Client::builder("http://localhost:8080").build().unwrap();
19902        let mut worker = Worker::new(client, "rust-workers");
19903        worker.register_workflow("streams.worker", |ctx, _| async move {
19904            ctx.append_workflow_stream(
19905                "output",
19906                &[WorkflowStreamAppendItem::new(json!("hello"))?],
19907                None,
19908            )?;
19909            ctx.close_workflow_stream("output", None)?;
19910            Ok(json!("done"))
19911        });
19912        for command_id in [None, Some(String::new()), Some("command-42".to_string())] {
19913            let mut task = workflow_task("streams.worker", Vec::new(), DEFAULT_CODEC);
19914            task.workflow_command_id = command_id.clone();
19915            let expected_identity = command_id
19916                .as_deref()
19917                .filter(|id| !id.is_empty())
19918                .unwrap_or(&task.task_id)
19919                .to_string();
19920            let commands = worker
19921                .execute_workflow_task(task)
19922                .expect("ordinary worker stream output");
19923            assert_eq!(commands.len(), 3);
19924            assert_eq!(
19925                commands[0]["workflow_stream"]["command_identity"],
19926                expected_identity
19927            );
19928            assert_eq!(
19929                commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
19930                format!("dw-stream:{expected_identity}:0:0")
19931            );
19932            assert_eq!(
19933                commands[1]["workflow_stream"]["command_identity"],
19934                expected_identity
19935            );
19936            assert_eq!(commands[1]["workflow_stream"]["operation"], "close");
19937            let history = commands[..2]
19938                .iter()
19939                .enumerate()
19940                .map(|(index, command)| {
19941                    history_event(
19942                        "SideEffectRecorded",
19943                        json!({"sequence":index + 1,"result":command["result"]}),
19944                    )
19945                })
19946                .collect();
19947            let mut replay = workflow_task("streams.worker", history, DEFAULT_CODEC);
19948            replay.task_id = "replacement-task".to_string();
19949            let replayed = worker
19950                .execute_workflow_task(replay)
19951                .expect("replacement worker consumes recorded stream effects");
19952            assert_eq!(replayed.len(), 1);
19953            assert_eq!(replayed[0]["type"], "complete_workflow");
19954        }
19955        let mut missing = workflow_task("streams.worker", Vec::new(), DEFAULT_CODEC);
19956        missing.task_id.clear();
19957        assert!(matches!(
19958            worker.execute_workflow_task(missing),
19959            Err(Error::MissingWorkflowCommandIdentity)
19960        ));
19961    }
19962
19963    #[test]
19964    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
19965        fn worker(calls: Arc<AtomicUsize>) -> Worker {
19966            let client = Client::new("http://127.0.0.1:8080").expect("client");
19967            let mut worker = Worker::new(client, "rust-workers");
19968            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
19969                let calls = Arc::clone(&calls);
19970                async move {
19971                    let captured = ctx.side_effect(|| {
19972                        calls.fetch_add(1, Ordering::SeqCst);
19973                        "captured-once".to_string()
19974                    })?;
19975                    let version = ctx.get_version("cold-restart", 1, 2)?;
19976                    Ok(json!({"captured": captured, "version": version}))
19977                }
19978            });
19979            worker
19980        }
19981
19982        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
19983            WorkflowTask {
19984                task_id: "wft-side-effect-version".to_string(),
19985                workflow_command_id: None,
19986                workflow_id: Some("wf-side-effect-version".to_string()),
19987                run_id: Some("run-side-effect-version".to_string()),
19988                workflow_type: "rust.side-effect-version".to_string(),
19989                cancel_requested: false,
19990                payload_codec: DEFAULT_CODEC.to_string(),
19991                arguments: Some(
19992                    encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
19993                ),
19994                history_events,
19995                total_history_events: None,
19996                history_size_bytes: None,
19997                continue_as_new_recommended: None,
19998                history_budget_pressure: None,
19999                next_history_page_token: None,
20000                workflow_task_attempt: 1,
20001                workflow_signal_id: None,
20002                signal_name: None,
20003                signal_arguments: None,
20004                workflow_update_id: None,
20005                update_name: None,
20006                lease_owner: Some("rust-worker".to_string()),
20007            }
20008        }
20009
20010        let calls = Arc::new(AtomicUsize::new(0));
20011        let initial = worker(Arc::clone(&calls))
20012            .execute_workflow_task(task(Vec::new()))
20013            .expect("initial execution");
20014        assert_eq!(
20015            initial
20016                .iter()
20017                .map(|command| &command["type"])
20018                .collect::<Vec<_>>(),
20019            vec![
20020                "record_side_effect",
20021                "record_version_marker",
20022                "complete_workflow"
20023            ]
20024        );
20025        assert_eq!(calls.load(Ordering::SeqCst), 1);
20026
20027        let restarted = worker(Arc::clone(&calls));
20028        let replayed = restarted
20029            .execute_workflow_task(task(vec![
20030                history_event(
20031                    "SideEffectRecorded",
20032                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
20033                ),
20034                history_event(
20035                    "VersionMarkerRecorded",
20036                    json!({
20037                        "sequence": 2,
20038                        "change_id": "cold-restart",
20039                        "version": 2,
20040                        "min_supported": 1,
20041                        "max_supported": 2,
20042                    }),
20043                ),
20044            ]))
20045            .expect("cold replay");
20046        assert_eq!(replayed.len(), 1);
20047        assert_eq!(replayed[0]["type"], "complete_workflow");
20048        assert_eq!(calls.load(Ordering::SeqCst), 1);
20049    }
20050
20051    #[test]
20052    fn side_effect_replay_rejects_changed_rust_value_type() {
20053        let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
20054        let ctx = workflow_context(vec![history_event(
20055            "SideEffectRecorded",
20056            json!({"sequence": 1, "result": result}),
20057        )]);
20058        let error = ctx
20059            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
20060            .expect_err("changed type must fail replay");
20061        assert!(matches!(
20062            error,
20063            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20064                if reason == "side_effect_type_mismatch"
20065        ));
20066    }
20067
20068    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
20069        vec![
20070            history_event(
20071                "ActivityScheduled",
20072                json!({
20073                    "sequence": 1,
20074                    "activity_type": "flaky",
20075                    "activity_execution_id": "act-1",
20076                    "activity": {
20077                        "id": "act-1",
20078                        "sequence": 1,
20079                        "type": "flaky",
20080                        "queue": "critical-activities",
20081                        "execution_mode": null,
20082                        "retry_policy": {
20083                            "snapshot_version": 1,
20084                            "max_attempts": 3,
20085                            "backoff_seconds": [2, 4],
20086                            "start_to_close_timeout": 30,
20087                            "schedule_to_start_timeout": 5,
20088                            "schedule_to_close_timeout": 90,
20089                            "heartbeat_timeout": 10,
20090                            "non_retryable_error_types": ["PermanentError"]
20091                        }
20092                    }
20093                }),
20094            ),
20095            history_event(
20096                "ActivityStarted",
20097                json!({
20098                    "sequence": 1,
20099                    "activity_type": "flaky",
20100                    "activity_execution_id": "act-1",
20101                    "activity_attempt_id": "attempt-1",
20102                    "attempt_number": 1
20103                }),
20104            ),
20105            history_event(
20106                "ActivityRetryScheduled",
20107                json!({
20108                    "sequence": 1,
20109                    "activity_type": "flaky",
20110                    "activity_execution_id": "act-1",
20111                    "activity_attempt_id": "attempt-1",
20112                    "attempt_number": 1,
20113                    "retry_after_attempt": 1,
20114                    "retry_backoff_seconds": 2,
20115                    "failure_category": "activity",
20116                    "exception_type": "TransientError"
20117                }),
20118            ),
20119            history_event(
20120                "ActivityStarted",
20121                json!({
20122                    "sequence": 1,
20123                    "activity_type": "flaky",
20124                    "activity_execution_id": "act-1",
20125                    "activity_attempt_id": "attempt-2",
20126                    "attempt_number": 2
20127                }),
20128            ),
20129            history_event(
20130                "ActivityCompleted",
20131                json!({
20132                    "sequence": 1,
20133                    "activity_type": "flaky",
20134                    "activity_execution_id": "act-1",
20135                    "activity_attempt_id": "attempt-2",
20136                    "attempt_number": 2,
20137                    "payload_codec": DEFAULT_CODEC,
20138                    "result": fixture_envelope(json!({"status":"recovered"}))
20139                }),
20140            ),
20141        ]
20142    }
20143
20144    fn retry_activity_options() -> ActivityOptions {
20145        ActivityOptions::new()
20146            .task_queue("critical-activities")
20147            .retry_policy(
20148                ActivityRetryPolicy::new(3)
20149                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
20150                    .non_retryable_error_type("PermanentError"),
20151            )
20152            .start_to_close_timeout(Duration::from_secs(30))
20153            .schedule_to_start_timeout(Duration::from_secs(5))
20154            .schedule_to_close_timeout(Duration::from_secs(90))
20155            .heartbeat_timeout(Duration::from_secs(10))
20156    }
20157
20158    #[test]
20159    fn fixed_avro_value_round_trips_json_values() {
20160        let value = json!({"greeting": "hello", "count": 3, "ok": true});
20161        let envelope = PayloadEnvelope::avro(&value).expect("encode");
20162        assert_eq!(envelope.codec, DEFAULT_CODEC);
20163        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
20164    }
20165
20166    #[tokio::test]
20167    async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
20168        let client = Client::new("http://127.0.0.1:8080").expect("client");
20169        let mut worker = Worker::new(client, "rust-workers");
20170        worker.register_typed_workflow(
20171            "typed.contract.workflow",
20172            |_ctx, input: TypedContract| async move { Ok(input) },
20173        );
20174        worker.register_typed_activity(
20175            "typed.contract.activity",
20176            |_ctx, input: TypedContract| async move { Ok(input) },
20177        );
20178
20179        let expected = typed_contract();
20180        let arguments = AvroValue::Array(vec![
20181            AvroValue::from_serialize(&expected).expect("typed request")
20182        ]);
20183        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
20184        let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
20185        workflow.arguments = Some(envelope.clone());
20186        let commands = worker
20187            .execute_workflow_task(workflow)
20188            .expect("typed workflow task");
20189        let workflow_result: TypedContract =
20190            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
20191                .expect("workflow result envelope")
20192                .deserialize()
20193                .expect("workflow result type");
20194        assert_eq!(workflow_result, expected);
20195
20196        let activity = ActivityTask {
20197            task_id: "typed-contract-activity".to_string(),
20198            activity_attempt_id: Some("typed-contract-attempt".to_string()),
20199            attempt_id: None,
20200            activity_type: "typed.contract.activity".to_string(),
20201            payload_codec: DEFAULT_CODEC.to_string(),
20202            arguments: Some(envelope),
20203            attempt_number: 1,
20204            lease_owner: Some("rust-worker".to_string()),
20205        };
20206        let activity_result: TypedContract = worker
20207            .execute_activity_task(activity)
20208            .await
20209            .expect("typed activity task")
20210            .deserialize()
20211            .expect("activity result type");
20212        assert_eq!(activity_result, expected);
20213    }
20214
20215    #[tokio::test]
20216    async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
20217        let client = Client::new("http://127.0.0.1:8080").expect("client");
20218        let mut worker = Worker::new(client, "rust-workers");
20219        worker.register_typed_workflow(
20220            "typed.shape.workflow",
20221            |_ctx, input: TypedContract| async move { Ok(input) },
20222        );
20223        worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
20224            Ok(f64::NAN)
20225        });
20226
20227        let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
20228        workflow.arguments = Some(
20229            encode_typed_envelope(
20230                &AvroValue::Array(vec![
20231                    AvroValue::String("first".to_string()),
20232                    AvroValue::String("second".to_string()),
20233                ]),
20234                DEFAULT_CODEC,
20235            )
20236            .expect("malformed typed arguments"),
20237        );
20238        let commands = worker
20239            .execute_workflow_task(workflow)
20240            .expect("shape mismatch becomes a workflow failure");
20241        let message = commands[0]["message"].as_str().expect("failure message");
20242        assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
20243        assert!(message.contains(type_name::<TypedContract>()));
20244        assert!(message.contains("task carried 2 arguments"));
20245
20246        let activity = ActivityTask {
20247            task_id: "typed-unsupported-activity".to_string(),
20248            activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
20249            attempt_id: None,
20250            activity_type: "typed.unsupported.activity".to_string(),
20251            payload_codec: DEFAULT_CODEC.to_string(),
20252            arguments: Some(
20253                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
20254                    .expect("unit arguments"),
20255            ),
20256            attempt_number: 1,
20257            lease_owner: Some("rust-worker".to_string()),
20258        };
20259        let Error::HandlerType {
20260            handler_kind,
20261            handler_name,
20262            value_kind,
20263            rust_type,
20264            message,
20265        } = worker
20266            .execute_activity_task(activity)
20267            .await
20268            .expect_err("non-finite handler output must fail")
20269        else {
20270            panic!("expected contextual handler type failure");
20271        };
20272        assert_eq!(handler_kind, HandlerKind::Activity);
20273        assert_eq!(handler_name, "typed.unsupported.activity");
20274        assert_eq!(value_kind, HandlerValueKind::Result);
20275        assert_eq!(rust_type, type_name::<f64>());
20276        assert!(message.contains("non_finite_float"));
20277    }
20278
20279    #[tokio::test]
20280    async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
20281        #[derive(Clone, Default)]
20282        struct State {
20283            observed: Option<TypedContract>,
20284        }
20285
20286        let client = Client::new("http://127.0.0.1:8080").expect("client");
20287        let mut worker = Worker::new(client, "rust-workers");
20288        worker.register_typed_replayed_workflow(
20289            "typed.contract.replayed",
20290            State::default,
20291            |ctx, input: TypedContract, state| async move {
20292                let result: TypedContract =
20293                    ctx.activity_typed("typed.contract.activity", input).await?;
20294                state.update(|current| current.observed = Some(result.clone()))?;
20295                Ok(result)
20296            },
20297        );
20298        worker.register_replayed_query::<State, _, _>(
20299            "typed.contract.replayed",
20300            "observed",
20301            |_ctx, state, _args| async move {
20302                Ok(json!(state.observed.as_ref().map(|value| value.signed)))
20303            },
20304        );
20305
20306        let expected = typed_contract();
20307        let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
20308        let workflow_arguments =
20309            encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
20310                .expect("workflow arguments");
20311        let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
20312        let task = QueryTask {
20313            query_task_id: "typed-replay-query".to_string(),
20314            query_task_attempt: 1,
20315            lease_owner: Some("rust-worker".to_string()),
20316            workflow_id: Some("typed-replay".to_string()),
20317            run_id: Some("typed-replay-run".to_string()),
20318            workflow_type: "typed.contract.replayed".to_string(),
20319            query_name: "observed".to_string(),
20320            payload_codec: DEFAULT_CODEC.to_string(),
20321            workflow_arguments: Some(workflow_arguments),
20322            query_arguments: Some(
20323                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
20324                    .expect("query arguments"),
20325            ),
20326            history_events: vec![
20327                history_event(
20328                    "ActivityScheduled",
20329                    json!({
20330                        "sequence": 1,
20331                        "activity_type": "typed.contract.activity"
20332                    }),
20333                ),
20334                history_event(
20335                    "ActivityCompleted",
20336                    json!({
20337                        "sequence": 1,
20338                        "activity_type": "typed.contract.activity",
20339                        "payload_codec": DEFAULT_CODEC,
20340                        "result": result
20341                    }),
20342                ),
20343            ],
20344            history_export: None,
20345            run_status: Some("completed".to_string()),
20346        };
20347
20348        assert_eq!(
20349            worker
20350                .execute_query_task(task)
20351                .await
20352                .expect("typed replay query")
20353                .deserialize::<i64>()
20354                .expect("query result"),
20355            expected.signed
20356        );
20357    }
20358
20359    #[tokio::test]
20360    async fn update_context_preserves_workflow_input_and_committed_signals() {
20361        let client = Client::new("http://127.0.0.1:8080").expect("client");
20362        let mut worker = Worker::new(client, "rust-workers");
20363        worker.register_workflow("snapshot", |_ctx, _input| async move { Ok(Value::Null) });
20364        let workflow_input = AvroValue::Array(vec![typed_fidelity_probe()]);
20365        let signal_arguments = vec![AvroValue::Bytes(vec![0, 255]), AvroValue::Long(7)];
20366        let expected_input = workflow_input.clone();
20367        let expected_signals = signal_arguments.clone();
20368        let handler = move |context: QueryContext, arguments: AvroValue| {
20369            let expected_input = expected_input.clone();
20370            let expected_signals = expected_signals.clone();
20371            async move {
20372                assert_eq!(context.workflow_id.as_deref(), Some("wf-snapshot"));
20373                assert_eq!(context.run_id.as_deref(), Some("run-snapshot"));
20374                assert_eq!(context.workflow_input_avro_value(), &expected_input);
20375                assert_eq!(context.workflow_input(), &expected_input.into_json()?);
20376                assert_eq!(
20377                    context.signals_avro_value("changed"),
20378                    vec![expected_signals.clone()]
20379                );
20380                assert_eq!(
20381                    context.signals("changed"),
20382                    vec![expected_signals
20383                        .into_iter()
20384                        .map(AvroValue::into_json)
20385                        .collect::<Result<Vec<_>>>()?]
20386                );
20387                assert_eq!(
20388                    context.signal_events()[0].id.as_deref(),
20389                    Some("signal-original")
20390                );
20391                assert_eq!(context.signal_events()[0].workflow_sequence, Some(1));
20392                Ok(arguments)
20393            }
20394        };
20395        worker.register_query_avro_value("snapshot", "inspect", handler.clone());
20396        worker.register_update_avro_value("snapshot", "inspect", handler);
20397        let update_arguments = AvroValue::Array(vec![AvroValue::String("update-only".into())]);
20398        let update_envelope =
20399            encode_typed_envelope(&update_arguments, DEFAULT_CODEC).expect("update input");
20400        let mut task = workflow_task(
20401            "snapshot",
20402            vec![
20403                history_event(
20404                    "SignalReceived",
20405                    json!({
20406                        "signal_id": "signal-original", "signal_name": "changed", "workflow_sequence": 1,
20407                        "arguments": encode_typed_envelope(&AvroValue::Array(signal_arguments), DEFAULT_CODEC).expect("signal input"),
20408                    }),
20409                ),
20410                history_event(
20411                    "UpdateAccepted",
20412                    json!({
20413                        "update_id": "update-original", "update_name": "inspect", "arguments": update_envelope.clone(),
20414                    }),
20415                ),
20416            ],
20417            DEFAULT_CODEC,
20418        );
20419        task.arguments =
20420            Some(encode_typed_envelope(&workflow_input, DEFAULT_CODEC).expect("workflow input"));
20421        task.workflow_update_id = Some("update-original".into());
20422        task.update_name = Some("inspect".into());
20423        let query = QueryTask {
20424            query_task_id: "query-snapshot".into(),
20425            query_task_attempt: 1,
20426            lease_owner: None,
20427            workflow_id: task.workflow_id.clone(),
20428            run_id: task.run_id.clone(),
20429            workflow_type: task.workflow_type.clone(),
20430            query_name: "inspect".into(),
20431            payload_codec: DEFAULT_CODEC.into(),
20432            workflow_arguments: task.arguments.clone(),
20433            query_arguments: Some(update_envelope),
20434            history_events: task.history_events.clone(),
20435            history_export: None,
20436            run_status: Some("running".into()),
20437        };
20438        assert_eq!(
20439            worker
20440                .execute_query_task(query)
20441                .await
20442                .expect("query snapshot"),
20443            update_arguments
20444        );
20445        for _ in 0..2 {
20446            let commands = worker
20447                .execute_workflow_task(task.clone())
20448                .expect("cold update snapshot");
20449            assert_eq!(commands[0]["type"], "complete_update");
20450            assert_eq!(commands[0]["update_id"], "update-original");
20451            assert_eq!(
20452                decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
20453                    .expect("update result"),
20454                update_arguments
20455            );
20456        }
20457    }
20458
20459    #[test]
20460    fn committed_signal_snapshot_counts_identities_once() {
20461        let arguments = encode_value_envelope(&json!([7]), DEFAULT_CODEC).expect("signal input");
20462        let received = history_event(
20463            "SignalReceived",
20464            json!({
20465                "signal_id": "signal-first", "workflow_command_id": "command-first",
20466                "signal_name": "changed", "arguments": arguments.clone(),
20467            }),
20468        );
20469        let applied = history_event(
20470            "SignalApplied",
20471            json!({
20472                "signal_id": "signal-first", "workflow_command_id": "command-first",
20473                "signal_name": "changed", "value": encode_value_envelope(&json!(7), DEFAULT_CODEC).expect("applied value"), "workflow_sequence": 1,
20474            }),
20475        );
20476        let distinct = history_event(
20477            "SignalReceived",
20478            json!({
20479                "signal_id": "signal-second", "workflow_command_id": "command-second",
20480                "signal_name": "changed", "arguments": arguments,
20481            }),
20482        );
20483        let history = vec![received.clone(), applied.clone(), distinct];
20484        let signals =
20485            history_signal_events(&history, None, DEFAULT_CODEC).expect("signal snapshot");
20486        assert_eq!(signals.len(), 2);
20487        assert_eq!(signals[0].id.as_deref(), Some("signal-first"));
20488        assert_eq!(signals[1].id.as_deref(), Some("signal-second"));
20489        assert_eq!(signals[0].workflow_sequence, Some(1));
20490        assert_eq!(signals[0].arguments, signals[1].arguments);
20491        let mut command_only = received.clone();
20492        command_only
20493            .payload
20494            .as_object_mut()
20495            .expect("payload")
20496            .remove("signal_id");
20497        let signals = history_signal_events(&[command_only, applied.clone()], None, DEFAULT_CODEC)
20498            .expect("command identity");
20499        assert_eq!(signals.len(), 1);
20500        assert_eq!(signals[0].id.as_deref(), Some("signal-first"));
20501        let mut conflicting = applied;
20502        conflicting.payload["value"] =
20503            encode_value_envelope(&json!(8), DEFAULT_CODEC).expect("different input");
20504        assert!(history_signal_events(&[received, conflicting], None, DEFAULT_CODEC).is_err());
20505    }
20506
20507    #[test]
20508    fn committed_signal_snapshot_preserves_argument_vectors_after_application() {
20509        for arguments in [
20510            vec![],
20511            vec![AvroValue::Array(vec![
20512                AvroValue::Long(1),
20513                AvroValue::Long(2),
20514            ])],
20515            vec![AvroValue::Bytes(vec![0, 255])],
20516            vec![AvroValue::Null],
20517            vec![AvroValue::Boolean(false)],
20518            vec![
20519                AvroValue::Long(1),
20520                AvroValue::Array(vec![AvroValue::Long(2)]),
20521            ],
20522        ] {
20523            let applied = match arguments.as_slice() {
20524                [] => AvroValue::Boolean(true),
20525                [value] => value.clone(),
20526                values => AvroValue::Array(values.to_vec()),
20527            };
20528            for applied in [applied, AvroValue::Array(arguments.clone())] {
20529                let events = vec![
20530                    history_event(
20531                        "SignalReceived",
20532                        json!({
20533                            "signal_id": "original", "signal_name": "changed",
20534                            "arguments": encode_typed_envelope(&AvroValue::Array(arguments.clone()), DEFAULT_CODEC).expect("signal arguments"),
20535                        }),
20536                    ),
20537                    history_event(
20538                        "SignalApplied",
20539                        json!({
20540                            "signal_id": "original", "signal_name": "changed",
20541                            "value": encode_typed_envelope(&applied, DEFAULT_CODEC).expect("application value"),
20542                        }),
20543                    ),
20544                ];
20545                let signals = history_signal_events(&events, None, DEFAULT_CODEC)
20546                    .expect("canonical signal snapshot");
20547                assert_eq!(signals.len(), 1);
20548                assert_eq!(signals[0].avro_arguments, arguments);
20549            }
20550        }
20551    }
20552
20553    #[tokio::test]
20554    async fn update_context_rejects_malformed_snapshot_before_handler() {
20555        let client = Client::new("http://127.0.0.1:8080").expect("client");
20556        let mut worker = Worker::new(client, "rust-workers");
20557        worker.register_workflow("snapshot", |_ctx, _input| async move { Ok(Value::Null) });
20558        let calls = Arc::new(AtomicUsize::new(0));
20559        let observed = Arc::clone(&calls);
20560        worker.register_update("snapshot", "inspect", move |_ctx, _arguments| {
20561            observed.fetch_add(1, Ordering::SeqCst);
20562            async { Ok(Value::Null) }
20563        });
20564        let mut task = workflow_task(
20565            "snapshot",
20566            vec![history_event(
20567                "UpdateAccepted",
20568                json!({
20569                    "update_id": "update-original", "update_name": "inspect",
20570                    "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("update input"),
20571                }),
20572            )],
20573            DEFAULT_CODEC,
20574        );
20575        task.workflow_update_id = Some("update-original".into());
20576        task.update_name = Some("inspect".into());
20577        let mut bad_input = task.clone();
20578        bad_input.arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "invalid"}));
20579        assert!(worker.execute_workflow_task(bad_input).is_err());
20580        task.history_events.insert(0, history_event("SignalReceived", json!({
20581            "signal_name": "changed", "arguments": {"codec": DEFAULT_CODEC, "blob": "invalid"},
20582        })));
20583        assert!(worker.execute_workflow_task(task).is_err());
20584        assert_eq!(calls.load(Ordering::SeqCst), 0);
20585    }
20586
20587    #[tokio::test]
20588    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
20589        let client = Client::new("http://127.0.0.1:8080").expect("client");
20590        let mut worker = Worker::new(client, "rust-workers");
20591        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
20592        worker
20593            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
20594        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
20595            Ok(input)
20596        });
20597        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
20598            Ok(input)
20599        });
20600        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
20601            Ok(AvroValue::Array(
20602                ctx.wait_signal_avro_value("changed").await?,
20603            ))
20604        });
20605
20606        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
20607        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
20608
20609        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
20610        workflow.arguments = Some(envelope.clone());
20611        let commands = worker
20612            .execute_workflow_task(workflow)
20613            .expect("typed workflow task");
20614        assert_eq!(commands[0]["type"], "complete_workflow");
20615        assert_eq!(
20616            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
20617                .expect("typed workflow result"),
20618            arguments
20619        );
20620
20621        let activity = ActivityTask {
20622            task_id: "activity-typed".to_string(),
20623            activity_attempt_id: Some("attempt-typed".to_string()),
20624            attempt_id: None,
20625            activity_type: "typed.activity".to_string(),
20626            payload_codec: DEFAULT_CODEC.to_string(),
20627            arguments: Some(envelope.clone()),
20628            attempt_number: 1,
20629            lease_owner: Some("rust-worker".to_string()),
20630        };
20631        assert_eq!(
20632            worker
20633                .execute_activity_task(activity)
20634                .await
20635                .expect("typed activity result"),
20636            arguments
20637        );
20638
20639        let query = QueryTask {
20640            query_task_id: "query-typed".to_string(),
20641            query_task_attempt: 1,
20642            lease_owner: Some("rust-worker".to_string()),
20643            workflow_id: Some("typed-1".to_string()),
20644            run_id: Some("run-typed".to_string()),
20645            workflow_type: "typed.echo".to_string(),
20646            query_name: "inspect".to_string(),
20647            payload_codec: DEFAULT_CODEC.to_string(),
20648            workflow_arguments: Some(
20649                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
20650                    .expect("workflow input"),
20651            ),
20652            query_arguments: Some(envelope.clone()),
20653            history_events: Vec::new(),
20654            history_export: None,
20655            run_status: Some("running".to_string()),
20656        };
20657        assert_eq!(
20658            worker
20659                .execute_query_task(query)
20660                .await
20661                .expect("typed query result"),
20662            arguments
20663        );
20664
20665        let mut update = workflow_task(
20666            "typed.echo",
20667            vec![history_event(
20668                "UpdateAccepted",
20669                json!({
20670                    "update_id": "update-typed",
20671                    "update_name": "replace",
20672                    "arguments": envelope.clone(),
20673                }),
20674            )],
20675            DEFAULT_CODEC,
20676        );
20677        update.workflow_update_id = Some("update-typed".to_string());
20678        update.update_name = Some("replace".to_string());
20679        let commands = worker
20680            .execute_workflow_task(update)
20681            .expect("typed update task");
20682        assert_eq!(commands[0]["type"], "complete_update");
20683        assert_eq!(
20684            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
20685                .expect("typed update result"),
20686            arguments
20687        );
20688
20689        let mut signal = workflow_task(
20690            "typed.signal",
20691            vec![history_event(
20692                "SignalReceived",
20693                json!({
20694                    "signal_id": "signal-typed",
20695                    "signal_name": "changed",
20696                    "arguments": envelope.clone(),
20697                }),
20698            )],
20699            DEFAULT_CODEC,
20700        );
20701        signal.workflow_signal_id = Some("signal-typed".to_string());
20702        signal.signal_name = Some("changed".to_string());
20703        signal.signal_arguments = Some(envelope);
20704        let commands = worker
20705            .execute_workflow_task(signal)
20706            .expect("typed signal resume");
20707        assert_eq!(
20708            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
20709                .expect("typed signal result"),
20710            arguments
20711        );
20712    }
20713
20714    #[tokio::test]
20715    async fn typed_helpers_never_parse_json_inspection_projection() {
20716        let collision_values = projection_collision_probe();
20717        let expected = AvroValue::Array(collision_values.clone());
20718        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
20719
20720        let activity_context = workflow_context_with_codec(
20721            vec![history_event(
20722                "ActivityCompleted",
20723                json!({
20724                    "sequence": 1,
20725                    "activity_type": "collision.activity",
20726                    "payload_codec": DEFAULT_CODEC,
20727                    "result": envelope.clone(),
20728                }),
20729            )],
20730            DEFAULT_CODEC,
20731        );
20732        assert_eq!(
20733            activity_context
20734                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
20735                .await
20736                .expect("typed activity collision result"),
20737            expected
20738        );
20739
20740        let signal_context = workflow_context_with_codec(
20741            vec![
20742                history_event(
20743                    "SignalWaitOpened",
20744                    json!({"sequence": 1, "signal_name": "collision"}),
20745                ),
20746                history_event(
20747                    "SignalApplied",
20748                    json!({
20749                        "sequence": 1,
20750                        "signal_name": "collision",
20751                        "payload_codec": DEFAULT_CODEC,
20752                        "value": envelope.clone(),
20753                    }),
20754                ),
20755            ],
20756            DEFAULT_CODEC,
20757        );
20758        assert_eq!(
20759            signal_context
20760                .wait_signal_avro_value("collision")
20761                .await
20762                .expect("typed signal collision arguments"),
20763            collision_values
20764        );
20765
20766        let child_context = workflow_context_with_codec(
20767            vec![
20768                history_event(
20769                    "ChildWorkflowScheduled",
20770                    json!({
20771                        "sequence": 1,
20772                        "child_workflow_instance_id": "collision-child",
20773                        "child_workflow_run_id": "collision-run",
20774                        "child_workflow_type": "collision.child",
20775                    }),
20776                ),
20777                history_event(
20778                    "ChildRunCompleted",
20779                    json!({
20780                        "sequence": 1,
20781                        "child_workflow_instance_id": "collision-child",
20782                        "child_workflow_run_id": "collision-run",
20783                        "child_workflow_type": "collision.child",
20784                        "payload_codec": DEFAULT_CODEC,
20785                        "result": envelope,
20786                    }),
20787                ),
20788            ],
20789            DEFAULT_CODEC,
20790        );
20791        let child = child_context
20792            .start_child_workflow_avro_value(
20793                "collision.child",
20794                ChildWorkflowOptions::new("collision-workers"),
20795                AvroValue::Array(Vec::new()),
20796            )
20797            .await
20798            .expect("typed child collision result");
20799        assert_eq!(child.result, expected);
20800    }
20801
20802    #[tokio::test]
20803    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
20804        let client = Client::new("http://127.0.0.1:8080").expect("client");
20805        let mut worker = Worker::new(client, "rust-workers");
20806        worker.register_replayed_workflow_avro_value(
20807            "typed.replayed",
20808            || (),
20809            |_ctx, input, _state| async move { Ok(input) },
20810        );
20811        worker.register_replayed_query_avro_value::<(), _, _>(
20812            "typed.replayed",
20813            "inspect",
20814            |ctx, _state, args| async move {
20815                let mut signals = ctx.signals_avro_value("collision");
20816                let signal = signals
20817                    .pop()
20818                    .map(AvroValue::Array)
20819                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
20820                Ok(AvroValue::Array(vec![
20821                    ctx.workflow_input_avro_value().clone(),
20822                    signal,
20823                    args,
20824                ]))
20825            },
20826        );
20827        let arguments = AvroValue::Array(projection_collision_probe());
20828        let signal_arguments =
20829            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
20830        let task = QueryTask {
20831            query_task_id: "query-typed-replay".to_string(),
20832            query_task_attempt: 1,
20833            lease_owner: Some("rust-worker".to_string()),
20834            workflow_id: Some("typed-replay".to_string()),
20835            run_id: Some("run-typed-replay".to_string()),
20836            workflow_type: "typed.replayed".to_string(),
20837            query_name: "inspect".to_string(),
20838            payload_codec: DEFAULT_CODEC.to_string(),
20839            workflow_arguments: Some(
20840                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
20841            ),
20842            query_arguments: Some(
20843                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
20844            ),
20845            history_events: vec![history_event(
20846                "SignalReceived",
20847                json!({
20848                    "signal_id": "collision-signal",
20849                    "signal_name": "collision",
20850                    "workflow_sequence": 1,
20851                    "payload_codec": DEFAULT_CODEC,
20852                    "arguments": signal_arguments,
20853                }),
20854            )],
20855            history_export: None,
20856            run_status: Some("completed".to_string()),
20857        };
20858
20859        assert_eq!(
20860            worker
20861                .execute_query_task(task)
20862                .await
20863                .expect("typed replay query"),
20864            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
20865        );
20866    }
20867
20868    #[test]
20869    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
20870        let value = BTreeMap::from([(1_i32, "integer key")]);
20871        let error = PayloadEnvelope::avro(&value)
20872            .expect_err("integer map keys must fail")
20873            .to_string();
20874
20875        assert!(error.contains("invalid_map_key"));
20876    }
20877
20878    #[test]
20879    fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
20880        let envelope = PayloadEnvelope {
20881            codec: "json".to_string(),
20882            blob: r#"{"greeting":"hello"}"#.to_string(),
20883        };
20884
20885        let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
20886        let diagnostic = error.to_string();
20887        assert!(diagnostic.contains("unsupported_payload_codec"));
20888        assert!(diagnostic.contains("codec=\"avro\""));
20889        assert!(diagnostic.contains("HTTP document transport"));
20890    }
20891
20892    #[test]
20893    fn untagged_json_payload_value_fails_closed() {
20894        let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
20895            .expect_err("untagged JSON payload values must fail");
20896        let diagnostic = error.to_string();
20897        assert!(diagnostic.contains("unsupported_payload_codec"));
20898        assert!(diagnostic.contains("untagged durable payload"));
20899        assert!(diagnostic.contains("HTTP document transport"));
20900    }
20901
20902    #[test]
20903    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
20904        let envelope = PayloadEnvelope {
20905            codec: DEFAULT_CODEC.to_string(),
20906            blob: BASE64.encode([0x01]),
20907        };
20908
20909        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
20910        assert!(error.to_string().contains("invalid_payload_framing"));
20911    }
20912
20913    #[tokio::test]
20914    async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
20915        let server = MockWorkerServer::start();
20916        let client = Client::builder(server.base_url())
20917            .timeout(Duration::from_secs(2))
20918            .build()
20919            .expect("client");
20920        let invalid_commands = [
20921            json!({
20922                "type": "complete_workflow",
20923                "result": {"codec": "json", "blob": null}
20924            }),
20925            json!({
20926                "type": "schedule_activity",
20927                "arguments": {"codec": "yaml", "blob": "ignored"}
20928            }),
20929            json!({
20930                "type": "start_child_workflow",
20931                "arguments": {"codec": DEFAULT_CODEC, "blob": null}
20932            }),
20933            json!({"type": "continue_as_new", "arguments": []}),
20934            json!({"type": "complete_update"}),
20935            json!({"type": "record_side_effect", "result": null}),
20936            json!({
20937                "type": "start_service_operation",
20938                "payload_codec": DEFAULT_CODEC,
20939                "request_payload": "raw-avro-bytes"
20940            }),
20941        ];
20942
20943        for command in invalid_commands {
20944            let error = client
20945                .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
20946                .await
20947                .expect_err("invalid durable payload must fail locally");
20948            let diagnostic = error.to_string();
20949            assert!(
20950                diagnostic.contains("unsupported_payload_codec")
20951                    || diagnostic.contains("invalid_payload_envelope")
20952                    || diagnostic.contains("untagged durable payload"),
20953                "unexpected validation diagnostic: {diagnostic}"
20954            );
20955        }
20956
20957        assert_eq!(
20958            server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
20959            0,
20960            "invalid command payloads must not reach HTTP transport"
20961        );
20962    }
20963
20964    #[test]
20965    fn workflow_completion_validates_only_protocol_owned_payload_slots() {
20966        let envelope = fixture_envelope(json!({"codec": "customer-value"}));
20967        let commands = [
20968            json!({"type": "complete_workflow", "result": envelope.clone()}),
20969            json!({"type": "schedule_activity", "arguments": envelope.clone()}),
20970            json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
20971            json!({"type": "continue_as_new", "arguments": envelope.clone()}),
20972            json!({"type": "complete_update", "result": envelope.clone()}),
20973            json!({"type": "record_side_effect", "result": envelope.clone()}),
20974            json!({
20975                "type": "start_service_operation",
20976                "payload_codec": DEFAULT_CODEC,
20977                "request_payload": envelope.clone()
20978            }),
20979            json!({
20980                "type": "complete_workflow",
20981                "result": envelope,
20982                "metadata": {
20983                    "codec": "json",
20984                    "payload_codec": "customer-codec",
20985                    "result": {"codec": "yaml", "blob": null}
20986                }
20987            }),
20988        ];
20989
20990        validate_workflow_task_commands(&commands)
20991            .expect("customer metadata must not become a protocol codec declaration");
20992    }
20993
20994    #[test]
20995    fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
20996        assert_eq!(
20997            decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
20998            AvroValue::Array(Vec::new())
20999        );
21000        assert_eq!(
21001            decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
21002            AvroValue::Array(Vec::new())
21003        );
21004
21005        let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
21006        signal.signal_name = Some("empty-signal".to_string());
21007        signal.signal_arguments = None;
21008        let decoded = decode_resume_signal(&signal)
21009            .expect("valid Avro signal")
21010            .expect("named signal resumes the workflow");
21011        assert!(decoded.arguments.is_empty());
21012    }
21013
21014    #[tokio::test]
21015    async fn malformed_task_level_codecs_become_pre_handler_failures() {
21016        let client = Client::new("http://127.0.0.1:8080").expect("client");
21017        let mut worker = Worker::new(client, "rust-workers");
21018        let handler_calls = Arc::new(AtomicUsize::new(0));
21019
21020        let calls = Arc::clone(&handler_calls);
21021        worker.register_workflow("codec.workflow", move |_ctx, _args| {
21022            calls.fetch_add(1, Ordering::SeqCst);
21023            async move { Ok(Value::Null) }
21024        });
21025        let calls = Arc::clone(&handler_calls);
21026        worker.register_activity("codec.activity", move |_ctx, _args| {
21027            calls.fetch_add(1, Ordering::SeqCst);
21028            async move { Ok(Value::Null) }
21029        });
21030        let calls = Arc::clone(&handler_calls);
21031        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
21032            calls.fetch_add(1, Ordering::SeqCst);
21033            async move { Ok(Value::Null) }
21034        });
21035
21036        let mut failures = Vec::new();
21037        for codec_case in [
21038            InvalidTaskPayloadCodec::Missing,
21039            InvalidTaskPayloadCodec::Null,
21040            InvalidTaskPayloadCodec::NonString,
21041        ] {
21042            let mut workflow = json!({
21043                "task_id": format!("workflow-{}", codec_case.label()),
21044                "workflow_type": "codec.workflow"
21045            });
21046            codec_case.apply(&mut workflow);
21047            match serde_json::from_value::<WorkflowTask>(workflow) {
21048                Ok(task) => match worker.execute_workflow_task(task) {
21049                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
21050                    outcome => failures.push(format!(
21051                        "workflow {} codec returned {outcome:?}",
21052                        codec_case.label()
21053                    )),
21054                },
21055                Err(error) => failures.push(format!(
21056                    "workflow {} codec failed transport deserialization: {error}",
21057                    codec_case.label()
21058                )),
21059            }
21060
21061            let mut activity = json!({
21062                "task_id": format!("activity-{}", codec_case.label()),
21063                "activity_attempt_id": format!("attempt-{}", codec_case.label()),
21064                "activity_type": "codec.activity",
21065                "attempt_number": 1
21066            });
21067            codec_case.apply(&mut activity);
21068            match serde_json::from_value::<ActivityTask>(activity) {
21069                Ok(task) => match worker.execute_activity_task(task).await {
21070                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
21071                    outcome => failures.push(format!(
21072                        "activity {} codec returned {outcome:?}",
21073                        codec_case.label()
21074                    )),
21075                },
21076                Err(error) => failures.push(format!(
21077                    "activity {} codec failed transport deserialization: {error}",
21078                    codec_case.label()
21079                )),
21080            }
21081
21082            let mut query = json!({
21083                "query_task_id": format!("query-{}", codec_case.label()),
21084                "workflow_type": "codec.workflow",
21085                "query_name": "known"
21086            });
21087            codec_case.apply(&mut query);
21088            match serde_json::from_value::<QueryTask>(query) {
21089                Ok(task) => match worker.execute_query_task(task).await {
21090                    Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
21091                    outcome => failures.push(format!(
21092                        "query {} codec returned {outcome:?}",
21093                        codec_case.label()
21094                    )),
21095                },
21096                Err(error) => failures.push(format!(
21097                    "query {} codec failed transport deserialization: {error}",
21098                    codec_case.label()
21099                )),
21100            }
21101        }
21102
21103        assert!(failures.is_empty(), "{}", failures.join("\n"));
21104        assert_eq!(
21105            handler_calls.load(Ordering::SeqCst),
21106            0,
21107            "invalid task codecs must not invoke a handler"
21108        );
21109    }
21110
21111    #[tokio::test]
21112    async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
21113        for codec_case in [
21114            InvalidTaskPayloadCodec::Missing,
21115            InvalidTaskPayloadCodec::Null,
21116            InvalidTaskPayloadCodec::NonString,
21117        ] {
21118            let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
21119            let client = Client::builder(server.base_url())
21120                .timeout(Duration::from_secs(2))
21121                .build()
21122                .expect("client");
21123            let mut worker = Worker::new(client, "rust-workers")
21124                .worker_id("codec-worker")
21125                .poll_timeout(Duration::from_millis(10));
21126            let handler_calls = Arc::new(AtomicUsize::new(0));
21127
21128            let calls = Arc::clone(&handler_calls);
21129            worker.register_workflow("codec.workflow", move |_ctx, _args| {
21130                calls.fetch_add(1, Ordering::SeqCst);
21131                async move { Ok(Value::Null) }
21132            });
21133            let calls = Arc::clone(&handler_calls);
21134            worker.register_activity("codec.activity", move |_ctx, _args| {
21135                calls.fetch_add(1, Ordering::SeqCst);
21136                async move { Ok(Value::Null) }
21137            });
21138            let calls = Arc::clone(&handler_calls);
21139            worker.register_query("codec.workflow", "known", move |_ctx, _args| {
21140                calls.fetch_add(1, Ordering::SeqCst);
21141                async move { Ok(Value::Null) }
21142            });
21143
21144            assert_eq!(
21145                worker.run_once().await.expect("invalid tasks are settled"),
21146                3,
21147                "all {} codec tasks must be handled",
21148                codec_case.label()
21149            );
21150            assert_eq!(
21151                handler_calls.load(Ordering::SeqCst),
21152                0,
21153                "{} task codecs must fail before every handler",
21154                codec_case.label()
21155            );
21156
21157            for path in [
21158                "/api/worker/workflow-tasks/codec-workflow/fail",
21159                "/api/worker/activity-tasks/codec-activity/fail",
21160                "/api/worker/query-tasks/codec-query/fail",
21161            ] {
21162                let body = server.request_body(path);
21163                assert!(
21164                    body["failure"]["message"]
21165                        .as_str()
21166                        .is_some_and(|message| message.contains("unsupported_payload_codec")),
21167                    "{path} must receive the stable codec diagnostic for the {} case: {body}",
21168                    codec_case.label()
21169                );
21170            }
21171            assert_eq!(
21172                server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
21173                    ["reason"],
21174                "query_payload_decode_failed"
21175            );
21176            for path in [
21177                "/api/worker/workflow-tasks/codec-workflow/complete",
21178                "/api/worker/activity-tasks/codec-activity/complete",
21179                "/api/worker/query-tasks/codec-query/complete",
21180            ] {
21181                assert_eq!(
21182                    server.request_count(path),
21183                    0,
21184                    "invalid {} codec task reached {path}",
21185                    codec_case.label()
21186                );
21187            }
21188        }
21189    }
21190
21191    #[tokio::test]
21192    async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
21193        let client = Client::new("http://127.0.0.1:8080").expect("client");
21194        let mut worker = Worker::new(client, "rust-workers");
21195        let handler_calls = Arc::new(AtomicUsize::new(0));
21196
21197        let calls = Arc::clone(&handler_calls);
21198        worker.register_workflow("codec.workflow", move |_ctx, _args| {
21199            calls.fetch_add(1, Ordering::SeqCst);
21200            async move { Ok(Value::Null) }
21201        });
21202        let calls = Arc::clone(&handler_calls);
21203        worker.register_activity("codec.activity", move |_ctx, _args| {
21204            calls.fetch_add(1, Ordering::SeqCst);
21205            async move { Ok(Value::Null) }
21206        });
21207        let calls = Arc::clone(&handler_calls);
21208        worker.register_update("codec.workflow", "known", move |_ctx, _args| {
21209            calls.fetch_add(1, Ordering::SeqCst);
21210            async move { Ok(Value::Null) }
21211        });
21212        let calls = Arc::clone(&handler_calls);
21213        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
21214            calls.fetch_add(1, Ordering::SeqCst);
21215            async move { Ok(Value::Null) }
21216        });
21217
21218        let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
21219        workflow.payload_codec = "json".to_string();
21220        workflow.arguments = None;
21221        let error = worker
21222            .execute_workflow_task(workflow)
21223            .expect_err("task codec must be checked before workflow invocation");
21224        assert!(error.to_string().contains("unsupported_payload_codec"));
21225
21226        let activity = ActivityTask {
21227            task_id: "activity-invalid-codec".to_string(),
21228            activity_attempt_id: None,
21229            attempt_id: None,
21230            activity_type: "codec.activity".to_string(),
21231            payload_codec: "unknown".to_string(),
21232            arguments: None,
21233            attempt_number: 1,
21234            lease_owner: None,
21235        };
21236        let error = worker
21237            .execute_activity_task(activity)
21238            .await
21239            .expect_err("task codec must be checked before activity invocation");
21240        assert!(error.to_string().contains("unsupported_payload_codec"));
21241
21242        let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
21243        update.workflow_update_id = Some("update-invalid-codec".to_string());
21244        update.update_name = Some("known".to_string());
21245        update.history_events.push(history_event(
21246            "UpdateAccepted",
21247            json!({
21248                "update_id": "update-invalid-codec",
21249                "update_name": "known",
21250                "arguments": {"codec": "json", "blob": null}
21251            }),
21252        ));
21253        let error = worker
21254            .execute_workflow_task(update)
21255            .expect_err("nested update codec must be checked before handler lookup");
21256        assert!(error.to_string().contains("unsupported_payload_codec"));
21257
21258        let query: QueryTask = serde_json::from_value(json!({
21259            "query_task_id": "query-invalid-codec",
21260            "workflow_type": "codec.workflow",
21261            "query_name": "known",
21262            "payload_codec": DEFAULT_CODEC,
21263            "workflow_arguments": null,
21264            "query_arguments": null,
21265            "history_export": {
21266                "payloads": {"codec": DEFAULT_CODEC},
21267                "signals": [{
21268                    "name": "empty",
21269                    "payload_codec": "json",
21270                    "arguments": null
21271                }]
21272            }
21273        }))
21274        .expect("query task");
21275        let failure = worker
21276            .execute_query_task(query)
21277            .await
21278            .expect_err("exported signal codec must be checked before query invocation");
21279        assert_eq!(failure.reason, "query_payload_decode_failed");
21280        assert!(failure.message.contains("unsupported_payload_codec"));
21281
21282        let exported_history: QueryTask = serde_json::from_value(json!({
21283            "query_task_id": "query-invalid-history-codec",
21284            "workflow_type": "codec.workflow",
21285            "query_name": "known",
21286            "payload_codec": DEFAULT_CODEC,
21287            "history_export": {
21288                "payloads": {"codec": DEFAULT_CODEC},
21289                "history_events": [{
21290                    "type": "ActivityCompleted",
21291                    "payload": {"payload_codec": "unknown", "result": null}
21292                }]
21293            }
21294        }))
21295        .expect("query task");
21296        let failure = worker
21297            .execute_query_task(exported_history)
21298            .await
21299            .expect_err("exported history codec must be checked before query invocation");
21300        assert_eq!(failure.reason, "query_payload_decode_failed");
21301        assert!(failure.message.contains("unsupported_payload_codec"));
21302        assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
21303
21304        let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
21305        unknown_workflow.arguments = None;
21306        unknown_workflow.history_events.push(history_event(
21307            "SignalReceived",
21308            json!({
21309                "signal_name": "empty",
21310                "payload_codec": "json",
21311                "arguments": null
21312            }),
21313        ));
21314        let error = worker
21315            .execute_workflow_task(unknown_workflow)
21316            .expect_err("history codec must precede unknown workflow outcome");
21317        assert!(error.to_string().contains("unsupported_payload_codec"));
21318
21319        let unknown_activity = ActivityTask {
21320            task_id: "activity-unknown".to_string(),
21321            activity_attempt_id: None,
21322            attempt_id: None,
21323            activity_type: "missing".to_string(),
21324            payload_codec: "json".to_string(),
21325            arguments: None,
21326            attempt_number: 1,
21327            lease_owner: None,
21328        };
21329        let error = worker
21330            .execute_activity_task(unknown_activity)
21331            .await
21332            .expect_err("codec must precede unknown activity outcome");
21333        assert!(error.to_string().contains("unsupported_payload_codec"));
21334
21335        let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
21336        unknown_update.payload_codec = "json".to_string();
21337        unknown_update.arguments = None;
21338        unknown_update.workflow_update_id = Some("update-unknown".to_string());
21339        unknown_update.update_name = Some("missing".to_string());
21340        let error = worker
21341            .execute_workflow_task(unknown_update)
21342            .expect_err("codec must precede fail_update shortcut");
21343        assert!(error.to_string().contains("unsupported_payload_codec"));
21344
21345        let unknown_query: QueryTask = serde_json::from_value(json!({
21346            "query_task_id": "query-unknown",
21347            "workflow_type": "missing",
21348            "query_name": "missing",
21349            "payload_codec": "json",
21350            "workflow_arguments": null,
21351            "query_arguments": null
21352        }))
21353        .expect("query task");
21354        let failure = worker
21355            .execute_query_task(unknown_query)
21356            .await
21357            .expect_err("codec must precede unknown query outcome");
21358        assert_eq!(failure.reason, "query_payload_decode_failed");
21359        assert!(failure.message.contains("unsupported_payload_codec"));
21360    }
21361
21362    #[tokio::test]
21363    async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
21364        let client = Client::new("http://127.0.0.1:8080").expect("client");
21365        let worker = Worker::new(client, "rust-workers");
21366
21367        for event_type in ["SignalReceived", "SignalApplied"] {
21368            for (payload_field, codec) in [
21369                ("value", "json"),
21370                ("input", "unknown"),
21371                ("arguments", "json"),
21372            ] {
21373                let payload = json!({
21374                    "signal_name": "empty",
21375                    payload_field: {"codec": codec, "blob": null}
21376                });
21377                let workflow = workflow_task(
21378                    "missing",
21379                    vec![history_event(event_type, payload.clone())],
21380                    DEFAULT_CODEC,
21381                );
21382                let error = worker
21383                    .execute_workflow_task(workflow)
21384                    .expect_err("signal payload codec must precede unknown workflow outcome");
21385                assert!(
21386                    error.to_string().contains("unsupported_payload_codec"),
21387                    "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
21388                );
21389
21390                let query: QueryTask = serde_json::from_value(json!({
21391                    "query_task_id": format!("query-{event_type}-{payload_field}"),
21392                    "workflow_type": "missing",
21393                    "query_name": "missing",
21394                    "payload_codec": DEFAULT_CODEC,
21395                    "workflow_arguments": null,
21396                    "query_arguments": null,
21397                    "history_events": [{
21398                        "event_type": event_type,
21399                        "payload": payload
21400                    }]
21401                }))
21402                .expect("query task");
21403                let failure = worker
21404                    .execute_query_task(query)
21405                    .await
21406                    .expect_err("signal payload codec must precede unknown query outcome");
21407                assert_eq!(
21408                    failure.reason, "query_payload_decode_failed",
21409                    "{event_type}.{payload_field} returned an unrelated query outcome"
21410                );
21411                assert!(
21412                    failure.message.contains("unsupported_payload_codec"),
21413                    "{event_type}.{payload_field} returned an unrelated query error: {}",
21414                    failure.message
21415                );
21416            }
21417        }
21418    }
21419
21420    #[test]
21421    fn workflow_context_schedules_activity_until_completion_is_in_history() {
21422        let ctx = WorkflowContext {
21423            cancellation_scope_id: "root".into(),
21424            state: Arc::new(Mutex::new(
21425                WorkflowState::new_with_identity(
21426                    Vec::new(),
21427                    Some("wf-parent".to_string()),
21428                    Some("run-parent".to_string()),
21429                    "rust-workers".to_string(),
21430                    DEFAULT_CODEC.to_string(),
21431                    None,
21432                )
21433                .expect("workflow state"),
21434            )),
21435        };
21436
21437        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
21438        let mut task_context = TaskContext::from_waker(noop_waker_ref());
21439        assert!(matches!(
21440            call.as_mut().poll(&mut task_context),
21441            Poll::Pending
21442        ));
21443
21444        let commands = ctx.take_commands().expect("commands");
21445        assert_eq!(commands[0]["type"], "schedule_activity");
21446        assert_eq!(commands[0]["activity_type"], "hello.activity");
21447    }
21448
21449    #[test]
21450    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
21451        let ctx = workflow_context(Vec::new());
21452        let options = ActivityOptions::new()
21453            .task_queue("payments")
21454            .retry_policy(
21455                ActivityRetryPolicy::new(4)
21456                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
21457                    .non_retryable_error_type("ValidationError"),
21458            )
21459            .start_to_close_timeout(Duration::from_secs(120))
21460            .schedule_to_start_timeout(Duration::from_secs(10))
21461            .schedule_to_close_timeout(Duration::from_secs(300))
21462            .heartbeat_timeout(Duration::from_secs(15));
21463        let mut call = Box::pin(ctx.activity_with_options(
21464            "charge-card",
21465            options,
21466            json!([{"order_id": "o-1"}]),
21467        ));
21468        let mut task_context = TaskContext::from_waker(noop_waker_ref());
21469
21470        assert!(matches!(
21471            call.as_mut().poll(&mut task_context),
21472            Poll::Pending
21473        ));
21474        assert!(matches!(
21475            call.as_mut().poll(&mut task_context),
21476            Poll::Pending
21477        ));
21478
21479        let commands = ctx.take_commands().expect("activity command");
21480        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
21481        assert_eq!(commands[0]["queue"], "payments");
21482        assert_eq!(
21483            commands[0]["retry_policy"],
21484            json!({
21485                "max_attempts": 4,
21486                "backoff_seconds": [1, 3, 9],
21487                "non_retryable_error_types": ["ValidationError"],
21488            })
21489        );
21490        assert_eq!(commands[0]["start_to_close_timeout"], 120);
21491        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
21492        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
21493        assert_eq!(commands[0]["heartbeat_timeout"], 15);
21494    }
21495
21496    fn completed_worker_session_history(session: &WorkerSessionOptions) -> Vec<HistoryEvent> {
21497        vec![
21498            history_event(
21499                "ActivityScheduled",
21500                json!({"sequence":1,"activity_type":"render",
21501                "activity":{"queue":"gpu-workers","worker_session":session.to_wire().unwrap()}}),
21502            ),
21503            history_event(
21504                "ActivityCompleted",
21505                json!({"sequence":1,"activity_type":"render",
21506                "payload_codec":DEFAULT_CODEC,
21507                "result":encode_typed_envelope(&AvroValue::Long(42),DEFAULT_CODEC).unwrap()}),
21508            ),
21509        ]
21510    }
21511
21512    #[test]
21513    fn worker_session_activity_routes_to_session_queue_once() {
21514        let ctx = workflow_context(Vec::new());
21515        let session = WorkerSessionOptions::new("render-1").queue("gpu-workers");
21516        let mut call = Box::pin(
21517            ctx.activity("render", json!([]))
21518                .in_worker_session(session.clone()),
21519        );
21520        let mut cx = TaskContext::from_waker(noop_waker_ref());
21521        assert!(call.as_mut().poll(&mut cx).is_pending());
21522        assert!(call.as_mut().poll(&mut cx).is_pending());
21523        let commands = ctx.take_commands().unwrap();
21524        assert_eq!(commands.len(), 1);
21525        assert_eq!(commands[0]["queue"], "gpu-workers");
21526        assert_eq!(commands[0]["worker_session"], session.to_wire().unwrap());
21527    }
21528
21529    #[tokio::test]
21530    async fn worker_session_cold_replay_uses_recorded_avro_result() {
21531        let session = WorkerSessionOptions::new("render-1").queue("gpu-workers");
21532        for _replacement in 0..2 {
21533            let ctx = workflow_context(completed_worker_session_history(&session));
21534            let result: i64 = ctx
21535                .activity("render", json!([]))
21536                .in_worker_session(session.clone())
21537                .typed()
21538                .await
21539                .unwrap();
21540            assert_eq!(result, 42);
21541            assert!(ctx.take_commands().unwrap().is_empty());
21542        }
21543    }
21544
21545    #[test]
21546    fn worker_session_cold_replay_rejects_changed_identity_and_lifetime() {
21547        let original = WorkerSessionOptions::new("render-1").queue("gpu-workers");
21548        for changed in [
21549            Some(WorkerSessionOptions::new("render-2").queue("gpu-workers")),
21550            Some(original.clone().ttl_seconds(60)),
21551            Some(original.clone().requirements(["gpu:l4"])),
21552            None,
21553        ] {
21554            let ctx = workflow_context(completed_worker_session_history(&original));
21555            let mut call = ctx.activity_with_options(
21556                "render",
21557                ActivityOptions::new().task_queue("gpu-workers"),
21558                json!([]),
21559            );
21560            if let Some(changed) = changed {
21561                call = call.in_worker_session(changed);
21562            }
21563            let mut call = Box::pin(call);
21564            let mut cx = TaskContext::from_waker(noop_waker_ref());
21565            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
21566                call.as_mut().poll(&mut cx)
21567            else {
21568                panic!("changed session must fail replay");
21569            };
21570            assert_eq!(failure.reason, "activity_worker_session_mismatch");
21571            assert!(ctx.take_commands().unwrap().is_empty());
21572        }
21573    }
21574
21575    #[test]
21576    fn worker_session_local_and_contradictory_queue_routing_emit_no_commands() {
21577        let session = WorkerSessionOptions::new("render-1").queue("gpu-workers");
21578        let ctx = workflow_context(Vec::new());
21579        let mut local = Box::pin(
21580            ctx.local_activity("render", json!([]))
21581                .in_worker_session(session.clone()),
21582        );
21583        let mut cx = TaskContext::from_waker(noop_waker_ref());
21584        assert!(matches!(
21585            local.as_mut().poll(&mut cx),
21586            Poll::Ready(Err(Error::WorkerLoop(_)))
21587        ));
21588        assert!(ctx.take_commands().unwrap().is_empty());
21589        let mut remote = Box::pin(
21590            ctx.activity_with_options(
21591                "render",
21592                ActivityOptions::new().task_queue("other"),
21593                json!([]),
21594            )
21595            .in_worker_session(session),
21596        );
21597        assert!(matches!(
21598            remote.as_mut().poll(&mut cx),
21599            Poll::Ready(Err(Error::WorkerLoop(_)))
21600        ));
21601        assert!(ctx.take_commands().unwrap().is_empty());
21602    }
21603
21604    #[test]
21605    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
21606        let ctx = workflow_context(Vec::new());
21607        let options = ActivityOptions::new().retry_policy(
21608            ActivityRetryPolicy::new(3)
21609                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
21610        );
21611        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
21612        let mut task_context = TaskContext::from_waker(noop_waker_ref());
21613
21614        assert!(matches!(
21615            call.as_mut().poll(&mut task_context),
21616            Poll::Pending
21617        ));
21618        assert_eq!(
21619            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
21620            json!([1, 2])
21621        );
21622    }
21623
21624    #[test]
21625    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
21626        let cases = [
21627            (
21628                ActivityOptions::new().task_queue("  "),
21629                ActivityOptionsErrorKind::EmptyTaskQueue,
21630            ),
21631            (
21632                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
21633                ActivityOptionsErrorKind::EmptyRetryPolicy,
21634            ),
21635            (
21636                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
21637                ActivityOptionsErrorKind::InvalidMaxAttempts,
21638            ),
21639            (
21640                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
21641                    max_attempts: None,
21642                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
21643                    non_retryable_error_types: Vec::new(),
21644                }),
21645                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
21646            ),
21647            (
21648                ActivityOptions::new().retry_policy(
21649                    ActivityRetryPolicy::new(2)
21650                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
21651                ),
21652                ActivityOptionsErrorKind::TooManyBackoffIntervals,
21653            ),
21654            (
21655                ActivityOptions::new().retry_policy(
21656                    ActivityRetryPolicy::new(2).exponential_backoff(
21657                        Duration::from_secs(1),
21658                        0,
21659                        None,
21660                    ),
21661                ),
21662                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
21663            ),
21664            (
21665                ActivityOptions::new()
21666                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
21667                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
21668            ),
21669            (
21670                ActivityOptions::new().retry_policy(
21671                    ActivityRetryPolicy::new(10_002).exponential_backoff(
21672                        Duration::from_secs(1),
21673                        1,
21674                        None,
21675                    ),
21676                ),
21677                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
21678            ),
21679            (
21680                ActivityOptions::new().retry_policy(
21681                    ActivityRetryPolicy::new(2)
21682                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
21683                ),
21684                ActivityOptionsErrorKind::BackoffOverflow,
21685            ),
21686        ];
21687
21688        for (options, expected_kind) in cases {
21689            let ctx = workflow_context(Vec::new());
21690            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
21691            let mut task_context = TaskContext::from_waker(noop_waker_ref());
21692            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
21693                call.as_mut().poll(&mut task_context)
21694            else {
21695                panic!("expected typed activity validation error");
21696            };
21697            assert_eq!(error.kind, expected_kind);
21698            assert!(ctx.take_commands().expect("commands").is_empty());
21699        }
21700    }
21701
21702    #[test]
21703    fn activity_options_validate_positive_and_ordered_timeouts() {
21704        let zero_timeout_cases = [
21705            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
21706            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
21707            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
21708            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
21709        ];
21710        for options in zero_timeout_cases {
21711            assert_eq!(
21712                options.validate().expect_err("zero timeout").kind,
21713                ActivityOptionsErrorKind::TimeoutNotPositive
21714            );
21715        }
21716
21717        let ordering_cases = [
21718            ActivityOptions::new()
21719                .heartbeat_timeout(Duration::from_secs(11))
21720                .start_to_close_timeout(Duration::from_secs(10)),
21721            ActivityOptions::new()
21722                .start_to_close_timeout(Duration::from_secs(31))
21723                .schedule_to_close_timeout(Duration::from_secs(30)),
21724            ActivityOptions::new()
21725                .schedule_to_start_timeout(Duration::from_secs(31))
21726                .schedule_to_close_timeout(Duration::from_secs(30)),
21727        ];
21728        for options in ordering_cases {
21729            assert_eq!(
21730                options.validate().expect_err("timeout order").kind,
21731                ActivityOptionsErrorKind::TimeoutOrder
21732            );
21733        }
21734
21735        assert_eq!(
21736            ActivityOptions::new()
21737                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
21738                .validate()
21739                .expect_err("protocol integer overflow")
21740                .kind,
21741            ActivityOptionsErrorKind::TimeoutOverflow
21742        );
21743    }
21744
21745    #[test]
21746    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
21747        let ctx = workflow_context(completed_retry_activity_history());
21748        let mut call =
21749            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
21750        let mut task_context = TaskContext::from_waker(noop_waker_ref());
21751
21752        assert!(matches!(
21753            call.as_mut().poll(&mut task_context),
21754            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
21755        ));
21756        assert!(ctx.take_commands().expect("commands").is_empty());
21757        ctx.ensure_history_consumed().expect("history consumed");
21758    }
21759
21760    #[test]
21761    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
21762        let mut options = retry_activity_options();
21763        options
21764            .retry_policy
21765            .as_mut()
21766            .expect("retry policy")
21767            .non_retryable_error_types
21768            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
21769
21770        let new_ctx = workflow_context(Vec::new());
21771        let mut new_call =
21772            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
21773        let mut task_context = TaskContext::from_waker(noop_waker_ref());
21774        assert!(matches!(
21775            new_call.as_mut().poll(&mut task_context),
21776            Poll::Pending
21777        ));
21778        let commands = new_ctx.take_commands().expect("commands");
21779        assert_eq!(commands.len(), 1);
21780        assert_eq!(
21781            commands[0]["retry_policy"]["non_retryable_error_types"],
21782            json!(["PermanentError"])
21783        );
21784
21785        let replay_ctx = workflow_context(completed_retry_activity_history());
21786        let mut replay_call =
21787            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
21788        assert!(matches!(
21789            replay_call.as_mut().poll(&mut task_context),
21790            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
21791        ));
21792        assert!(replay_ctx.take_commands().expect("commands").is_empty());
21793        replay_ctx
21794            .ensure_history_consumed()
21795            .expect("history consumed");
21796    }
21797
21798    #[test]
21799    fn replayed_intermediate_retry_remains_pending_across_restarts() {
21800        let history = completed_retry_activity_history()
21801            .into_iter()
21802            .take(3)
21803            .collect::<Vec<_>>();
21804
21805        for _restart in 0..2 {
21806            let ctx = workflow_context(history.clone());
21807            let mut call =
21808                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
21809            let mut task_context = TaskContext::from_waker(noop_waker_ref());
21810            assert!(matches!(
21811                call.as_mut().poll(&mut task_context),
21812                Poll::Pending
21813            ));
21814            assert!(ctx.take_commands().expect("commands").is_empty());
21815        }
21816    }
21817
21818    #[test]
21819    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
21820        let mut changed_queue = retry_activity_options();
21821        changed_queue.task_queue = Some("different-queue".to_string());
21822
21823        let mut changed_max_attempts = retry_activity_options();
21824        let retry_policy = changed_max_attempts
21825            .retry_policy
21826            .as_mut()
21827            .expect("retry policy");
21828        retry_policy.max_attempts = Some(4);
21829
21830        let mut changed_backoff = retry_activity_options();
21831        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
21832        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
21833            Duration::from_secs(3),
21834            Duration::from_secs(4),
21835        ]));
21836
21837        let mut changed_non_retryable_types = retry_activity_options();
21838        let retry_policy = changed_non_retryable_types
21839            .retry_policy
21840            .as_mut()
21841            .expect("retry policy");
21842        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
21843
21844        let mut changed_start_to_close = retry_activity_options();
21845        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
21846        let mut changed_schedule_to_start = retry_activity_options();
21847        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
21848        let mut changed_schedule_to_close = retry_activity_options();
21849        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
21850        let mut changed_heartbeat = retry_activity_options();
21851        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
21852
21853        let cases = [
21854            (changed_queue, "activity_task_queue_mismatch"),
21855            (changed_max_attempts, "activity_retry_policy_mismatch"),
21856            (changed_backoff, "activity_retry_policy_mismatch"),
21857            (
21858                changed_non_retryable_types,
21859                "activity_retry_policy_mismatch",
21860            ),
21861            (changed_start_to_close, "activity_retry_policy_mismatch"),
21862            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
21863            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
21864            (changed_heartbeat, "activity_retry_policy_mismatch"),
21865        ];
21866
21867        for (options, expected_reason) in cases {
21868            let ctx = workflow_context(completed_retry_activity_history());
21869            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
21870            let mut task_context = TaskContext::from_waker(noop_waker_ref());
21871            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
21872                call.as_mut().poll(&mut task_context)
21873            else {
21874                panic!("changed activity options must fail replay");
21875            };
21876            assert_eq!(failure.reason, expected_reason);
21877            assert_eq!(failure.sequence, Some(1));
21878            assert!(ctx.take_commands().expect("commands").is_empty());
21879        }
21880    }
21881
21882    #[test]
21883    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
21884        let cases = [
21885            (
21886                "execution_mode",
21887                json!("local"),
21888                "activity_execution_mode_mismatch",
21889            ),
21890            (
21891                "snapshot_version",
21892                json!(2),
21893                "activity_retry_policy_mismatch",
21894            ),
21895        ];
21896
21897        for (field, value, expected_reason) in cases {
21898            let mut history = completed_retry_activity_history();
21899            let activity = history[0].payload["activity"]
21900                .as_object_mut()
21901                .expect("activity snapshot");
21902            if field == "execution_mode" {
21903                activity.insert(field.to_string(), value);
21904            } else {
21905                activity["retry_policy"]
21906                    .as_object_mut()
21907                    .expect("retry snapshot")
21908                    .insert(field.to_string(), value);
21909            }
21910
21911            let ctx = workflow_context(history);
21912            let mut call =
21913                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
21914            let mut task_context = TaskContext::from_waker(noop_waker_ref());
21915            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
21916                call.as_mut().poll(&mut task_context)
21917            else {
21918                panic!("changed {field} must fail replay");
21919            };
21920            assert_eq!(failure.reason, expected_reason);
21921            assert_eq!(failure.sequence, Some(1));
21922            assert!(ctx.take_commands().expect("commands").is_empty());
21923        }
21924    }
21925
21926    #[test]
21927    fn remote_activity_rejects_local_history_markers_without_option_snapshot() {
21928        for (nested, field, value, terminal_only) in [
21929            (false, "execution_mode", json!("local"), false),
21930            (false, "local_activity", json!(true), false),
21931            (true, "local_activity", json!(true), false),
21932            (false, "local_activity", json!(true), true),
21933        ] {
21934            let mut history = completed_retry_activity_history();
21935            history[0]
21936                .payload
21937                .as_object_mut()
21938                .unwrap()
21939                .remove("activity");
21940            let event = if terminal_only {
21941                history.last_mut().unwrap()
21942            } else {
21943                &mut history[0]
21944            };
21945            if nested {
21946                event.payload["activity"] = json!({(field): value});
21947            } else {
21948                event.payload[field] = value;
21949            }
21950            let ctx = workflow_context(history);
21951            let mut call =
21952                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
21953            let mut task_context = TaskContext::from_waker(noop_waker_ref());
21954            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
21955                call.as_mut().poll(&mut task_context)
21956            else {
21957                panic!("local history must not replay as a remote activity: {nested}/{field}/{terminal_only}");
21958            };
21959            assert_eq!(failure.reason, "activity_execution_mode_mismatch");
21960            assert_eq!(failure.sequence, Some(1));
21961            assert!(ctx.take_commands().expect("commands").is_empty());
21962        }
21963    }
21964
21965    #[test]
21966    fn activity_history_rejects_conflicting_local_execution_markers() {
21967        let mut history = completed_retry_activity_history();
21968        history.last_mut().unwrap().payload["local_activity"] = json!(true);
21969        let error = WorkflowState::new(history, "queue".into(), DEFAULT_CODEC.into(), None)
21970            .expect_err("remote schedule cannot become a local completion");
21971        assert!(
21972            matches!(error, Error::NonDeterministicReplay(ReplayFailure { reason, sequence: Some(1), .. })
21973            if reason == "activity_execution_mode_conflict")
21974        );
21975    }
21976
21977    #[test]
21978    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
21979        let mut history = completed_retry_activity_history();
21980        let activity = history[0].payload["activity"]
21981            .as_object_mut()
21982            .expect("activity snapshot");
21983        activity.remove("execution_mode");
21984        activity.remove("retry_policy");
21985
21986        let mut current = retry_activity_options();
21987        current.start_to_close_timeout = Some(Duration::from_secs(45));
21988        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
21989        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
21990        current.heartbeat_timeout = Some(Duration::from_secs(12));
21991
21992        let ctx = workflow_context(history);
21993        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
21994        let mut task_context = TaskContext::from_waker(noop_waker_ref());
21995        assert!(matches!(
21996            call.as_mut().poll(&mut task_context),
21997            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
21998        ));
21999        assert!(ctx.take_commands().expect("commands").is_empty());
22000        ctx.ensure_history_consumed().expect("history consumed");
22001    }
22002
22003    #[test]
22004    fn terminal_activity_failed_after_start_returns_typed_failure() {
22005        let history = vec![
22006            history_event(
22007                "ActivityScheduled",
22008                json!({
22009                    "sequence": 1,
22010                    "activity_type": "flaky",
22011                    "activity_execution_id": "act-terminal",
22012                    "activity": {
22013                        "id": "act-terminal",
22014                        "sequence": 1,
22015                        "type": "flaky",
22016                        "queue": "critical-activities",
22017                        "retry_policy": {
22018                            "snapshot_version": 1,
22019                            "max_attempts": 3,
22020                            "backoff_seconds": [2, 4],
22021                            "non_retryable_error_types": ["PermanentError"]
22022                        }
22023                    }
22024                }),
22025            ),
22026            history_event(
22027                "ActivityStarted",
22028                json!({
22029                    "sequence": 1,
22030                    "activity_type": "flaky",
22031                    "activity_execution_id": "act-terminal",
22032                    "activity_attempt_id": "attempt-1",
22033                    "attempt_number": 1
22034                }),
22035            ),
22036            history_event(
22037                "ActivityFailed",
22038                json!({
22039                    "sequence": 1,
22040                    "activity_type": "flaky",
22041                    "activity_execution_id": "act-terminal",
22042                    "activity_attempt_id": "attempt-1",
22043                    "attempt_number": 1,
22044                    "failure_id": "failure-terminal",
22045                    "failure_category": "activity",
22046                    "exception_type": "PermanentError",
22047                    "message": "cannot retry",
22048                    "non_retryable": true
22049                }),
22050            ),
22051        ];
22052        let ctx = workflow_context(history);
22053        let mut call =
22054            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
22055        let mut task_context = TaskContext::from_waker(noop_waker_ref());
22056
22057        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
22058            call.as_mut().poll(&mut task_context)
22059        else {
22060            panic!("terminal ActivityFailed must settle the activity future");
22061        };
22062        assert_eq!(failure.kind, ActivityFailureKind::Failed);
22063        assert_eq!(
22064            failure.activity_execution_id.as_deref(),
22065            Some("act-terminal")
22066        );
22067        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
22068        assert!(failure.non_retryable);
22069        assert!(ctx.take_commands().expect("commands").is_empty());
22070        ctx.ensure_history_consumed().expect("history consumed");
22071    }
22072
22073    #[test]
22074    fn activity_terminal_events_return_machine_readable_failures() {
22075        let cases = [
22076            (
22077                "ActivityFailed",
22078                json!({
22079                    "sequence": 1,
22080                    "activity_type": "charge-card",
22081                    "activity_execution_id": "act-1",
22082                    "activity_attempt_id": "attempt-2",
22083                    "attempt_number": 2,
22084                    "failure_id": "failure-1",
22085                    "failure_category": "activity",
22086                    "exception_type": "PaymentDeclined",
22087                    "exception_class": "payments.PaymentDeclined",
22088                    "message": "card declined",
22089                    "non_retryable": true
22090                }),
22091                ActivityFailureKind::Failed,
22092                "activity",
22093            ),
22094            (
22095                "ActivityCancelled",
22096                json!({
22097                    "sequence": 1,
22098                    "activity_type": "charge-card",
22099                    "activity_execution_id": "act-1",
22100                    "activity_attempt_id": "attempt-1"
22101                }),
22102                ActivityFailureKind::Cancelled,
22103                "cancelled",
22104            ),
22105        ];
22106
22107        for (event_type, payload, expected_kind, expected_reason) in cases {
22108            let ctx = workflow_context(vec![history_event(event_type, payload)]);
22109            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
22110            let mut task_context = TaskContext::from_waker(noop_waker_ref());
22111            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
22112                call.as_mut().poll(&mut task_context)
22113            else {
22114                panic!("expected terminal activity failure");
22115            };
22116            assert_eq!(failure.kind, expected_kind);
22117            assert_eq!(failure.reason, expected_reason);
22118            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
22119            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
22120        }
22121    }
22122
22123    #[test]
22124    fn every_activity_timeout_class_is_typed() {
22125        for timeout_kind in [
22126            "start_to_close",
22127            "schedule_to_start",
22128            "schedule_to_close",
22129            "heartbeat",
22130        ] {
22131            let ctx = workflow_context(vec![history_event(
22132                "ActivityTimedOut",
22133                json!({
22134                    "sequence": 1,
22135                    "activity_type": "slow",
22136                    "activity_execution_id": "act-timeout",
22137                    "activity_attempt_id": "attempt-timeout",
22138                    "failure_category": "timeout",
22139                    "timeout_kind": timeout_kind,
22140                    "message": "deadline expired"
22141                }),
22142            )]);
22143            let mut call = Box::pin(ctx.activity("slow", json!([])));
22144            let mut task_context = TaskContext::from_waker(noop_waker_ref());
22145            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
22146                call.as_mut().poll(&mut task_context)
22147            else {
22148                panic!("expected timeout failure");
22149            };
22150            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
22151            assert_eq!(failure.reason, timeout_kind);
22152            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
22153            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
22154        }
22155    }
22156
22157    #[test]
22158    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
22159        let ctx = workflow_context(Vec::new());
22160        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
22161        let mut task_context = TaskContext::from_waker(noop_waker_ref());
22162
22163        assert!(matches!(
22164            sleep.as_mut().poll(&mut task_context),
22165            Poll::Pending
22166        ));
22167        assert!(matches!(
22168            sleep.as_mut().poll(&mut task_context),
22169            Poll::Pending
22170        ));
22171
22172        let commands = ctx.take_commands().expect("timer command");
22173        assert_eq!(
22174            commands,
22175            vec![json!({
22176                "type": "start_timer",
22177                "delay_seconds": 2,
22178            })]
22179        );
22180    }
22181
22182    #[test]
22183    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
22184        let history = vec![
22185            history_event(
22186                "TimerScheduled",
22187                json!({
22188                    "sequence": 1,
22189                    "timer_id": "timer-1",
22190                    "delay_seconds": 5,
22191                    "fire_at": "2026-07-11T12:00:05Z",
22192                }),
22193            ),
22194            history_event(
22195                "TimerFired",
22196                json!({
22197                    "sequence": 1,
22198                    "timer_id": "timer-1",
22199                    "delay_seconds": 5,
22200                    "fire_at": "2026-07-11T12:00:05Z",
22201                    "fired_at": "2026-07-11T12:00:05Z",
22202                }),
22203            ),
22204        ];
22205
22206        for _restart in 0..2 {
22207            let ctx = workflow_context(history.clone());
22208            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
22209            let mut task_context = TaskContext::from_waker(noop_waker_ref());
22210            assert!(matches!(
22211                sleep.as_mut().poll(&mut task_context),
22212                Poll::Ready(Ok(()))
22213            ));
22214            assert!(ctx.take_commands().expect("commands").is_empty());
22215            ctx.ensure_history_consumed().expect("history consumed");
22216        }
22217    }
22218
22219    #[test]
22220    fn workflow_sleep_rejects_changed_delay_during_replay() {
22221        let ctx = workflow_context(vec![
22222            history_event(
22223                "TimerScheduled",
22224                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
22225            ),
22226            history_event(
22227                "TimerFired",
22228                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
22229            ),
22230        ]);
22231        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
22232        let mut task_context = TaskContext::from_waker(noop_waker_ref());
22233
22234        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
22235            sleep.as_mut().poll(&mut task_context)
22236        else {
22237            panic!("changed timer delay must be rejected");
22238        };
22239        assert_eq!(failure.reason, "timer_delay_mismatch");
22240        assert_eq!(failure.sequence, Some(1));
22241    }
22242
22243    #[test]
22244    fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
22245        let ctx = workflow_context(Vec::new());
22246        let mut wait = Box::pin(
22247            ctx.wait_condition(
22248                ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
22249                    .timeout(Duration::from_millis(60_001)),
22250                || Ok(false),
22251            ),
22252        );
22253        let mut task_context = TaskContext::from_waker(noop_waker_ref());
22254
22255        assert!(matches!(
22256            wait.as_mut().poll(&mut task_context),
22257            Poll::Pending
22258        ));
22259        assert!(matches!(
22260            wait.as_mut().poll(&mut task_context),
22261            Poll::Pending
22262        ));
22263        assert_eq!(
22264            ctx.take_commands().expect("condition command"),
22265            vec![json!({
22266                "type": "open_condition_wait",
22267                "condition_wait_occurrence_id": "rust:condition-wait:0",
22268                "condition_key": "approval.ready",
22269                "condition_definition_fingerprint": "sha256:approval-v1",
22270                "timeout_seconds": 61,
22271            })]
22272        );
22273    }
22274
22275    #[test]
22276    fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
22277        let ctx = workflow_context(Vec::new());
22278        let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
22279        let mut task_context = TaskContext::from_waker(noop_waker_ref());
22280        assert!(matches!(
22281            satisfied.as_mut().poll(&mut task_context),
22282            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22283        ));
22284
22285        let mut timed_out = Box::pin(wait_condition!(
22286            ctx,
22287            "no-wait",
22288            timeout: Duration::ZERO,
22289            || Ok(false),
22290        ));
22291        assert!(matches!(
22292            timed_out.as_mut().poll(&mut task_context),
22293            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
22294        ));
22295        assert!(ctx.take_commands().expect("commands").is_empty());
22296    }
22297
22298    #[test]
22299    fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
22300        let signal_history = vec![
22301            history_event(
22302                "ConditionWaitOpened",
22303                json!({
22304                    "sequence": 4,
22305                    "condition_wait_id": "condition:4",
22306                    "condition_wait_occurrence_id": "rust:condition-wait:0",
22307                    "condition_key": "approval",
22308                    "condition_definition_fingerprint": "sha256:approval-v1",
22309                    "timeout_seconds": 30,
22310                }),
22311            ),
22312            history_event(
22313                "SignalReceived",
22314                json!({
22315                    "workflow_sequence": 4,
22316                    "signal_name": "approve",
22317                    "arguments": fixture_envelope(json!(["Ada"])),
22318                }),
22319            ),
22320        ];
22321        for _worker_before_or_after_restart in 0..2 {
22322            let ctx = workflow_context(signal_history.clone());
22323            let predicate_ctx = ctx.clone();
22324            let mut wait = Box::pin(
22325                ctx.wait_condition(
22326                    ConditionWaitOptions::new("approval", "sha256:approval-v1")
22327                        .timeout(Duration::from_secs(30)),
22328                    move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
22329                ),
22330            );
22331            let mut task_context = TaskContext::from_waker(noop_waker_ref());
22332            assert!(matches!(
22333                wait.as_mut().poll(&mut task_context),
22334                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22335            ));
22336            assert!(ctx.take_commands().expect("commands").is_empty());
22337            ctx.ensure_history_consumed().expect("condition consumed");
22338        }
22339
22340        let update_history = vec![
22341            history_event(
22342                "ConditionWaitOpened",
22343                json!({
22344                    "sequence": 7,
22345                    "condition_wait_id": "condition:7",
22346                    "condition_wait_occurrence_id": "rust:condition-wait:0",
22347                    "condition_key": "update-approval",
22348                    "condition_definition_fingerprint": "sha256:update-approval-v1",
22349                }),
22350            ),
22351            history_event(
22352                "UpdateApplied",
22353                json!({
22354                    "sequence": 7,
22355                    "update_id": "update-1",
22356                    "update_name": "approve",
22357                    "arguments": fixture_envelope(json!([true])),
22358                }),
22359            ),
22360        ];
22361        let ctx = workflow_context(update_history);
22362        let predicate_ctx = ctx.clone();
22363        let mut wait = Box::pin(ctx.wait_condition(
22364            ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
22365            move || {
22366                Ok(predicate_ctx
22367                    .updates("approve")?
22368                    .first()
22369                    .and_then(|arguments| arguments.first())
22370                    .and_then(Value::as_bool)
22371                    == Some(true))
22372            },
22373        ));
22374        let mut task_context = TaskContext::from_waker(noop_waker_ref());
22375        assert!(matches!(
22376            wait.as_mut().poll(&mut task_context),
22377            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22378        ));
22379        assert!(ctx.take_commands().expect("commands").is_empty());
22380        ctx.ensure_history_consumed().expect("condition consumed");
22381    }
22382
22383    #[test]
22384    fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
22385        let open_history = vec![
22386            history_event(
22387                "ConditionWaitOpened",
22388                json!({
22389                    "sequence": 3,
22390                    "condition_wait_id": "condition:3",
22391                    "condition_wait_occurrence_id": "rust:condition-wait:0",
22392                    "condition_key": "two-votes",
22393                    "condition_definition_fingerprint": "sha256:two-votes-v1",
22394                    "timeout_seconds": 120,
22395                }),
22396            ),
22397            history_event(
22398                "SignalReceived",
22399                json!({
22400                    "workflow_sequence": 3,
22401                    "signal_name": "vote",
22402                    "arguments": fixture_envelope(json!(["first"])),
22403                }),
22404            ),
22405        ];
22406        for _worker_before_or_after_restart in 0..2 {
22407            let ctx = workflow_context(open_history.clone());
22408            let predicate_ctx = ctx.clone();
22409            let mut wait = Box::pin(
22410                ctx.wait_condition(
22411                    ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
22412                        .timeout(Duration::from_secs(120)),
22413                    move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
22414                ),
22415            );
22416            let mut task_context = TaskContext::from_waker(noop_waker_ref());
22417            assert!(matches!(
22418                wait.as_mut().poll(&mut task_context),
22419                Poll::Pending
22420            ));
22421            assert_eq!(
22422                ctx.take_commands().expect("reopened condition"),
22423                vec![json!({
22424                    "type": "open_condition_wait",
22425                    "condition_wait_occurrence_id": "rust:condition-wait:0",
22426                    "condition_key": "two-votes",
22427                    "condition_definition_fingerprint": "sha256:two-votes-v1",
22428                    "timeout_seconds": 120,
22429                })]
22430            );
22431        }
22432
22433        let satisfied_ctx = workflow_context(vec![
22434            history_event(
22435                "ConditionWaitOpened",
22436                json!({
22437                    "sequence": 5,
22438                    "condition_wait_id": "condition:5",
22439                    "condition_wait_occurrence_id": "rust:condition-wait:0",
22440                    "condition_key": "approval",
22441                    "condition_definition_fingerprint": "sha256:approval-v1",
22442                }),
22443            ),
22444            history_event(
22445                "ConditionWaitSatisfied",
22446                json!({
22447                    "sequence": 5,
22448                    "condition_wait_id": "condition:5",
22449                    "condition_wait_occurrence_id": "rust:condition-wait:0",
22450                    "condition_key": "approval",
22451                    "condition_definition_fingerprint": "sha256:approval-v1",
22452                }),
22453            ),
22454        ]);
22455        let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
22456            ConditionWaitOptions::new("approval", "sha256:approval-v1"),
22457            || Ok(false),
22458        ));
22459        let mut task_context = TaskContext::from_waker(noop_waker_ref());
22460        assert!(matches!(
22461            satisfied.as_mut().poll(&mut task_context),
22462            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22463        ));
22464
22465        let timed_out_ctx = workflow_context(vec![
22466            history_event(
22467                "ConditionWaitOpened",
22468                json!({
22469                    "sequence": 8,
22470                    "condition_wait_id": "condition:8",
22471                    "condition_wait_occurrence_id": "rust:condition-wait:0",
22472                    "condition_key": "approval-timeout",
22473                    "condition_definition_fingerprint": "sha256:approval-timeout-v1",
22474                    "timeout_seconds": 5,
22475                }),
22476            ),
22477            history_event(
22478                "TimerScheduled",
22479                json!({
22480                    "sequence": 9,
22481                    "timer_id": "condition-timer:9",
22482                    "timer_kind": "condition_timeout",
22483                    "condition_wait_id": "condition:8",
22484                    "delay_seconds": 5,
22485                }),
22486            ),
22487            history_event(
22488                "TimerFired",
22489                json!({
22490                    "sequence": 9,
22491                    "timer_id": "condition-timer:9",
22492                    "timer_kind": "condition_timeout",
22493                    "condition_wait_id": "condition:8",
22494                    "delay_seconds": 5,
22495                }),
22496            ),
22497        ]);
22498        let mut timed_out = Box::pin(
22499            timed_out_ctx.wait_condition(
22500                ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
22501                    .timeout(Duration::from_secs(5)),
22502                || Ok(true),
22503            ),
22504        );
22505        assert!(matches!(
22506            timed_out.as_mut().poll(&mut task_context),
22507            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
22508        ));
22509    }
22510
22511    #[test]
22512    fn satisfied_reopened_selection_condition_waits_for_its_canonical_winner() {
22513        let ctx = workflow_context(reopened_selection_condition_history());
22514        {
22515            let state = ctx.state.lock().unwrap();
22516            assert!(matches!(
22517                selection_member_recorded_command(&state, 2),
22518                Some(RecordedCommand::ConditionWait {
22519                    sequence: 3,
22520                    result: None,
22521                    ..
22522                })
22523            ));
22524        }
22525        let mut selected = Box::pin(ctx.select_keyed(vec![
22526            ("timer", ParallelOperation::timer(Duration::from_secs(300))),
22527            (
22528                "votes",
22529                ParallelOperation::condition(
22530                    ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
22531                    || Ok(true),
22532                ),
22533            ),
22534        ]));
22535        let mut task_context = TaskContext::from_waker(noop_waker_ref());
22536        assert!(selected.as_mut().poll(&mut task_context).is_pending());
22537        assert!(
22538            ctx.take_commands().unwrap().is_empty(),
22539            "true predicate must not reopen the recorded wait"
22540        );
22541        ctx.ensure_history_consumed()
22542            .expect("physical reopens are consumed while canonical selection is pending");
22543    }
22544
22545    #[test]
22546    fn reopened_selection_condition_replays_its_latest_physical_winner() {
22547        for (terminal_type, acknowledge_timeout) in [
22548            ("ConditionWaitSatisfied", false),
22549            ("ConditionWaitTimedOut", false),
22550            ("TimerFired", false),
22551            ("TimerFired", true),
22552        ] {
22553            let mut history = reopened_selection_condition_history();
22554            let original_identity = history[1].payload["condition_wait_id"].clone();
22555            let uses_timer = terminal_type == "TimerFired";
22556            let mut resolution_payload = history[3].payload.clone();
22557            if uses_timer {
22558                for event in &mut history[1..] {
22559                    event.payload["timeout_seconds"] = json!(3);
22560                }
22561                resolution_payload = history[3].payload.clone();
22562                resolution_payload["timer_kind"] = json!("condition_timeout");
22563                resolution_payload["timer_id"] = json!("condition-timeout-3");
22564                resolution_payload["delay_seconds"] = json!(3);
22565                history.push(history_event("TimerScheduled", resolution_payload.clone()));
22566            }
22567            let mut resolution = history_event(terminal_type, resolution_payload.clone());
22568            resolution
22569                .raw
22570                .insert("id".into(), json!("latest-condition-resolution"));
22571            history.push(resolution);
22572            history.push(history_event(
22573                "SelectionResolved",
22574                json!({
22575                    "selection_group_id": "select-calls:1:2",
22576                    "selection_group_base_sequence": 1,
22577                    "selection_group_size": 2,
22578                    "member_key": "votes",
22579                    "member_index": 1,
22580                    "member_base_sequence": 2,
22581                    "member_size": 1,
22582                    "operation_kind": "condition",
22583                    "operation_identity": original_identity,
22584                    "outcome": "completed",
22585                    "resolution_event_id": "latest-condition-resolution",
22586                    "resolution_event_type": terminal_type,
22587                }),
22588            ));
22589            if acknowledge_timeout {
22590                history.push(history_event("ConditionWaitTimedOut", resolution_payload));
22591            }
22592            let ctx = workflow_context(history);
22593            let mut selected = Box::pin(ctx.select_keyed(vec![
22594                ("timer", ParallelOperation::timer(Duration::from_secs(300))),
22595                (
22596                    "votes",
22597                    ParallelOperation::condition(
22598                        if uses_timer {
22599                            ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
22600                                .timeout(Duration::from_secs(3))
22601                        } else {
22602                            ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
22603                        },
22604                        || Ok(true),
22605                    ),
22606                ),
22607            ]));
22608            let mut task_context = TaskContext::from_waker(noop_waker_ref());
22609            match selected.as_mut().poll(&mut task_context) {
22610                Poll::Ready(Ok(result)) => {
22611                    assert_eq!(result.key, SelectionKey::Name("votes".into()));
22612                    let expected = if terminal_type == "ConditionWaitSatisfied" {
22613                        ConditionWaitResult::Satisfied
22614                    } else {
22615                        ConditionWaitResult::TimedOut
22616                    };
22617                    assert_eq!(result.value, Some(ParallelResult::Condition(expected)));
22618                }
22619                other => panic!("latest physical winner must replay: {other:?}"),
22620            }
22621            assert!(ctx.take_commands().unwrap().is_empty());
22622            ctx.ensure_history_consumed()
22623                .expect("winner history consumed");
22624        }
22625    }
22626
22627    fn reopened_selection_condition_history() -> Vec<HistoryEvent> {
22628        let fixture: Value = serde_json::from_str(include_str!(
22629            "../tests/fixtures/replay-regressions/grouped-condition-physical-reopen.json"
22630        ))
22631        .expect("grouped condition fixture");
22632        fixture["history"]
22633            .as_array()
22634            .unwrap()
22635            .iter()
22636            .map(|event| {
22637                let mut payload = event["payload"].clone();
22638                let index = payload["parallel_group_index"].as_u64().unwrap();
22639                payload["parallel_group_id"] = json!("select-calls:1:2");
22640                payload["parallel_group_mode"] = json!("select");
22641                payload["selection_member_key"] = json!(if index == 0 { "timer" } else { "votes" });
22642                payload["selection_member_index"] = json!(index);
22643                payload["selection_member_base_sequence"] = json!(index + 1);
22644                payload["selection_member_size"] = json!(1);
22645                payload["selection_member_kind"] =
22646                    json!(if index == 0 { "timer" } else { "condition" });
22647                history_event(event["event_type"].as_str().unwrap(), payload)
22648            })
22649            .collect()
22650    }
22651
22652    #[test]
22653    fn grouped_condition_replay_rejects_unproven_physical_reopens() {
22654        let fixture: Value = serde_json::from_str(include_str!(
22655            "../tests/fixtures/replay-regressions/grouped-condition-physical-reopen.json"
22656        ))
22657        .expect("grouped condition fixture");
22658        let original = fixture["history"].as_array().expect("fixture history");
22659        let decode = |events: &[Value]| {
22660            let history = events
22661                .iter()
22662                .map(|event| {
22663                    history_event(
22664                        event["event_type"].as_str().unwrap(),
22665                        event["payload"].clone(),
22666                    )
22667                })
22668                .collect::<Vec<_>>();
22669            recorded_commands(
22670                &history,
22671                DEFAULT_CODEC,
22672                WorkflowIdentity {
22673                    workflow_id: None,
22674                    run_id: None,
22675                },
22676            )
22677        };
22678        assert!(decode(original).is_ok());
22679        for field in [
22680            "condition_wait_occurrence_id",
22681            "condition_key",
22682            "condition_definition_fingerprint",
22683            "timeout_seconds",
22684            "parallel_group_id",
22685            "parallel_group_size",
22686            "parallel_group_index",
22687        ] {
22688            let mut changed = original.clone();
22689            changed[3]["payload"][field] = match field {
22690                "timeout_seconds" | "parallel_group_size" => json!(30),
22691                "parallel_group_index" => json!(0),
22692                _ => json!("changed"),
22693            };
22694            assert!(
22695                decode(&changed).is_err(),
22696                "reopen must reject changed {field}"
22697            );
22698        }
22699        let mut unsettled = original.clone();
22700        unsettled.remove(2);
22701        assert!(
22702            decode(&unsettled).is_err(),
22703            "physical predecessor must be resolved"
22704        );
22705        let mut timed_out = original.clone();
22706        timed_out[2]["event_type"] = json!("ConditionWaitTimedOut");
22707        assert!(decode(&timed_out).is_err(), "timeout is terminal");
22708        let mut late_resolution = original.clone();
22709        late_resolution.swap(2, 3);
22710        assert!(
22711            decode(&late_resolution).is_err(),
22712            "later history cannot authorize a reopen"
22713        );
22714    }
22715
22716    #[test]
22717    fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
22718        let history = vec![
22719            history_event(
22720                "ConditionWaitOpened",
22721                json!({
22722                    "sequence": 3,
22723                    "condition_wait_id": "condition:3",
22724                    "condition_wait_occurrence_id": "rust:condition-wait:0",
22725                    "condition_key": "two-votes",
22726                    "condition_definition_fingerprint": "sha256:two-votes-v1",
22727                }),
22728            ),
22729            history_event(
22730                "SignalReceived",
22731                json!({
22732                    "workflow_sequence": 3,
22733                    "signal_name": "vote",
22734                    "arguments": fixture_envelope(json!(["first"])),
22735                }),
22736            ),
22737            history_event(
22738                "ConditionWaitSatisfied",
22739                json!({
22740                    "sequence": 3,
22741                    "condition_wait_id": "condition:3",
22742                    "condition_wait_occurrence_id": "rust:condition-wait:0",
22743                    "condition_key": "two-votes",
22744                    "condition_definition_fingerprint": "sha256:two-votes-v1",
22745                }),
22746            ),
22747            history_event(
22748                "ConditionWaitOpened",
22749                json!({
22750                    "sequence": 5,
22751                    "condition_wait_id": "condition:5",
22752                    "condition_wait_occurrence_id": "rust:condition-wait:0",
22753                    "condition_key": "two-votes",
22754                    "condition_definition_fingerprint": "sha256:two-votes-v1",
22755                }),
22756            ),
22757            history_event(
22758                "SignalReceived",
22759                json!({
22760                    "workflow_sequence": 5,
22761                    "signal_name": "vote",
22762                    "arguments": fixture_envelope(json!(["second"])),
22763                }),
22764            ),
22765            history_event(
22766                "ConditionWaitSatisfied",
22767                json!({
22768                    "sequence": 5,
22769                    "condition_wait_id": "condition:5",
22770                    "condition_wait_occurrence_id": "rust:condition-wait:0",
22771                    "condition_key": "two-votes",
22772                    "condition_definition_fingerprint": "sha256:two-votes-v1",
22773                }),
22774            ),
22775        ];
22776        for _cold_worker_or_restart in 0..2 {
22777            let ctx = workflow_context(history.clone());
22778            let predicate_ctx = ctx.clone();
22779            let mut wait = Box::pin(ctx.wait_condition(
22780                ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
22781                move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
22782            ));
22783            let mut task_context = TaskContext::from_waker(noop_waker_ref());
22784
22785            assert!(matches!(
22786                wait.as_mut().poll(&mut task_context),
22787                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22788            ));
22789            assert!(ctx.take_commands().expect("commands").is_empty());
22790            ctx.ensure_history_consumed()
22791                .expect("every physical wait-open is consumed");
22792        }
22793    }
22794
22795    #[test]
22796    fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
22797        let history = vec![
22798            history_event(
22799                "ConditionWaitOpened",
22800                json!({
22801                    "sequence": 3,
22802                    "condition_wait_id": "condition:3",
22803                    "condition_wait_occurrence_id": "rust:condition-wait:0",
22804                    "condition_key": "approved",
22805                    "condition_definition_fingerprint": "sha256:approved-v1",
22806                }),
22807            ),
22808            history_event(
22809                "UpdateApplied",
22810                json!({
22811                    "sequence": 3,
22812                    "update_id": "update-1",
22813                    "update_name": "approve",
22814                    "arguments": fixture_envelope(json!([false])),
22815                }),
22816            ),
22817            history_event(
22818                "ConditionWaitOpened",
22819                json!({
22820                    "sequence": 5,
22821                    "condition_wait_id": "condition:5",
22822                    "condition_wait_occurrence_id": "rust:condition-wait:0",
22823                    "condition_key": "approved",
22824                    "condition_definition_fingerprint": "sha256:approved-v1",
22825                }),
22826            ),
22827            history_event(
22828                "UpdateApplied",
22829                json!({
22830                    "sequence": 5,
22831                    "update_id": "update-2",
22832                    "update_name": "approve",
22833                    "arguments": fixture_envelope(json!([true])),
22834                }),
22835            ),
22836        ];
22837
22838        for _cold_worker_or_restart in 0..2 {
22839            let ctx = workflow_context(history.clone());
22840            let predicate_ctx = ctx.clone();
22841            let mut wait = Box::pin(ctx.wait_condition(
22842                ConditionWaitOptions::new("approved", "sha256:approved-v1"),
22843                move || {
22844                    Ok(predicate_ctx
22845                        .updates("approve")?
22846                        .last()
22847                        .and_then(|arguments| arguments.first())
22848                        .and_then(Value::as_bool)
22849                        == Some(true))
22850                },
22851            ));
22852            let mut task_context = TaskContext::from_waker(noop_waker_ref());
22853
22854            assert!(matches!(
22855                wait.as_mut().poll(&mut task_context),
22856                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22857            ));
22858            assert!(ctx.take_commands().expect("commands").is_empty());
22859            ctx.ensure_history_consumed()
22860                .expect("every update-driven reopen is consumed");
22861        }
22862    }
22863
22864    #[test]
22865    fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
22866        for (first_key, first_fingerprint, second_key, second_fingerprint) in [
22867            ("shared", "sha256:first", "shared", "sha256:second"),
22868            ("first", "sha256:shared", "second", "sha256:shared"),
22869            ("shared", "sha256:shared", "shared", "sha256:shared"),
22870            ("first", "sha256:first", "second", "sha256:second"),
22871        ] {
22872            let history = vec![
22873                history_event(
22874                    "ConditionWaitOpened",
22875                    json!({
22876                        "sequence": 3,
22877                        "condition_wait_id": "condition:3",
22878                        "condition_wait_occurrence_id": "rust:condition-wait:0",
22879                        "condition_key": first_key,
22880                        "condition_definition_fingerprint": first_fingerprint,
22881                    }),
22882                ),
22883                history_event(
22884                    "ConditionWaitSatisfied",
22885                    json!({
22886                        "sequence": 3,
22887                        "condition_wait_id": "condition:3",
22888                        "condition_wait_occurrence_id": "rust:condition-wait:0",
22889                        "condition_key": first_key,
22890                        "condition_definition_fingerprint": first_fingerprint,
22891                    }),
22892                ),
22893                history_event(
22894                    "ConditionWaitOpened",
22895                    json!({
22896                        "sequence": 4,
22897                        "condition_wait_id": "condition:4",
22898                        "condition_wait_occurrence_id": "rust:condition-wait:1",
22899                        "condition_key": second_key,
22900                        "condition_definition_fingerprint": second_fingerprint,
22901                    }),
22902                ),
22903                history_event(
22904                    "ConditionWaitSatisfied",
22905                    json!({
22906                        "sequence": 4,
22907                        "condition_wait_id": "condition:4",
22908                        "condition_wait_occurrence_id": "rust:condition-wait:1",
22909                        "condition_key": second_key,
22910                        "condition_definition_fingerprint": second_fingerprint,
22911                    }),
22912                ),
22913            ];
22914            for _cold_worker_or_restart in 0..2 {
22915                let ctx = workflow_context(history.clone());
22916                let mut task_context = TaskContext::from_waker(noop_waker_ref());
22917                let mut first = Box::pin(ctx.wait_condition(
22918                    ConditionWaitOptions::new(first_key, first_fingerprint),
22919                    || Ok(false),
22920                ));
22921                assert!(matches!(
22922                    first.as_mut().poll(&mut task_context),
22923                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22924                ));
22925
22926                let mut second = Box::pin(ctx.wait_condition(
22927                    ConditionWaitOptions::new(second_key, second_fingerprint),
22928                    || Ok(false),
22929                ));
22930                assert!(matches!(
22931                    second.as_mut().poll(&mut task_context),
22932                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22933                ));
22934                assert!(ctx.take_commands().expect("commands").is_empty());
22935                ctx.ensure_history_consumed()
22936                    .expect("each authored wait consumes one occurrence");
22937            }
22938        }
22939    }
22940
22941    #[test]
22942    fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
22943        fn worker() -> Worker {
22944            let client = Client::new("http://127.0.0.1:8080").expect("client");
22945            let mut worker = Worker::new(client, "rust-workers");
22946            worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
22947                let mut outcomes = Vec::new();
22948                for _ in 0..2 {
22949                    outcomes.push(
22950                        ctx.wait_condition(
22951                            ConditionWaitOptions::new("shared", "sha256:shared"),
22952                            || Ok(false),
22953                        )
22954                        .await?,
22955                    );
22956                }
22957                Ok(json!(outcomes))
22958            });
22959            worker
22960        }
22961
22962        let task = workflow_task(
22963            "rust.condition-loop",
22964            vec![
22965                history_event(
22966                    "ConditionWaitOpened",
22967                    json!({
22968                        "sequence": 1,
22969                        "condition_wait_id": "condition:1",
22970                        "condition_wait_occurrence_id": "rust:condition-wait:0",
22971                        "condition_key": "shared",
22972                        "condition_definition_fingerprint": "sha256:shared",
22973                    }),
22974                ),
22975                history_event(
22976                    "ConditionWaitSatisfied",
22977                    json!({
22978                        "sequence": 1,
22979                        "condition_wait_id": "condition:1",
22980                        "condition_wait_occurrence_id": "rust:condition-wait:0",
22981                        "condition_key": "shared",
22982                        "condition_definition_fingerprint": "sha256:shared",
22983                    }),
22984                ),
22985                history_event(
22986                    "ConditionWaitOpened",
22987                    json!({
22988                        "sequence": 2,
22989                        "condition_wait_id": "condition:2",
22990                        "condition_wait_occurrence_id": "rust:condition-wait:1",
22991                        "condition_key": "shared",
22992                        "condition_definition_fingerprint": "sha256:shared",
22993                    }),
22994                ),
22995                history_event(
22996                    "ConditionWaitSatisfied",
22997                    json!({
22998                        "sequence": 2,
22999                        "condition_wait_id": "condition:2",
23000                        "condition_wait_occurrence_id": "rust:condition-wait:1",
23001                        "condition_key": "shared",
23002                        "condition_definition_fingerprint": "sha256:shared",
23003                    }),
23004                ),
23005            ],
23006            DEFAULT_CODEC,
23007        );
23008
23009        for _cold_worker_or_restart in 0..2 {
23010            let commands = worker()
23011                .execute_workflow_task(task.clone())
23012                .expect("adjacent loop waits replay deterministically");
23013            assert_eq!(commands.len(), 1);
23014            assert_eq!(commands[0]["type"], "complete_workflow");
23015            assert_eq!(
23016                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
23017                json!(["satisfied", "satisfied"])
23018            );
23019        }
23020    }
23021
23022    #[test]
23023    fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
23024        let history = vec![history_event(
23025            "ConditionWaitOpened",
23026            json!({
23027                "sequence": 12,
23028                "condition_wait_id": "condition:12",
23029                "condition_wait_occurrence_id": "rust:condition-wait:0",
23030                "condition_key": "approval",
23031                "condition_definition_fingerprint": "sha256:approval-v1",
23032                "timeout_seconds": 30,
23033            }),
23034        )];
23035        for (options, expected_reason) in [
23036            (
23037                ConditionWaitOptions::new("changed", "sha256:approval-v1")
23038                    .timeout(Duration::from_secs(30)),
23039                "condition_wait_key_mismatch",
23040            ),
23041            (
23042                ConditionWaitOptions::new("approval", "sha256:approval-v2")
23043                    .timeout(Duration::from_secs(30)),
23044                "condition_wait_predicate_mismatch",
23045            ),
23046            (
23047                ConditionWaitOptions::new("approval", "sha256:approval-v1")
23048                    .timeout(Duration::from_secs(29)),
23049                "condition_wait_timeout_mismatch",
23050            ),
23051        ] {
23052            let ctx = workflow_context(history.clone());
23053            let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
23054            let mut task_context = TaskContext::from_waker(noop_waker_ref());
23055            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
23056                wait.as_mut().poll(&mut task_context)
23057            else {
23058                panic!("changed condition definition must fail replay");
23059            };
23060            assert_eq!(failure.reason, expected_reason);
23061            assert_eq!(failure.sequence, Some(12));
23062        }
23063    }
23064
23065    #[test]
23066    fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
23067        let error = WorkflowState::new(
23068            vec![history_event(
23069                "ConditionWaitOpened",
23070                json!({
23071                    "sequence": 12,
23072                    "condition_wait_id": "condition:12",
23073                    "condition_wait_occurrence_id": "rust:condition-wait:0",
23074                    "condition_key": "approval",
23075                }),
23076            )],
23077            "rust-workers".to_string(),
23078            DEFAULT_CODEC.to_string(),
23079            None,
23080        )
23081        .expect_err("condition history without a predicate fingerprint must fail");
23082
23083        assert!(matches!(
23084            error,
23085            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23086                if reason == "condition_wait_predicate_fingerprint_missing"
23087        ));
23088    }
23089
23090    #[test]
23091    fn condition_wait_history_requires_authored_occurrence_identity() {
23092        let error = WorkflowState::new(
23093            vec![history_event(
23094                "ConditionWaitOpened",
23095                json!({
23096                    "sequence": 12,
23097                    "condition_wait_id": "condition:12",
23098                    "condition_key": "approval",
23099                    "condition_definition_fingerprint": "sha256:approval-v1",
23100                }),
23101            )],
23102            "rust-workers".to_string(),
23103            DEFAULT_CODEC.to_string(),
23104            None,
23105        )
23106        .expect_err("condition history without occurrence identity must fail");
23107
23108        assert!(matches!(
23109            error,
23110            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23111                if reason == "condition_wait_occurrence_id_missing"
23112        ));
23113    }
23114
23115    #[test]
23116    fn typed_search_attribute_updates_validate_emit_and_replay() {
23117        let update = SearchAttributeUpdate::new()
23118            .keyword("OrderStatus", " waiting ")
23119            .expect("keyword")
23120            .int("Attempt", 3)
23121            .expect("int")
23122            .bool("Escalated", false)
23123            .expect("bool")
23124            .keyword_list("Regions", ["us-east", "eu-west"])
23125            .expect("list")
23126            .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
23127            .expect("datetime")
23128            .delete("LegacyStatus")
23129            .expect("delete");
23130        let ctx = workflow_context(Vec::new());
23131        ctx.upsert_search_attributes(update.clone())
23132            .expect("typed update");
23133        assert_eq!(
23134            ctx.take_commands().expect("search-attribute command"),
23135            vec![json!({
23136                "type": "upsert_search_attributes",
23137                "attributes": {
23138                    "Attempt": 3,
23139                    "Escalated": false,
23140                    "LegacyStatus": null,
23141                    "OrderStatus": "waiting",
23142                    "Regions": ["us-east", "eu-west"],
23143                    "UpdatedAt": "2026-08-22T04:00:00Z",
23144                },
23145                "attribute_types": {
23146                    "Attempt": "int",
23147                    "Escalated": "bool",
23148                    "OrderStatus": "keyword",
23149                    "Regions": "keyword_list",
23150                    "UpdatedAt": "datetime",
23151                },
23152            })]
23153        );
23154
23155        let replay = workflow_context(vec![history_event(
23156            "SearchAttributesUpserted",
23157            json!({
23158                "sequence": 6,
23159                "attributes": {
23160                    "Attempt": 3,
23161                    "Escalated": false,
23162                    "LegacyStatus": null,
23163                    "OrderStatus": "waiting",
23164                    "Regions": ["us-east", "eu-west"],
23165                    "UpdatedAt": "2026-08-22T04:00:00Z",
23166                },
23167                "attribute_types": {
23168                    "Attempt": "int",
23169                    "Escalated": "bool",
23170                    "OrderStatus": "keyword",
23171                    "Regions": "keyword_list",
23172                    "UpdatedAt": "datetime",
23173                },
23174                "merged": {},
23175            }),
23176        )]);
23177        replay
23178            .upsert_search_attributes(update)
23179            .expect("matching update replays");
23180        assert!(replay.take_commands().expect("commands").is_empty());
23181        replay.ensure_history_consumed().expect("history consumed");
23182
23183        let type_drift = workflow_context(vec![history_event(
23184            "SearchAttributesUpserted",
23185            json!({
23186                "sequence": 7,
23187                "attributes": {"OrderStatus": "waiting"},
23188                "attribute_types": {"OrderStatus": "keyword"},
23189                "merged": {"OrderStatus": "waiting"},
23190            }),
23191        )]);
23192        let error = type_drift
23193            .upsert_search_attributes(
23194                SearchAttributeUpdate::new()
23195                    .string("OrderStatus", "waiting")
23196                    .expect("string update"),
23197            )
23198            .expect_err("same JSON value with a changed type must fail replay");
23199        assert!(matches!(
23200            error,
23201            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23202                if reason == "search_attribute_type_mismatch"
23203        ));
23204
23205        let malformed_types = WorkflowState::new(
23206            vec![history_event(
23207                "SearchAttributesUpserted",
23208                json!({
23209                    "sequence": 8,
23210                    "attributes": {"OrderStatus": "waiting"},
23211                    "attribute_types": {"OrderStatus": "unsupported"},
23212                    "merged": {"OrderStatus": "waiting"},
23213                }),
23214            )],
23215            "rust-workers".to_string(),
23216            DEFAULT_CODEC.to_string(),
23217            None,
23218        )
23219        .expect_err("unsupported search-attribute type metadata must fail");
23220        assert!(matches!(
23221            malformed_types,
23222            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23223                if reason == "search_attribute_types_malformed"
23224        ));
23225
23226        assert!(matches!(
23227            SearchAttributeUpdate::new().keyword("bad key", "value"),
23228            Err(SearchAttributeUpdateError::InvalidKey(_))
23229        ));
23230        assert!(matches!(
23231            SearchAttributeUpdate::new().float("Ratio", f64::NAN),
23232            Err(SearchAttributeUpdateError::NonFiniteFloat(_))
23233        ));
23234        assert!(matches!(
23235            SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
23236            Err(SearchAttributeUpdateError::ValueTooLong { .. })
23237        ));
23238        assert!(matches!(
23239            SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
23240            Err(SearchAttributeUpdateError::InvalidDateTime(_))
23241        ));
23242        assert!(matches!(
23243            workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
23244            Err(Error::InvalidSearchAttributeUpdate(
23245                SearchAttributeUpdateError::Empty
23246            ))
23247        ));
23248    }
23249
23250    #[test]
23251    fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
23252        let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
23253        let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
23254
23255        assert!(SearchAttributeUpdate::new()
23256            .string("AsciiDescription", ascii)
23257            .is_ok());
23258        assert!(SearchAttributeUpdate::new()
23259            .string("Utf8Description", utf8)
23260            .is_ok());
23261        assert!(matches!(
23262            SearchAttributeUpdate::new().string(
23263                "TooLongDescription",
23264                "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
23265            ),
23266            Err(SearchAttributeUpdateError::ValueTooLong {
23267                kind: "string",
23268                limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
23269                ..
23270            })
23271        ));
23272    }
23273
23274    #[test]
23275    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
23276        let lone_fire = WorkflowState::new(
23277            vec![history_event(
23278                "TimerFired",
23279                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23280            )],
23281            "rust-workers".to_string(),
23282            DEFAULT_CODEC.to_string(),
23283            None,
23284        )
23285        .expect_err("TimerFired requires TimerScheduled");
23286        assert!(matches!(
23287            lone_fire,
23288            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23289                if reason == "timer_schedule_missing_or_duplicate"
23290        ));
23291
23292        let wrong_identity = WorkflowState::new(
23293            vec![
23294                history_event(
23295                    "TimerScheduled",
23296                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23297                ),
23298                history_event(
23299                    "TimerFired",
23300                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
23301                ),
23302            ],
23303            "rust-workers".to_string(),
23304            DEFAULT_CODEC.to_string(),
23305            None,
23306        )
23307        .expect_err("fire must match scheduled timer identity");
23308        assert!(matches!(
23309            wrong_identity,
23310            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23311                if reason == "timer_identity_mismatch"
23312        ));
23313
23314        let duplicate_fire = WorkflowState::new(
23315            vec![
23316                history_event(
23317                    "TimerScheduled",
23318                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23319                ),
23320                history_event(
23321                    "TimerFired",
23322                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23323                ),
23324                history_event(
23325                    "TimerFired",
23326                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23327                ),
23328            ],
23329            "rust-workers".to_string(),
23330            DEFAULT_CODEC.to_string(),
23331            None,
23332        )
23333        .expect_err("a durable timer cannot fire twice");
23334        assert!(matches!(
23335            duplicate_fire,
23336            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23337                if reason == "duplicate_timer_fire"
23338        ));
23339
23340        let wrong_fired_delay = WorkflowState::new(
23341            vec![
23342                history_event(
23343                    "TimerScheduled",
23344                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23345                ),
23346                history_event(
23347                    "TimerFired",
23348                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
23349                ),
23350            ],
23351            "rust-workers".to_string(),
23352            DEFAULT_CODEC.to_string(),
23353            None,
23354        )
23355        .expect_err("timer schedule and fire delays must agree");
23356        assert!(matches!(
23357            wrong_fired_delay,
23358            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23359                if reason == "timer_history_delay_mismatch"
23360        ));
23361    }
23362
23363    #[test]
23364    fn replay_rejects_activity_moved_before_recorded_timer() {
23365        let ctx = workflow_context(vec![
23366            history_event(
23367                "TimerScheduled",
23368                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23369            ),
23370            history_event(
23371                "TimerFired",
23372                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23373            ),
23374            history_event(
23375                "ActivityCompleted",
23376                json!({
23377                    "sequence": 2,
23378                    "activity_type": "after-timer",
23379                    "payload_codec": DEFAULT_CODEC,
23380                    "result": fixture_envelope(json!("done")),
23381                }),
23382            ),
23383        ]);
23384        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
23385        let mut task_context = TaskContext::from_waker(noop_waker_ref());
23386
23387        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
23388            activity.as_mut().poll(&mut task_context)
23389        else {
23390            panic!("reordered durable command must be rejected");
23391        };
23392        assert_eq!(failure.reason, "recorded_command_mismatch");
23393        assert_eq!(failure.sequence, Some(1));
23394        assert_eq!(failure.expected.as_deref(), Some("timer"));
23395        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
23396    }
23397
23398    #[test]
23399    fn workflow_context_emits_a_typed_named_signal_wait() {
23400        let ctx = workflow_context(Vec::new());
23401        let mut signal = Box::pin(ctx.wait_signal("finish"));
23402        let mut task_context = TaskContext::from_waker(noop_waker_ref());
23403
23404        assert!(matches!(
23405            signal.as_mut().poll(&mut task_context),
23406            Poll::Pending
23407        ));
23408        assert_eq!(
23409            ctx.take_commands().expect("signal-wait command"),
23410            vec![json!({
23411                "type": "open_signal_wait",
23412                "signal_name": "finish",
23413            })]
23414        );
23415    }
23416
23417    #[test]
23418    fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
23419        let ctx = workflow_context(Vec::new());
23420        let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
23421        let mut task_context = TaskContext::from_waker(noop_waker_ref());
23422
23423        let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
23424        else {
23425            panic!("runtime-reserved signal should be rejected");
23426        };
23427        assert!(message.contains("reserved by the workflow runtime"));
23428        assert!(ctx.take_commands().expect("commands").is_empty());
23429    }
23430
23431    #[tokio::test]
23432    async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
23433        let client = Client::builder("http://127.0.0.1:9")
23434            .build()
23435            .expect("client");
23436        let error = client
23437            .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
23438            .await
23439            .expect_err("runtime-reserved signal should be rejected before transport");
23440
23441        assert!(
23442            matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
23443        );
23444    }
23445
23446    #[test]
23447    fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
23448        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
23449            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
23450                value.to_string(),
23451            )]))
23452            .expect("message payload");
23453            json!({
23454                "schema": MESSAGE_STREAM_SCHEMA,
23455                "stream_name": "orders",
23456                "message_id": message_id,
23457                "position": position,
23458                "payload_envelope": payload,
23459            })
23460        }
23461
23462        fn opened(sequence: u64) -> HistoryEvent {
23463            history_event(
23464                "SignalWaitOpened",
23465                json!({
23466                    "sequence": sequence,
23467                    "signal_name": MESSAGE_STREAM_SIGNAL,
23468                }),
23469            )
23470        }
23471
23472        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
23473            history_event(
23474                "SignalApplied",
23475                json!({
23476                    "sequence": sequence,
23477                    "signal_name": MESSAGE_STREAM_SIGNAL,
23478                    "value": fixture_envelope(json!([delivery])),
23479                }),
23480            )
23481        }
23482
23483        fn received(delivery: Value) -> HistoryEvent {
23484            history_event(
23485                "SignalReceived",
23486                json!({
23487                    "signal_name": MESSAGE_STREAM_SIGNAL,
23488                    "arguments": fixture_envelope(json!([delivery])),
23489                    "payload_codec": DEFAULT_CODEC,
23490                }),
23491            )
23492        }
23493
23494        let client = Client::new("http://127.0.0.1:8080").expect("client");
23495        let mut worker = Worker::new(client, "rust-workers");
23496        worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
23497            let messages = ctx.message_stream("orders")?.receive(2).await?;
23498            Ok(json!(messages
23499                .into_iter()
23500                .map(|message| message.message_id)
23501                .collect::<Vec<_>>()))
23502        });
23503
23504        let first = delivery("message-1", 1, "one");
23505        let second = delivery("message-2", 2, "two");
23506        let batch = worker
23507            .execute_workflow_task_decision(workflow_task(
23508                "rust.message-stream-batch",
23509                vec![
23510                    opened(1),
23511                    received(first.clone()),
23512                    applied(1, first.clone()),
23513                    received(first.clone()),
23514                    received(second),
23515                ],
23516                DEFAULT_CODEC,
23517            ))
23518            .expect("worker task consumes the available batch");
23519
23520        assert_eq!(batch.commands.len(), 1);
23521        assert_eq!(batch.commands[0]["type"], "complete_workflow");
23522        assert_eq!(
23523            decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
23524                .expect("workflow result"),
23525            json!(["message-1", "message-2"])
23526        );
23527        assert_eq!(
23528            batch.message_stream_cursors,
23529            vec![json!({"stream_name": "orders", "through_position": 2})]
23530        );
23531        assert!(batch.message_stream_waits.is_empty());
23532
23533        let partial = worker
23534            .execute_workflow_task_decision(workflow_task(
23535                "rust.message-stream-batch",
23536                vec![opened(1), received(first.clone()), applied(1, first)],
23537                DEFAULT_CODEC,
23538            ))
23539            .expect("worker task returns without waiting for a missing second item");
23540        assert_eq!(partial.commands.len(), 1);
23541        assert_eq!(partial.commands[0]["type"], "complete_workflow");
23542        assert_eq!(
23543            decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
23544                .expect("workflow result"),
23545            json!(["message-1"])
23546        );
23547        assert_eq!(
23548            partial.message_stream_cursors,
23549            vec![json!({"stream_name": "orders", "through_position": 1})]
23550        );
23551        assert!(partial.message_stream_waits.is_empty());
23552    }
23553
23554    #[test]
23555    fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
23556        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
23557            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
23558                value.to_string(),
23559            )]))
23560            .expect("message payload");
23561            json!({
23562                "schema": MESSAGE_STREAM_SCHEMA,
23563                "stream_name": "orders",
23564                "message_id": message_id,
23565                "position": position,
23566                "payload_envelope": payload,
23567            })
23568        }
23569
23570        fn opened(sequence: u64) -> HistoryEvent {
23571            history_event(
23572                "SignalWaitOpened",
23573                json!({
23574                    "sequence": sequence,
23575                    "signal_name": MESSAGE_STREAM_SIGNAL,
23576                }),
23577            )
23578        }
23579
23580        fn received(delivery: Value) -> HistoryEvent {
23581            history_event(
23582                "SignalReceived",
23583                json!({
23584                    "signal_name": MESSAGE_STREAM_SIGNAL,
23585                    "arguments": fixture_envelope(json!([delivery])),
23586                    "payload_codec": DEFAULT_CODEC,
23587                }),
23588            )
23589        }
23590
23591        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
23592            history_event(
23593                "SignalApplied",
23594                json!({
23595                    "sequence": sequence,
23596                    "signal_name": MESSAGE_STREAM_SIGNAL,
23597                    "value": fixture_envelope(json!([delivery])),
23598                }),
23599            )
23600        }
23601
23602        let client = Client::new("http://127.0.0.1:8080").expect("client");
23603        let mut worker = Worker::new(client, "rust-workers");
23604        worker.register_workflow(
23605            "rust.message-stream-partial-batches",
23606            |ctx, _input| async move {
23607                let stream = ctx.message_stream("orders")?;
23608                let first = stream.receive(10).await?;
23609                let second = stream.receive(10).await?;
23610                Ok(json!([
23611                    first
23612                        .into_iter()
23613                        .map(|message| message.message_id)
23614                        .collect::<Vec<_>>(),
23615                    second
23616                        .into_iter()
23617                        .map(|message| message.message_id)
23618                        .collect::<Vec<_>>(),
23619                ]))
23620            },
23621        );
23622
23623        let first = delivery("message-1", 1, "one");
23624        let second = delivery("message-2", 2, "two");
23625        let decision = worker
23626            .execute_workflow_task_decision(workflow_task(
23627                "rust.message-stream-partial-batches",
23628                vec![
23629                    opened(1),
23630                    received(first.clone()),
23631                    applied(1, first),
23632                    opened(2),
23633                    received(second.clone()),
23634                    applied(2, second),
23635                ],
23636                DEFAULT_CODEC,
23637            ))
23638            .expect("cold replay preserves both authored receive boundaries");
23639
23640        assert_eq!(decision.commands.len(), 1);
23641        assert_eq!(decision.commands[0]["type"], "complete_workflow");
23642        assert_eq!(
23643            decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
23644                .expect("workflow result"),
23645            json!([["message-1"], ["message-2"]])
23646        );
23647        assert_eq!(
23648            decision.message_stream_cursors,
23649            vec![json!({"stream_name": "orders", "through_position": 2})]
23650        );
23651        assert!(decision.message_stream_waits.is_empty());
23652    }
23653
23654    #[test]
23655    fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
23656        let ctx = workflow_context(Vec::new());
23657        let stream = ctx.message_stream("orders").expect("message stream");
23658        let mut receive = Box::pin(stream.receive(10));
23659        let mut task_context = TaskContext::from_waker(noop_waker_ref());
23660
23661        assert!(matches!(
23662            receive.as_mut().poll(&mut task_context),
23663            Poll::Pending
23664        ));
23665        assert_eq!(
23666            ctx.take_commands().expect("message-stream wait command"),
23667            vec![json!({
23668                "type": "open_signal_wait",
23669                "signal_name": MESSAGE_STREAM_SIGNAL,
23670            })]
23671        );
23672        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
23673        assert!(cursors.is_empty());
23674        assert_eq!(
23675            waits,
23676            vec![json!({"stream_name": "orders", "after_position": 0})]
23677        );
23678    }
23679
23680    #[test]
23681    fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
23682        let ctx = workflow_context(vec![history_event(
23683            "SignalReceived",
23684            json!({
23685                "signal_name": MESSAGE_STREAM_SIGNAL,
23686                "arguments": fixture_envelope(json!([{
23687                    "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
23688                    "stream_name": "orders",
23689                    "through_position": 2,
23690                }])),
23691                "payload_codec": DEFAULT_CODEC,
23692            }),
23693        )]);
23694        let stream = ctx.message_stream("orders").expect("message stream");
23695        let mut receive = Box::pin(stream.receive(10));
23696        let mut task_context = TaskContext::from_waker(noop_waker_ref());
23697
23698        assert!(matches!(
23699            receive.as_mut().poll(&mut task_context),
23700            Poll::Pending
23701        ));
23702        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
23703        assert_eq!(
23704            cursors,
23705            vec![json!({"stream_name": "orders", "through_position": 2})]
23706        );
23707        assert_eq!(
23708            waits,
23709            vec![json!({"stream_name": "orders", "after_position": 2})]
23710        );
23711    }
23712
23713    #[test]
23714    fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
23715        let mut empty_map = BTreeMap::new();
23716        let mut nested = BTreeMap::new();
23717        nested.insert(
23718            "value".to_string(),
23719            AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
23720        );
23721        let values = vec![
23722            AvroValue::Bytes(vec![0, 255]),
23723            AvroValue::Long(1),
23724            AvroValue::Double(1.0),
23725            AvroValue::Array(Vec::new()),
23726            AvroValue::Map(std::mem::take(&mut empty_map)),
23727            AvroValue::Map(nested),
23728        ];
23729        let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
23730        let transport = vec![json!({
23731            "schema": MESSAGE_STREAM_SCHEMA,
23732            "stream_name": "orders",
23733            "message_id": "message-1",
23734            "position": 1,
23735            "payload_envelope": payload,
23736        })];
23737
23738        for _ in 0..2 {
23739            let Some(MessageStreamDelivery::Message(message)) =
23740                decode_message_stream_delivery(transport.clone()).expect("delivery")
23741            else {
23742                panic!("message delivery expected");
23743            };
23744            assert_eq!(message.arguments, values);
23745            assert!(matches!(message.arguments[1], AvroValue::Long(1)));
23746            assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
23747        }
23748    }
23749
23750    #[test]
23751    fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
23752        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
23753            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
23754                value.to_string(),
23755            )]))
23756            .expect("message payload");
23757            json!({
23758                "schema": MESSAGE_STREAM_SCHEMA,
23759                "stream_name": "orders",
23760                "message_id": message_id,
23761                "position": position,
23762                "payload_envelope": payload,
23763            })
23764        }
23765
23766        fn opened(sequence: u64) -> HistoryEvent {
23767            history_event(
23768                "SignalWaitOpened",
23769                json!({
23770                    "sequence": sequence,
23771                    "signal_name": MESSAGE_STREAM_SIGNAL,
23772                }),
23773            )
23774        }
23775
23776        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
23777            history_event(
23778                "SignalApplied",
23779                json!({
23780                    "sequence": sequence,
23781                    "signal_name": MESSAGE_STREAM_SIGNAL,
23782                    "value": fixture_envelope(json!([delivery])),
23783                }),
23784            )
23785        }
23786
23787        fn worker() -> Worker {
23788            let client = Client::new("http://127.0.0.1:8080").expect("client");
23789            let mut worker = Worker::new(client, "rust-workers");
23790            worker.register_workflow("rust.message-stream", |ctx, _input| async move {
23791                let stream = ctx.message_stream("orders")?;
23792                let first = stream.receive_one().await?;
23793                let second = stream.receive_one().await?;
23794                Ok(json!([first.message_id, second.message_id]))
23795            });
23796            worker
23797        }
23798
23799        fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
23800            let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
23801            task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
23802            task.signal_arguments = Some(fixture_envelope(json!([delivery])));
23803            task
23804        }
23805
23806        let waiting = worker()
23807            .execute_workflow_task_decision(workflow_task(
23808                "rust.message-stream",
23809                Vec::new(),
23810                DEFAULT_CODEC,
23811            ))
23812            .expect("first worker opens the stream wait");
23813        assert_eq!(
23814            waiting.commands,
23815            vec![json!({
23816                "type": "open_signal_wait",
23817                "signal_name": MESSAGE_STREAM_SIGNAL,
23818            })]
23819        );
23820        assert!(waiting.message_stream_cursors.is_empty());
23821        assert_eq!(
23822            waiting.message_stream_waits,
23823            vec![json!({"stream_name": "orders", "after_position": 0})]
23824        );
23825
23826        let first_delivery = delivery("message-1", 1, "one");
23827        let first_arrival = worker()
23828            .execute_workflow_task_decision(task_with_resume(
23829                vec![opened(1)],
23830                first_delivery.clone(),
23831            ))
23832            .expect("replacement worker consumes the first arrival");
23833        assert_eq!(
23834            first_arrival.commands,
23835            vec![json!({
23836                "type": "open_signal_wait",
23837                "signal_name": MESSAGE_STREAM_SIGNAL,
23838            })]
23839        );
23840        assert_eq!(
23841            first_arrival.message_stream_cursors,
23842            vec![json!({"stream_name": "orders", "through_position": 1})]
23843        );
23844        assert_eq!(
23845            first_arrival.message_stream_waits,
23846            vec![json!({"stream_name": "orders", "after_position": 1})]
23847        );
23848
23849        let second_delivery = delivery("message-2", 2, "two");
23850        let first_applied = applied(1, first_delivery);
23851        let completed = worker()
23852            .execute_workflow_task_decision(task_with_resume(
23853                vec![opened(1), first_applied.clone(), opened(2)],
23854                second_delivery.clone(),
23855            ))
23856            .expect("next replacement worker consumes the second arrival");
23857        assert_eq!(completed.commands.len(), 1);
23858        assert_eq!(completed.commands[0]["type"], "complete_workflow");
23859        assert_eq!(
23860            decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
23861                .expect("workflow result"),
23862            json!(["message-1", "message-2"])
23863        );
23864        assert_eq!(
23865            completed.message_stream_cursors,
23866            vec![json!({"stream_name": "orders", "through_position": 2})]
23867        );
23868        assert!(completed.message_stream_waits.is_empty());
23869
23870        let replay_history = vec![
23871            opened(1),
23872            first_applied,
23873            opened(2),
23874            applied(2, second_delivery),
23875        ];
23876        for _cold_worker_or_restart in 0..2 {
23877            let replayed = worker()
23878                .execute_workflow_task_decision(workflow_task(
23879                    "rust.message-stream",
23880                    replay_history.clone(),
23881                    DEFAULT_CODEC,
23882                ))
23883                .expect("cold worker replays each logical message exactly once");
23884            assert_eq!(replayed.commands.len(), 1);
23885            assert_eq!(
23886                decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
23887                    .expect("replayed workflow result"),
23888                json!(["message-1", "message-2"])
23889            );
23890            assert_eq!(
23891                replayed.message_stream_cursors,
23892                vec![json!({"stream_name": "orders", "through_position": 2})]
23893            );
23894            assert!(replayed.message_stream_waits.is_empty());
23895        }
23896    }
23897
23898    #[test]
23899    fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
23900        assert!(!worker_protocol_supports_message_streams("1.14"));
23901        assert!(worker_protocol_supports_message_streams("1.15"));
23902        assert!(worker_protocol_supports_message_streams("1.16"));
23903        assert!(worker_protocol_supports_message_streams(
23904            WORKER_PROTOCOL_VERSION
23905        ));
23906        assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
23907    }
23908
23909    #[test]
23910    fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
23911        let ctx = workflow_context(vec![
23912            history_event(
23913                "ConditionWaitOpened",
23914                json!({
23915                    "sequence": 1,
23916                    "condition_wait_id": "condition:1",
23917                    "condition_wait_occurrence_id": "rust:condition-wait:0",
23918                    "condition_key": "signal:finish",
23919                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
23920                }),
23921            ),
23922            history_event(
23923                "ConditionWaitSatisfied",
23924                json!({
23925                    "sequence": 1,
23926                    "condition_wait_id": "condition:1",
23927                    "condition_wait_occurrence_id": "rust:condition-wait:0",
23928                    "condition_key": "signal:finish",
23929                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
23930                }),
23931            ),
23932            history_event(
23933                "SignalReceived",
23934                json!({"signal_name": "finish", "arguments": []}),
23935            ),
23936        ]);
23937        let mut signal = Box::pin(ctx.wait_signal("finish"));
23938        let mut task_context = TaskContext::from_waker(noop_waker_ref());
23939
23940        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
23941            signal.as_mut().poll(&mut task_context)
23942        else {
23943            panic!("condition history must not resolve as a typed signal wait");
23944        };
23945        assert_eq!(failure.reason, "recorded_command_mismatch");
23946        assert_eq!(failure.expected.as_deref(), Some("condition wait"));
23947    }
23948
23949    #[test]
23950    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
23951        let signal_then_timer = vec![
23952            history_event(
23953                "SignalWaitOpened",
23954                json!({"sequence": 1, "signal_name": "go"}),
23955            ),
23956            history_event(
23957                "SignalApplied",
23958                json!({
23959                    "sequence": 1,
23960                    "signal_name": "go",
23961                    "value": fixture_envelope(json!(["now"])),
23962                }),
23963            ),
23964            history_event(
23965                "TimerScheduled",
23966                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
23967            ),
23968            history_event(
23969                "TimerFired",
23970                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
23971            ),
23972        ];
23973
23974        let ctx = workflow_context(signal_then_timer.clone());
23975        let mut signal = Box::pin(ctx.wait_signal("go"));
23976        let mut task_context = TaskContext::from_waker(noop_waker_ref());
23977        assert!(matches!(
23978            signal.as_mut().poll(&mut task_context),
23979            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
23980        ));
23981        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
23982        assert!(matches!(
23983            timer.as_mut().poll(&mut task_context),
23984            Poll::Ready(Ok(()))
23985        ));
23986        ctx.ensure_history_consumed()
23987            .expect("signal and timer history consumed in order");
23988
23989        let reordered = workflow_context(signal_then_timer);
23990        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
23991        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
23992            timer_first.as_mut().poll(&mut task_context)
23993        else {
23994            panic!("timer cannot consume signal-wait-first history");
23995        };
23996        assert_eq!(failure.reason, "recorded_command_mismatch");
23997        assert_eq!(failure.sequence, Some(1));
23998        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
23999
24000        let timer_then_signal = vec![
24001            history_event(
24002                "TimerScheduled",
24003                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24004            ),
24005            history_event(
24006                "TimerFired",
24007                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24008            ),
24009            history_event(
24010                "SignalWaitOpened",
24011                json!({"sequence": 2, "signal_name": "go"}),
24012            ),
24013            history_event(
24014                "SignalApplied",
24015                json!({
24016                    "sequence": 2,
24017                    "signal_name": "go",
24018                    "value": fixture_envelope(json!([])),
24019                }),
24020            ),
24021        ];
24022        let reordered = workflow_context(timer_then_signal);
24023        let mut signal_first = Box::pin(reordered.wait_signal("go"));
24024        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
24025            signal_first.as_mut().poll(&mut task_context)
24026        else {
24027            panic!("signal wait cannot consume timer-first history");
24028        };
24029        assert_eq!(failure.reason, "recorded_command_mismatch");
24030        assert_eq!(failure.sequence, Some(1));
24031        assert_eq!(failure.expected.as_deref(), Some("timer"));
24032    }
24033
24034    #[test]
24035    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
24036        let duplicate_timer = WorkflowState::new(
24037            vec![
24038                history_event(
24039                    "TimerScheduled",
24040                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24041                ),
24042                history_event(
24043                    "TimerScheduled",
24044                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
24045                ),
24046            ],
24047            "rust-workers".to_string(),
24048            DEFAULT_CODEC.to_string(),
24049            None,
24050        )
24051        .expect_err("one workflow sequence cannot schedule two timers");
24052        assert!(matches!(
24053            duplicate_timer,
24054            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
24055                if reason == "timer_schedule_missing_or_duplicate"
24056        ));
24057
24058        let colliding_kinds = WorkflowState::new(
24059            vec![
24060                history_event(
24061                    "TimerScheduled",
24062                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24063                ),
24064                history_event(
24065                    "ActivityCompleted",
24066                    json!({"sequence": 1, "activity_type": "same-sequence"}),
24067                ),
24068            ],
24069            "rust-workers".to_string(),
24070            DEFAULT_CODEC.to_string(),
24071            None,
24072        )
24073        .expect_err("one workflow sequence cannot identify two command kinds");
24074        assert!(matches!(
24075            colliding_kinds,
24076            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
24077                if reason == "durable_command_sequence_collision"
24078        ));
24079
24080        let duplicate_signal_wait = WorkflowState::new(
24081            vec![
24082                history_event(
24083                    "SignalWaitOpened",
24084                    json!({"sequence": 1, "signal_name": "go"}),
24085                ),
24086                history_event(
24087                    "SignalWaitOpened",
24088                    json!({"sequence": 1, "signal_name": "go"}),
24089                ),
24090            ],
24091            "rust-workers".to_string(),
24092            DEFAULT_CODEC.to_string(),
24093            None,
24094        )
24095        .expect_err("one workflow sequence cannot open two signal waits");
24096        assert!(matches!(
24097            duplicate_signal_wait,
24098            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
24099                if reason == "signal_wait_open_missing_or_duplicate"
24100        ));
24101    }
24102
24103    #[test]
24104    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
24105        let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
24106            .expect("side-effect result");
24107        let ctx = workflow_context(vec![history_event(
24108            "SideEffectRecorded",
24109            json!({"sequence": 99, "result": result}),
24110        )]);
24111
24112        let replayed: Value = ctx
24113            .side_effect(|| panic!("recorded side effect must not run"))
24114            .expect("positive global workflow sequence is valid");
24115        assert_eq!(replayed, json!({"captured": true}));
24116        ctx.ensure_history_consumed().expect("history consumed");
24117    }
24118
24119    #[test]
24120    fn workflow_history_rejects_zero_and_descending_command_sequences() {
24121        let result =
24122            encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
24123        let zero = WorkflowState::new(
24124            vec![history_event(
24125                "SideEffectRecorded",
24126                json!({"sequence": 0, "result": result.clone()}),
24127            )],
24128            "rust-workers".to_string(),
24129            DEFAULT_CODEC.to_string(),
24130            None,
24131        )
24132        .expect_err("durable command sequences must be positive");
24133        assert!(matches!(
24134            zero,
24135            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
24136                if reason == "durable_command_sequence_invalid"
24137        ));
24138
24139        let descending = WorkflowState::new(
24140            vec![
24141                history_event(
24142                    "SideEffectRecorded",
24143                    json!({"sequence": 3, "result": result}),
24144                ),
24145                history_event(
24146                    "VersionMarkerRecorded",
24147                    json!({
24148                        "sequence": 2,
24149                        "change_id": "descending-marker",
24150                        "version": 1,
24151                        "min_supported": 1,
24152                        "max_supported": 1,
24153                    }),
24154                ),
24155            ],
24156            "rust-workers".to_string(),
24157            DEFAULT_CODEC.to_string(),
24158            None,
24159        )
24160        .expect_err("new durable commands must remain strictly ordered");
24161        let Error::NonDeterministicReplay(failure) = descending else {
24162            panic!("expected typed replay failure");
24163        };
24164        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
24165        assert_eq!(failure.sequence, Some(2));
24166        assert_eq!(
24167            failure.expected.as_deref(),
24168            Some("workflow sequence greater than 3")
24169        );
24170        assert_eq!(failure.actual.as_deref(), Some("2"));
24171    }
24172
24173    #[test]
24174    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
24175        fn worker() -> Worker {
24176            let client = Client::new("http://127.0.0.1:8080").expect("client");
24177            let mut worker = Worker::new(client, "rust-workers");
24178            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
24179                ctx.wait_signal("finish").await?;
24180                let marker: String =
24181                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
24182                assert_eq!(marker, "after-finish");
24183                Ok(json!("finished"))
24184            });
24185            worker
24186        }
24187
24188        let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
24189            .expect("side-effect result");
24190        let task = workflow_task(
24191            "rust.finish-after-gaps",
24192            vec![
24193                history_event(
24194                    "SignalWaitOpened",
24195                    json!({"sequence": 1, "signal_name": "finish"}),
24196                ),
24197                history_event(
24198                    "SignalReceived",
24199                    json!({
24200                        "signal_id": "increment-3",
24201                        "signal_name": "increment",
24202                        "workflow_sequence": 2,
24203                        "payload_codec": DEFAULT_CODEC,
24204                        "arguments": fixture_envelope(json!([3])),
24205                    }),
24206                ),
24207                history_event(
24208                    "SignalReceived",
24209                    json!({
24210                        "signal_id": "increment-5",
24211                        "signal_name": "increment",
24212                        "workflow_sequence": 3,
24213                        "payload_codec": DEFAULT_CODEC,
24214                        "arguments": fixture_envelope(json!([5])),
24215                    }),
24216                ),
24217                history_event(
24218                    "SignalReceived",
24219                    json!({
24220                        "signal_id": "finish",
24221                        "signal_name": "finish",
24222                        "workflow_sequence": 4,
24223                        "payload_codec": DEFAULT_CODEC,
24224                        "arguments": fixture_envelope(json!([])),
24225                    }),
24226                ),
24227                history_event(
24228                    "SignalApplied",
24229                    json!({
24230                        "sequence": 1,
24231                        "signal_id": "finish",
24232                        "signal_name": "finish",
24233                        "payload_codec": DEFAULT_CODEC,
24234                        "value": fixture_envelope(json!([])),
24235                    }),
24236                ),
24237                history_event(
24238                    "SideEffectRecorded",
24239                    json!({"sequence": 5, "result": marker}),
24240                ),
24241            ],
24242            DEFAULT_CODEC,
24243        );
24244
24245        for _original_or_cold_worker in 0..2 {
24246            let commands = worker()
24247                .execute_workflow_task(task.clone())
24248                .expect("signal gaps preserve deterministic replay");
24249            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
24250            assert_eq!(commands[0]["type"], "complete_workflow");
24251            assert_eq!(
24252                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
24253                json!("finished")
24254            );
24255        }
24256    }
24257
24258    #[test]
24259    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
24260        let ctx = workflow_context(Vec::new());
24261        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
24262        let mut task_context = TaskContext::from_waker(noop_waker_ref());
24263        assert!(matches!(
24264            sleep.as_mut().poll(&mut task_context),
24265            Poll::Ready(Err(Error::TimerDurationOverflow))
24266        ));
24267        assert!(ctx.take_commands().expect("commands").is_empty());
24268    }
24269
24270    #[test]
24271    fn workflow_memo_update_emits_canonical_command_and_replays_once() {
24272        let entries = AvroValue::Map(BTreeMap::from([
24273            ("text".to_string(), AvroValue::String("same".to_string())),
24274            (
24275                "nested".to_string(),
24276                AvroValue::Map(BTreeMap::from([
24277                    ("beta".to_string(), AvroValue::Long(2)),
24278                    ("alpha".to_string(), AvroValue::Long(1)),
24279                ])),
24280            ),
24281            ("long".to_string(), AvroValue::Long(7)),
24282            ("double".to_string(), AvroValue::Double(7.0)),
24283            ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
24284        ]));
24285        let ctx = workflow_context(Vec::new());
24286        ctx.upsert_memo(entries.clone()).expect("valid memo update");
24287        let commands = ctx.take_commands().expect("commands");
24288
24289        assert_eq!(commands.len(), 1);
24290        assert_eq!(commands[0]["type"], "upsert_memo");
24291        let server_entries = json!({
24292            "codec": "avro",
24293            "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
24294        });
24295        assert_eq!(
24296            commands[0]["entries"]
24297                .as_object()
24298                .expect("entries envelope")
24299                .keys()
24300                .collect::<Vec<_>>(),
24301            vec!["blob", "codec"]
24302        );
24303        assert_eq!(commands[0]["entries"], server_entries);
24304        let wire_entries =
24305            decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
24306        assert_eq!(wire_entries, entries);
24307
24308        let history = vec![history_event(
24309            "MemoUpserted",
24310            json!({
24311                "sequence": 1,
24312                "entries": server_entries.clone(),
24313                "merged": server_entries,
24314            }),
24315        )];
24316        let replay = workflow_context(history.clone());
24317        replay
24318            .upsert_memo(entries.clone())
24319            .expect("matching replay identity");
24320        assert!(replay.take_commands().expect("replay commands").is_empty());
24321
24322        let changed_types = AvroValue::Map(BTreeMap::from([
24323            ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
24324            (
24325                "nested".to_string(),
24326                AvroValue::Map(BTreeMap::from([
24327                    ("alpha".to_string(), AvroValue::Long(1)),
24328                    ("beta".to_string(), AvroValue::Long(2)),
24329                ])),
24330            ),
24331            ("long".to_string(), AvroValue::Double(7.0)),
24332            ("double".to_string(), AvroValue::Long(7)),
24333            ("binary".to_string(), AvroValue::String("same".to_string())),
24334        ]));
24335        let error = workflow_context(history)
24336            .upsert_memo(changed_types)
24337            .expect_err("memo replay identity must preserve Avro value types");
24338        assert!(matches!(
24339            error,
24340            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
24341        ));
24342    }
24343
24344    #[test]
24345    fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
24346        let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
24347            .expect("memo envelope");
24348        let replay = workflow_context(vec![history_event(
24349            "MemoUpserted",
24350            json!({
24351                "sequence": 1,
24352                "entries": original.clone(),
24353                "merged": original
24354            }),
24355        )]);
24356        let error = replay
24357            .upsert_memo(json!({"stage": "changed"}))
24358            .expect_err("changed memo update must fail replay");
24359        assert!(matches!(
24360            error,
24361            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
24362        ));
24363
24364        let invalid = workflow_context(Vec::new())
24365            .upsert_memo(
24366                json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
24367            )
24368            .expect_err("oversized key");
24369        assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
24370    }
24371
24372    #[test]
24373    fn workflow_memo_replay_distinguishes_signed_zero_identity() {
24374        let negative_zero = AvroValue::Map(BTreeMap::from([(
24375            "reading".to_string(),
24376            AvroValue::Double(-0.0),
24377        )]));
24378        let negative_zero_envelope =
24379            encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
24380        let history = vec![history_event(
24381            "MemoUpserted",
24382            json!({
24383                "sequence": 1,
24384                "entries": negative_zero_envelope.clone(),
24385                "merged": negative_zero_envelope,
24386            }),
24387        )];
24388
24389        workflow_context(history.clone())
24390            .upsert_memo(negative_zero)
24391            .expect("matching negative-zero history identity");
24392
24393        let error = workflow_context(history)
24394            .upsert_memo(AvroValue::Map(BTreeMap::from([(
24395                "reading".to_string(),
24396                AvroValue::Double(0.0),
24397            )])))
24398            .expect_err("positive zero must not consume negative-zero memo history");
24399        assert!(matches!(
24400            error,
24401            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
24402        ));
24403    }
24404
24405    #[test]
24406    fn workflow_memo_capability_requires_flag_and_command_advertisement() {
24407        let supported = json!({
24408            "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
24409            "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
24410        });
24411        assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
24412        assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
24413            "workflow_memo_updates": {"supported": false},
24414            "supported_workflow_task_commands": ["upsert_memo"]
24415        }))));
24416        assert!(commands_use_workflow_memo_updates(&[json!({
24417            "type": "upsert_memo",
24418            "entries": {"stage": "processing"}
24419        })]));
24420    }
24421
24422    #[test]
24423    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
24424        let client = Client::new("http://127.0.0.1:8080").expect("client");
24425        let mut worker = Worker::new(client, "rust-workers");
24426        worker.register_workflow("rust.timer", |ctx, _input| async move {
24427            ctx.sleep(Duration::from_secs(5)).await?;
24428            ctx.activity("after-timer", json!([])).await
24429        });
24430
24431        let task = |history_events| WorkflowTask {
24432            task_id: "wft-rust-timer-1".to_string(),
24433            workflow_command_id: None,
24434            workflow_id: Some("wf-rust-timer".to_string()),
24435            run_id: Some("run-rust-timer".to_string()),
24436            workflow_type: "rust.timer".to_string(),
24437            cancel_requested: false,
24438            payload_codec: DEFAULT_CODEC.to_string(),
24439            arguments: Some(
24440                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
24441            ),
24442            history_events,
24443            total_history_events: None,
24444            history_size_bytes: None,
24445            continue_as_new_recommended: None,
24446            history_budget_pressure: None,
24447            next_history_page_token: None,
24448            workflow_task_attempt: 1,
24449            workflow_signal_id: None,
24450            signal_name: None,
24451            signal_arguments: None,
24452            workflow_update_id: None,
24453            update_name: None,
24454            lease_owner: Some("rust-worker".to_string()),
24455        };
24456
24457        let initial = worker
24458            .execute_workflow_task(task(Vec::new()))
24459            .expect("initial timer task");
24460        assert_eq!(
24461            initial,
24462            vec![json!({"type": "start_timer", "delay_seconds": 5})]
24463        );
24464
24465        let activity_result =
24466            encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
24467        let replayed = worker
24468            .execute_workflow_task(task(vec![
24469                history_event(
24470                    "TimerScheduled",
24471                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24472                ),
24473                history_event(
24474                    "TimerFired",
24475                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24476                ),
24477                history_event(
24478                    "ActivityCompleted",
24479                    json!({
24480                        "sequence": 2,
24481                        "activity_type": "after-timer",
24482                        "payload_codec": DEFAULT_CODEC,
24483                        "result": activity_result,
24484                    }),
24485                ),
24486            ]))
24487            .expect("replayed workflow task");
24488        assert_eq!(replayed.len(), 1);
24489        assert_eq!(replayed[0]["type"], "complete_workflow");
24490        assert_eq!(
24491            decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
24492            json!("done")
24493        );
24494    }
24495
24496    #[test]
24497    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
24498        let client = Client::new("http://127.0.0.1:8080").expect("client");
24499        let mut worker = Worker::new(client, "rust-workers");
24500        worker.register_workflow("rust.continue", |ctx, _input| async move {
24501            ctx.continue_as_new_with_options(
24502                ContinueAsNewOptions::new()
24503                    .workflow_type("rust.next")
24504                    .task_queue("next-workers"),
24505                json!([2, {"cursor": "next"}]),
24506            )
24507        });
24508
24509        let commands = worker
24510            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
24511            .expect("continue-as-new command");
24512
24513        assert_eq!(commands.len(), 1);
24514        assert_eq!(commands[0]["type"], "continue_as_new");
24515        assert_eq!(commands[0]["workflow_type"], "rust.next");
24516        assert_eq!(commands[0]["queue"], "next-workers");
24517        assert_eq!(
24518            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
24519                .expect("continue-as-new arguments"),
24520            json!([2, {"cursor": "next"}])
24521        );
24522    }
24523
24524    #[test]
24525    fn continue_as_new_preserves_typed_arguments() {
24526        let client = Client::new("http://127.0.0.1:8080").expect("client");
24527        let mut worker = Worker::new(client, "rust-workers");
24528        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
24529            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
24530            unreachable!("continue-as-new returns a control-flow error")
24531        });
24532
24533        let commands = worker
24534            .execute_workflow_task(workflow_task(
24535                "rust.typed-continue",
24536                Vec::new(),
24537                DEFAULT_CODEC,
24538            ))
24539            .expect("typed continue-as-new command");
24540
24541        assert_eq!(commands[0]["type"], "continue_as_new");
24542        assert_eq!(
24543            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
24544                .expect("typed continue arguments"),
24545            AvroValue::Array(vec![typed_fidelity_probe()])
24546        );
24547    }
24548
24549    #[test]
24550    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
24551        let client = Client::new("http://127.0.0.1:8080").expect("client");
24552        let mut worker = Worker::new(client, "rust-workers");
24553        worker.register_workflow("rust.continue", |ctx, _input| async move {
24554            ctx.continue_as_new(json!([2]))
24555        });
24556        let task = workflow_task(
24557            "rust.continue",
24558            vec![history_event(
24559                "WorkflowContinuedAsNew",
24560                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
24561            )],
24562            DEFAULT_CODEC,
24563        );
24564
24565        for _worker_restart_or_redelivery in 0..2 {
24566            let commands = worker
24567                .execute_workflow_task(task.clone())
24568                .expect("recorded transition replays");
24569            assert!(
24570                commands.is_empty(),
24571                "replay must not emit another successor"
24572            );
24573        }
24574    }
24575
24576    #[test]
24577    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
24578        let ctx = workflow_context(Vec::new());
24579        let error = ctx
24580            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
24581            .expect_err("blank queue must be rejected");
24582
24583        let Error::InvalidContinueAsNewOptions(error) = error else {
24584            panic!("expected typed continue-as-new validation error");
24585        };
24586        assert_eq!(error.field, "task_queue");
24587        assert!(ctx.take_commands().expect("commands").is_empty());
24588    }
24589
24590    #[test]
24591    fn workflow_context_exposes_server_history_budget() {
24592        let client = Client::new("http://127.0.0.1:8080").expect("client");
24593        let mut worker = Worker::new(client, "rust-workers");
24594        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
24595            let budget = ctx.history_budget()?;
24596            Ok(json!({
24597                "events": budget.event_count,
24598                "bytes": budget.size_bytes,
24599                "recommended": budget.continue_as_new_recommended,
24600                "pressure": budget.pressure,
24601            }))
24602        });
24603        let task: WorkflowTask = serde_json::from_value(json!({
24604            "task_id": "task-history-budget",
24605            "workflow_type": "rust.history-budget",
24606            "payload_codec": DEFAULT_CODEC,
24607            "history_events": [],
24608            "total_history_events": 480,
24609            "history_size_bytes": 1_048_576,
24610            "continue_as_new_recommended": true,
24611            "history_budget_pressure": "continue_as_new_recommended",
24612        }))
24613        .expect("published workflow task");
24614
24615        let commands = worker
24616            .execute_workflow_task(task)
24617            .expect("history-budget workflow");
24618        let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
24619        assert_eq!(result["events"], 480);
24620        assert_eq!(result["bytes"], 1_048_576);
24621        assert_eq!(result["recommended"], true);
24622        assert_eq!(result["pressure"], "continue_as_new_recommended");
24623    }
24624
24625    #[test]
24626    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
24627        let client = Client::new("http://127.0.0.1:8080").expect("client");
24628        let mut worker = Worker::new(client, "rust-workers");
24629        worker.register_workflow("rust.failing", |_ctx, _input| async move {
24630            Err(Error::Codec("rust_conformance_failure".to_string()))
24631        });
24632        let task = WorkflowTask {
24633            task_id: "wft-rust-failing-1".to_string(),
24634            workflow_command_id: None,
24635            workflow_id: Some("wf-rust-failing".to_string()),
24636            run_id: Some("run-rust-failing".to_string()),
24637            workflow_type: "rust.failing".to_string(),
24638            cancel_requested: false,
24639            payload_codec: DEFAULT_CODEC.to_string(),
24640            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
24641            history_events: Vec::new(),
24642            total_history_events: Some(0),
24643            history_size_bytes: None,
24644            continue_as_new_recommended: None,
24645            history_budget_pressure: None,
24646            next_history_page_token: None,
24647            workflow_task_attempt: 1,
24648            workflow_signal_id: None,
24649            signal_name: None,
24650            signal_arguments: None,
24651            workflow_update_id: None,
24652            update_name: None,
24653            lease_owner: Some("rust-worker".to_string()),
24654        };
24655
24656        let commands = worker
24657            .execute_workflow_task(task)
24658            .expect("handler failure becomes a workflow command");
24659
24660        assert_eq!(commands.len(), 1);
24661        assert_eq!(commands[0]["type"], "fail_workflow");
24662        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
24663        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
24664        assert_eq!(commands[0]["non_retryable"], false);
24665        assert_eq!(
24666            commands[0]["message"],
24667            "codec error: rust_conformance_failure"
24668        );
24669        assert_eq!(
24670            commands[0]["exception"]["message"],
24671            "codec error: rust_conformance_failure"
24672        );
24673    }
24674
24675    #[test]
24676    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
24677        let client = Client::new("http://127.0.0.1:8080").expect("client");
24678        let mut worker = Worker::new(client, "rust-workers");
24679        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
24680            let _: String = ctx.side_effect(|| "captured".to_string())?;
24681            Err(Error::WorkerLoop("application failure".to_string()))
24682        });
24683
24684        let commands = worker
24685            .execute_workflow_task(workflow_task(
24686                "rust.failing-after-side-effect",
24687                Vec::new(),
24688                DEFAULT_CODEC,
24689            ))
24690            .expect("ordinary failure remains a workflow decision");
24691
24692        assert_eq!(commands.len(), 2);
24693        assert_eq!(commands[0]["type"], "record_side_effect");
24694        assert_eq!(commands[1]["type"], "fail_workflow");
24695    }
24696
24697    #[test]
24698    fn uncaught_recorded_activity_failure_claims_only_its_persisted_boundary() {
24699        let client = Client::new("http://127.0.0.1:8080").expect("client");
24700        let mut worker = Worker::new(client, "rust-workers");
24701        worker.register_workflow("rust.redrive", |ctx, _input| async move {
24702            ctx.activity("greet", json!([])).await?;
24703            Ok(Value::Null)
24704        });
24705        worker.register_workflow("rust.redrive-translated", |ctx, _input| async move {
24706            match ctx.activity("greet", json!([])).await {
24707                Err(Error::ActivityFailed(_)) => Err(Error::WorkerLoop("translated".to_string())),
24708                result => result,
24709            }
24710        });
24711
24712        let failed = history_event(
24713            "ActivityFailed",
24714            json!({
24715                "sequence": 1,
24716                "activity_type": "greet",
24717                "activity_execution_id": "activity-1",
24718                "message": "failed"
24719            }),
24720        );
24721        let commands = worker
24722            .execute_workflow_task(workflow_task(
24723                "rust.redrive",
24724                vec![failed.clone()],
24725                DEFAULT_CODEC,
24726            ))
24727            .expect("recorded failure becomes a workflow command");
24728        assert_eq!(commands[0]["type"], "fail_workflow");
24729        assert_eq!(commands[0]["failed_step_sequence"], 1);
24730        assert_eq!(commands[0]["failed_activity_execution_id"], "activity-1");
24731
24732        let translated = worker
24733            .execute_workflow_task(workflow_task(
24734                "rust.redrive-translated",
24735                vec![failed],
24736                DEFAULT_CODEC,
24737            ))
24738            .expect("translated failure becomes a workflow command");
24739        assert_eq!(translated[0]["type"], "fail_workflow");
24740        assert!(translated[0].get("failed_step_sequence").is_none());
24741        assert!(translated[0].get("failed_activity_execution_id").is_none());
24742
24743        let without_identity = worker
24744            .execute_workflow_task(workflow_task(
24745                "rust.redrive",
24746                vec![history_event(
24747                    "ActivityFailed",
24748                    json!({"sequence": 1, "activity_type": "greet", "message": "failed"}),
24749                )],
24750                DEFAULT_CODEC,
24751            ))
24752            .expect("incomplete identity still fails the workflow");
24753        assert!(without_identity[0].get("failed_step_sequence").is_none());
24754
24755        let timed_out = worker
24756            .execute_workflow_task(workflow_task(
24757                "rust.redrive",
24758                vec![history_event(
24759                    "ActivityTimedOut",
24760                    json!({
24761                        "sequence": 1,
24762                        "activity_type": "greet",
24763                        "activity_execution_id": "activity-timeout",
24764                    }),
24765                )],
24766                DEFAULT_CODEC,
24767            ))
24768            .expect("timeout still fails the workflow");
24769        assert!(timed_out[0].get("failed_step_sequence").is_none());
24770    }
24771
24772    #[test]
24773    fn redriven_history_reuses_completed_prefix_and_reschedules_only_failed_step() {
24774        let client = Client::new("http://127.0.0.1:8080").expect("client");
24775        let mut worker = Worker::new(client, "rust-workers");
24776        worker.register_workflow("rust.redrive-successor", |ctx, _input| async move {
24777            let first = ctx.activity("first", json!([])).await?;
24778            let second = ctx.activity("second", json!([first.clone()])).await?;
24779            Ok(json!({"first": first, "second": second}))
24780        });
24781        let first = history_event(
24782            "ActivityCompleted",
24783            json!({
24784                "sequence": 1,
24785                "activity_type": "first",
24786                "result": encode_value_envelope(&json!("recorded"), DEFAULT_CODEC).expect("first result"),
24787                "payload_codec": DEFAULT_CODEC,
24788                "reused_from_run_id": "failed-run",
24789                "reused_activity_execution_id": "original-first",
24790            }),
24791        );
24792
24793        let retry = worker
24794            .execute_workflow_task(workflow_task(
24795                "rust.redrive-successor",
24796                vec![first.clone()],
24797                DEFAULT_CODEC,
24798            ))
24799            .expect("reused result replays before failed step");
24800        assert_eq!(retry.len(), 1);
24801        assert_eq!(retry[0]["type"], "schedule_activity");
24802        assert_eq!(retry[0]["activity_type"], "second");
24803        assert_eq!(
24804            decode_wire_value(&retry[0]["arguments"], DEFAULT_CODEC).expect("second arguments"),
24805            json!(["recorded"]),
24806        );
24807
24808        let second = history_event(
24809            "ActivityCompleted",
24810            json!({
24811                "sequence": 2,
24812                "activity_type": "second",
24813                "result": encode_value_envelope(&json!("retried"), DEFAULT_CODEC).expect("second result"),
24814                "payload_codec": DEFAULT_CODEC,
24815            }),
24816        );
24817        let completed = worker
24818            .execute_workflow_task(workflow_task(
24819                "rust.redrive-successor",
24820                vec![first, second],
24821                DEFAULT_CODEC,
24822            ))
24823            .expect("retried step completes the successor");
24824        assert_eq!(completed.len(), 1);
24825        assert_eq!(completed[0]["type"], "complete_workflow");
24826        assert_eq!(
24827            decode_wire_value(&completed[0]["result"], DEFAULT_CODEC).expect("workflow result"),
24828            json!({"first": "recorded", "second": "retried"}),
24829        );
24830    }
24831
24832    #[test]
24833    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
24834        let client = Client::new("http://127.0.0.1:8080").expect("client");
24835        let mut worker = Worker::new(client, "rust-workers");
24836        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
24837            Err(Error::WorkerLoop("application failure".to_string()))
24838        });
24839        let result =
24840            encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
24841
24842        let error = worker
24843            .execute_workflow_task(workflow_task(
24844                "rust.removed-side-effect",
24845                vec![history_event(
24846                    "SideEffectRecorded",
24847                    json!({"sequence": 1, "result": result}),
24848                )],
24849                DEFAULT_CODEC,
24850            ))
24851            .expect_err("removed committed history must not become fail_workflow");
24852
24853        let Error::NonDeterministicReplay(failure) = error else {
24854            panic!("expected typed replay failure");
24855        };
24856        assert_eq!(failure.reason, "recorded_commands_unconsumed");
24857        assert_eq!(failure.sequence, Some(1));
24858        assert_eq!(failure.expected.as_deref(), Some("side effect"));
24859    }
24860
24861    #[test]
24862    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
24863        let client = Client::new("http://127.0.0.1:8080").expect("client");
24864        let mut worker = Worker::new(client, "rust-workers");
24865        worker.register_workflow(
24866            "rust.side-effect-before-marker-error",
24867            |ctx, _input| async move {
24868                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
24869                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
24870                ctx.get_version("restart-safe", 2, 2)?;
24871                Ok(Value::Null)
24872            },
24873        );
24874
24875        let error = worker
24876            .execute_workflow_task(workflow_task(
24877                "rust.side-effect-before-marker-error",
24878                vec![history_event(
24879                    "VersionMarkerRecorded",
24880                    json!({
24881                        "sequence": 1,
24882                        "change_id": "restart-safe",
24883                        "version": 1,
24884                        "min_supported": 1,
24885                        "max_supported": 1,
24886                    }),
24887                )],
24888                DEFAULT_CODEC,
24889            ))
24890            .expect_err("replay error must return no queued workflow commands");
24891
24892        let Error::NonDeterministicReplay(failure) = error else {
24893            panic!("expected typed replay failure");
24894        };
24895        assert_eq!(failure.reason, "version_marker_incompatible_range");
24896        assert_eq!(failure.sequence, Some(1));
24897    }
24898
24899    #[test]
24900    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
24901        let client = Client::new("http://127.0.0.1:8080").expect("client");
24902        let mut worker = Worker::new(client, "rust-workers");
24903        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
24904            ctx.sleep(Duration::from_secs(5)).await?;
24905            Ok(json!({"status": "timer fired"}))
24906        });
24907
24908        let task = WorkflowTask {
24909            task_id: "wft-rust-timer-pending".to_string(),
24910            workflow_command_id: None,
24911            workflow_id: Some("wf-rust-timer".to_string()),
24912            run_id: Some("run-rust-timer".to_string()),
24913            workflow_type: "rust.timer.pending".to_string(),
24914            cancel_requested: false,
24915            payload_codec: DEFAULT_CODEC.to_string(),
24916            arguments: Some(
24917                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
24918            ),
24919            history_events: vec![history_event(
24920                "TimerScheduled",
24921                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24922            )],
24923            total_history_events: Some(1),
24924            history_size_bytes: None,
24925            continue_as_new_recommended: None,
24926            history_budget_pressure: None,
24927            next_history_page_token: None,
24928            workflow_task_attempt: 1,
24929            workflow_signal_id: None,
24930            signal_name: None,
24931            signal_arguments: None,
24932            workflow_update_id: None,
24933            update_name: None,
24934            lease_owner: Some("rust-worker".to_string()),
24935        };
24936
24937        for _redelivery_or_restart in 0..2 {
24938            let commands = worker
24939                .execute_workflow_task(task.clone())
24940                .expect("recorded timer remains pending");
24941            assert!(
24942                commands.is_empty(),
24943                "recorded timer must not be rescheduled"
24944            );
24945        }
24946    }
24947
24948    #[test]
24949    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
24950        let client = Client::new("http://127.0.0.1:8080").expect("client");
24951        let mut worker = Worker::new(client, "rust-workers");
24952        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
24953            Ok(json!({"status": "completed"}))
24954        });
24955        let task = WorkflowTask {
24956            task_id: "wft-rust-timer-removed".to_string(),
24957            workflow_command_id: None,
24958            workflow_id: Some("wf-rust-timer".to_string()),
24959            run_id: Some("run-rust-timer".to_string()),
24960            workflow_type: "rust.timer.removed".to_string(),
24961            cancel_requested: false,
24962            payload_codec: DEFAULT_CODEC.to_string(),
24963            arguments: Some(
24964                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
24965            ),
24966            history_events: vec![
24967                history_event(
24968                    "TimerScheduled",
24969                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24970                ),
24971                history_event(
24972                    "TimerFired",
24973                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24974                ),
24975            ],
24976            total_history_events: Some(2),
24977            history_size_bytes: None,
24978            continue_as_new_recommended: None,
24979            history_budget_pressure: None,
24980            next_history_page_token: None,
24981            workflow_task_attempt: 1,
24982            workflow_signal_id: None,
24983            signal_name: None,
24984            signal_arguments: None,
24985            workflow_update_id: None,
24986            update_name: None,
24987            lease_owner: Some("rust-worker".to_string()),
24988        };
24989
24990        let Error::NonDeterministicReplay(failure) = worker
24991            .execute_workflow_task(task)
24992            .expect_err("removed timer must fail replay")
24993        else {
24994            panic!("expected typed replay failure");
24995        };
24996        assert_eq!(failure.reason, "recorded_commands_unconsumed");
24997        assert_eq!(failure.sequence, Some(1));
24998    }
24999
25000    #[test]
25001    fn workflow_context_emits_explicit_child_workflow_contract() {
25002        let ctx = WorkflowContext {
25003            cancellation_scope_id: "root".into(),
25004            state: Arc::new(Mutex::new(
25005                WorkflowState::new_with_identity(
25006                    Vec::new(),
25007                    Some("wf-parent".to_string()),
25008                    Some("run-parent".to_string()),
25009                    "parent-workers".to_string(),
25010                    DEFAULT_CODEC.to_string(),
25011                    None,
25012                )
25013                .expect("workflow state"),
25014            )),
25015        };
25016        let options = ChildWorkflowOptions::new("python-workers")
25017            .parent_close_policy(ParentClosePolicy::RequestCancel)
25018            .retry_policy(ChildWorkflowRetryPolicy {
25019                max_attempts: Some(3),
25020                backoff_seconds: vec![1, 5],
25021                non_retryable_error_types: vec!["ValidationError".to_string()],
25022            })
25023            .execution_timeout_seconds(600)
25024            .run_timeout_seconds(120);
25025        let mut call = Box::pin(ctx.start_child_workflow(
25026            "python.fulfil-order",
25027            options,
25028            json!([{"order_id": "order-42"}]),
25029        ));
25030        let mut task_context = TaskContext::from_waker(noop_waker_ref());
25031
25032        assert!(matches!(
25033            call.as_mut().poll(&mut task_context),
25034            Poll::Pending
25035        ));
25036        let commands = ctx.take_commands().expect("commands");
25037        assert_eq!(commands.len(), 1);
25038        let command = &commands[0];
25039        assert_eq!(command["type"], "start_child_workflow");
25040        assert_eq!(command["workflow_type"], "python.fulfil-order");
25041        assert_eq!(command["queue"], "python-workers");
25042        assert_eq!(command["parent_close_policy"], "request_cancel");
25043        assert_eq!(command["retry_policy"]["max_attempts"], 3);
25044        assert_eq!(command["execution_timeout_seconds"], 600);
25045        assert_eq!(command["run_timeout_seconds"], 120);
25046        assert_eq!(
25047            decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
25048            json!([{"order_id": "order-42"}])
25049        );
25050    }
25051
25052    fn child_parent_worker() -> Worker {
25053        let client = Client::new("http://127.0.0.1:8080").expect("client");
25054        let mut worker = Worker::new(client, "rust-parent-workers");
25055        worker.register_workflow("rust.parent", |ctx, _input| async move {
25056            let child = ctx
25057                .start_child_workflow(
25058                    "python.child",
25059                    ChildWorkflowOptions::new("python-child-workers")
25060                        .parent_close_policy(ParentClosePolicy::Terminate),
25061                    json!([{"codec_probe": [1, true, "rust"]}]),
25062                )
25063                .await?;
25064            Ok(json!({
25065                "parent_workflow_id": child.parent.workflow_id,
25066                "parent_run_id": child.parent.run_id,
25067                "child_workflow_id": child.child.workflow_id,
25068                "child_run_id": child.child.run_id,
25069                "child_workflow_type": child.child_workflow_type,
25070                "result": child.result,
25071            }))
25072        });
25073        worker
25074    }
25075
25076    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
25077        WorkflowTask {
25078            task_id: "wft-child-parent".to_string(),
25079            workflow_command_id: None,
25080            workflow_id: Some("wf-parent".to_string()),
25081            run_id: Some("run-parent".to_string()),
25082            workflow_type: "rust.parent".to_string(),
25083            cancel_requested: false,
25084            payload_codec: DEFAULT_CODEC.to_string(),
25085            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
25086            history_events: vec![
25087                HistoryEvent {
25088                    event_type: "ChildWorkflowScheduled".to_string(),
25089                    payload: json!({
25090                        "sequence": 1,
25091                        "child_call_id": "call-child",
25092                        "child_workflow_instance_id": "wf-child",
25093                        "child_workflow_run_id": "run-child",
25094                        "child_workflow_type": "python.child",
25095                        "parent_close_policy": "terminate",
25096                    }),
25097                    raw: HashMap::new(),
25098                },
25099                HistoryEvent {
25100                    event_type: event_type.to_string(),
25101                    payload,
25102                    raw: HashMap::new(),
25103                },
25104            ],
25105            total_history_events: Some(2),
25106            history_size_bytes: None,
25107            continue_as_new_recommended: None,
25108            history_budget_pressure: None,
25109            next_history_page_token: None,
25110            workflow_task_attempt: 1,
25111            workflow_signal_id: None,
25112            signal_name: None,
25113            signal_arguments: None,
25114            workflow_update_id: None,
25115            update_name: None,
25116            lease_owner: Some("rust-worker".to_string()),
25117        }
25118    }
25119
25120    #[test]
25121    fn committed_child_result_replays_without_starting_a_duplicate() {
25122        let worker = child_parent_worker();
25123        let task = child_parent_task(
25124            "ChildRunCompleted",
25125            json!({
25126                "sequence": 1,
25127                "child_call_id": "call-child",
25128                "child_workflow_instance_id": "wf-child",
25129                "child_workflow_run_id": "run-child",
25130                "child_workflow_type": "python.child",
25131                "payload_codec": DEFAULT_CODEC,
25132                "result": fixture_envelope(json!({"from":"python","ok":true})),
25133            }),
25134        );
25135
25136        for _restart in 0..2 {
25137            let commands = worker
25138                .execute_workflow_task(task.clone())
25139                .expect("replayed parent task");
25140            assert_eq!(commands.len(), 1);
25141            assert_eq!(commands[0]["type"], "complete_workflow");
25142            assert!(!commands
25143                .iter()
25144                .any(|command| command["type"] == "start_child_workflow"));
25145            let output =
25146                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
25147            assert_eq!(output["parent_workflow_id"], "wf-parent");
25148            assert_eq!(output["parent_run_id"], "run-parent");
25149            assert_eq!(output["child_workflow_id"], "wf-child");
25150            assert_eq!(output["child_run_id"], "run-child");
25151            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
25152        }
25153    }
25154
25155    #[test]
25156    fn typed_child_arguments_and_results_survive_replay() {
25157        let client = Client::new("http://127.0.0.1:8080").expect("client");
25158        let mut worker = Worker::new(client, "rust-parent-workers");
25159        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
25160            let child = ctx
25161                .start_child_workflow_avro_value(
25162                    "python.typed-child",
25163                    ChildWorkflowOptions::new("python-workers"),
25164                    AvroValue::Array(vec![typed_fidelity_probe()]),
25165                )
25166                .await?;
25167            Ok(child.result)
25168        });
25169
25170        let initial = worker
25171            .execute_workflow_task(workflow_task(
25172                "rust.typed-parent",
25173                Vec::new(),
25174                DEFAULT_CODEC,
25175            ))
25176            .expect("typed child start");
25177        assert_eq!(initial[0]["type"], "start_child_workflow");
25178        assert_eq!(
25179            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
25180                .expect("typed child arguments"),
25181            AvroValue::Array(vec![typed_fidelity_probe()])
25182        );
25183
25184        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25185            .expect("typed child result");
25186        let task = workflow_task(
25187            "rust.typed-parent",
25188            vec![
25189                history_event(
25190                    "ChildWorkflowScheduled",
25191                    json!({
25192                        "sequence": 1,
25193                        "child_call_id": "call-typed",
25194                        "child_workflow_instance_id": "wf-child",
25195                        "child_workflow_run_id": "run-child",
25196                        "child_workflow_type": "python.typed-child",
25197                    }),
25198                ),
25199                history_event(
25200                    "ChildRunCompleted",
25201                    json!({
25202                        "sequence": 1,
25203                        "child_call_id": "call-typed",
25204                        "child_workflow_instance_id": "wf-child",
25205                        "child_workflow_run_id": "run-child",
25206                        "child_workflow_type": "python.typed-child",
25207                        "payload_codec": DEFAULT_CODEC,
25208                        "result": result,
25209                    }),
25210                ),
25211            ],
25212            DEFAULT_CODEC,
25213        );
25214
25215        let commands = worker
25216            .execute_workflow_task(task)
25217            .expect("typed child replay");
25218        assert_eq!(commands[0]["type"], "complete_workflow");
25219        assert_eq!(
25220            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
25221                .expect("typed parent result"),
25222            typed_fidelity_probe()
25223        );
25224    }
25225
25226    #[test]
25227    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
25228        let worker = child_parent_worker();
25229        let mut task = child_parent_task("unused", Value::Null);
25230        task.history_events.truncate(1);
25231        task.total_history_events = Some(1);
25232
25233        for _redelivery_or_restart in 0..2 {
25234            let commands = worker
25235                .execute_workflow_task(task.clone())
25236                .expect("recorded child remains pending");
25237            assert!(
25238                commands.is_empty(),
25239                "recorded pending child must not be started again"
25240            );
25241        }
25242    }
25243
25244    #[test]
25245    fn child_cancellation_becomes_stable_parent_failure_command() {
25246        let worker = child_parent_worker();
25247        let task = child_parent_task(
25248            "ChildRunCancelled",
25249            json!({
25250                "sequence": 1,
25251                "child_workflow_instance_id": "wf-child",
25252                "child_workflow_run_id": "run-child",
25253                "child_workflow_type": "python.child",
25254                "failure_id": "failure-child",
25255                "failure_category": "cancelled",
25256                "message": "cancelled by parent-close policy",
25257            }),
25258        );
25259
25260        let commands = worker
25261            .execute_workflow_task(task)
25262            .expect("parent settlement");
25263        assert_eq!(commands.len(), 1);
25264        assert_eq!(commands[0]["type"], "fail_workflow");
25265        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
25266        assert_eq!(
25267            commands[0]["exception"]["properties"]["reason"],
25268            "cancelled"
25269        );
25270        assert_eq!(
25271            commands[0]["exception"]["properties"]["child_workflow_run_id"],
25272            "run-child"
25273        );
25274    }
25275
25276    #[test]
25277    fn workflow_can_handle_typed_child_failure() {
25278        let client = Client::new("http://127.0.0.1:8080").expect("client");
25279        let mut worker = Worker::new(client, "rust-parent-workers");
25280        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
25281            match ctx
25282                .start_child_workflow(
25283                    "python.child",
25284                    ChildWorkflowOptions::new("python-child-workers"),
25285                    json!([]),
25286                )
25287                .await
25288            {
25289                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
25290                    "reason": failure.reason,
25291                    "failure_id": failure.failure_id,
25292                    "exception_class": failure.exception_class,
25293                    "child_run_id": failure.child_workflow_run_id,
25294                })),
25295                Err(error) => Err(error),
25296                Ok(_) => Err(Error::WorkerLoop(
25297                    "child unexpectedly succeeded".to_string(),
25298                )),
25299            }
25300        });
25301        let mut task = child_parent_task(
25302            "ChildRunFailed",
25303            json!({
25304                "sequence": 1,
25305                "child_workflow_instance_id": "wf-child",
25306                "child_workflow_run_id": "run-child",
25307                "child_workflow_type": "python.child",
25308                "failure_id": "failure-child",
25309                "failure_category": "child_workflow",
25310                "message": "payment rejected",
25311                "exception": {
25312                    "type": "PaymentRejected",
25313                    "class": "payments.PaymentRejected",
25314                    "message": "payment rejected"
25315                }
25316            }),
25317        );
25318        task.workflow_type = "rust.handled-parent".to_string();
25319        task.history_events[0].payload["parent_close_policy"] = json!("abandon");
25320
25321        let commands = worker.execute_workflow_task(task).expect("handled failure");
25322        assert_eq!(commands[0]["type"], "complete_workflow");
25323        let output =
25324            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
25325        assert_eq!(output["reason"], "child_workflow");
25326        assert_eq!(output["failure_id"], "failure-child");
25327        assert_eq!(output["exception_class"], "payments.PaymentRejected");
25328        assert_eq!(output["child_run_id"], "run-child");
25329    }
25330
25331    #[test]
25332    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
25333        let client = Client::new("http://127.0.0.1:8080").expect("client");
25334        let mut worker = Worker::new(client, "rust-workers");
25335
25336        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
25337            let signal = ctx.wait_signal("start").await?;
25338            let name = signal
25339                .first()
25340                .and_then(|value| value.as_str())
25341                .unwrap_or("world");
25342            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
25343            Ok(json!({
25344                "greeting": greeting,
25345                "language": "rust"
25346            }))
25347        });
25348
25349        let signal_arguments =
25350            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
25351        let task = WorkflowTask {
25352            task_id: "wft-rust-signal-1".to_string(),
25353            workflow_command_id: None,
25354            workflow_id: Some("wf-rust-hello".to_string()),
25355            run_id: Some("run-rust-hello".to_string()),
25356            workflow_type: "rust.hello_workflow".to_string(),
25357            cancel_requested: false,
25358            payload_codec: DEFAULT_CODEC.to_string(),
25359            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
25360            history_events: vec![HistoryEvent {
25361                event_type: "SignalReceived".to_string(),
25362                payload: json!({
25363                    "signal_id": "sig-rust-1",
25364                    "signal_name": "start"
25365                }),
25366                raw: HashMap::new(),
25367            }],
25368            total_history_events: Some(1),
25369            history_size_bytes: None,
25370            continue_as_new_recommended: None,
25371            history_budget_pressure: None,
25372            next_history_page_token: None,
25373            workflow_task_attempt: 1,
25374            workflow_signal_id: Some("sig-rust-1".to_string()),
25375            signal_name: Some("start".to_string()),
25376            signal_arguments: Some(signal_arguments),
25377            workflow_update_id: None,
25378            update_name: None,
25379            lease_owner: Some("rust-worker".to_string()),
25380        };
25381
25382        let commands = worker.execute_workflow_task(task).expect("workflow task");
25383
25384        assert_eq!(commands.len(), 1);
25385        assert_eq!(commands[0]["type"], "schedule_activity");
25386        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
25387        assert_eq!(
25388            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
25389            json!(["Rust"])
25390        );
25391    }
25392
25393    #[test]
25394    fn workflow_task_appends_paginated_history_events() {
25395        let mut task = WorkflowTask {
25396            task_id: "wft-rust-pages-1".to_string(),
25397            workflow_command_id: None,
25398            workflow_id: Some("wf-rust-pages".to_string()),
25399            run_id: Some("run-rust-pages".to_string()),
25400            workflow_type: "rust.hello_workflow".to_string(),
25401            cancel_requested: false,
25402            payload_codec: DEFAULT_CODEC.to_string(),
25403            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
25404            history_events: vec![HistoryEvent {
25405                event_type: "WorkflowStarted".to_string(),
25406                payload: json!({}),
25407                raw: HashMap::new(),
25408            }],
25409            total_history_events: Some(3),
25410            history_size_bytes: None,
25411            continue_as_new_recommended: None,
25412            history_budget_pressure: None,
25413            next_history_page_token: Some("MQ==".to_string()),
25414            workflow_task_attempt: 1,
25415            workflow_signal_id: None,
25416            signal_name: None,
25417            signal_arguments: None,
25418            workflow_update_id: None,
25419            update_name: None,
25420            lease_owner: Some("rust-worker".to_string()),
25421        };
25422
25423        task.append_history_page(WorkflowTaskHistoryPage {
25424            history_events: vec![
25425                HistoryEvent {
25426                    event_type: "SignalReceived".to_string(),
25427                    payload: json!({
25428                        "signal_id": "sig-rust-1",
25429                        "signal_name": "start",
25430                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
25431                            .expect("signal arguments")
25432                    }),
25433                    raw: HashMap::new(),
25434                },
25435                HistoryEvent {
25436                    event_type: "MarkerRecorded".to_string(),
25437                    payload: json!({"sequence": 3}),
25438                    raw: HashMap::new(),
25439                },
25440            ],
25441            total_history_events: Some(3),
25442            next_history_page_token: None,
25443        });
25444
25445        assert_eq!(task.history_events.len(), 3);
25446        assert_eq!(task.total_history_events, Some(3));
25447        assert_eq!(task.next_history_page_token, None);
25448
25449        let signal = task
25450            .history_events
25451            .iter()
25452            .find(|event| event.event_type == "SignalReceived")
25453            .expect("signal event");
25454        assert_eq!(
25455            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
25456            vec![AvroValue::String("Rust".to_string())]
25457        );
25458    }
25459
25460    #[tokio::test]
25461    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
25462        let client = Client::new("http://127.0.0.1:8080").expect("client");
25463        let mut worker = Worker::new(client, "rust-workers");
25464        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25465        worker.register_query("counter", "current", |ctx, _args| async move {
25466            let mut count = 0_i64;
25467            for signal in ctx.signal_events() {
25468                let value = signal
25469                    .arguments
25470                    .first()
25471                    .and_then(Value::as_i64)
25472                    .unwrap_or_default();
25473                match signal.name.as_str() {
25474                    "increment" => count += value,
25475                    "set" => count = value,
25476                    _ => {}
25477                }
25478            }
25479            Ok(json!(count))
25480        });
25481
25482        let task = QueryTask {
25483            query_task_id: "query-rust-counter".to_string(),
25484            query_task_attempt: 1,
25485            lease_owner: Some("rust-worker".to_string()),
25486            workflow_id: Some("counter-1".to_string()),
25487            run_id: Some("run-counter-1".to_string()),
25488            workflow_type: "counter".to_string(),
25489            query_name: "current".to_string(),
25490            payload_codec: DEFAULT_CODEC.to_string(),
25491            workflow_arguments: Some(
25492                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
25493            ),
25494            query_arguments: Some(
25495                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
25496            ),
25497            history_events: vec![
25498                HistoryEvent {
25499                    event_type: "SignalReceived".to_string(),
25500                    payload: json!({
25501                        "signal_id": "php-signal-1",
25502                        "signal_name": "increment",
25503                        "workflow_sequence": 1,
25504                        "payload_codec": DEFAULT_CODEC,
25505                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
25506                    }),
25507                    raw: HashMap::new(),
25508                },
25509                HistoryEvent {
25510                    event_type: "SignalReceived".to_string(),
25511                    payload: json!({
25512                        "signal_id": "python-signal-2",
25513                        "signal_name": "increment",
25514                        "workflow_sequence": 2,
25515                        "payload_codec": DEFAULT_CODEC,
25516                        "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
25517                    }),
25518                    raw: HashMap::new(),
25519                },
25520                HistoryEvent {
25521                    event_type: "SignalReceived".to_string(),
25522                    payload: json!({
25523                        "signal_id": "rust-signal-3",
25524                        "signal_name": "set",
25525                        "workflow_sequence": 3,
25526                        "payload_codec": DEFAULT_CODEC,
25527                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
25528                    }),
25529                    raw: HashMap::new(),
25530                },
25531            ],
25532            history_export: None,
25533            run_status: Some("completed".to_string()),
25534        };
25535
25536        let result = worker.execute_query_task(task).await.expect("query result");
25537        assert_eq!(result.into_json().expect("query projection"), json!(0));
25538    }
25539
25540    #[tokio::test]
25541    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
25542        let worker = replay_counter_worker();
25543        let running_history = json!([
25544            {
25545                "type": "ActivityCompleted",
25546                "payload": {
25547                    "sequence": 1,
25548                    "activity_type": "load-counter",
25549                    "payload_codec": DEFAULT_CODEC,
25550                    "result": fixture_envelope(json!("loaded"))
25551                }
25552            },
25553            {
25554                "type": "SignalWaitOpened",
25555                "payload": {
25556                    "sequence": 3,
25557                    "signal_name": "increment"
25558                }
25559            },
25560            {
25561                "type": "SignalReceived",
25562                "payload": {
25563                    "signal_id": "signal-3",
25564                    "signal_name": "increment",
25565                    "workflow_sequence": 2,
25566                    "payload_codec": DEFAULT_CODEC,
25567                    "arguments": fixture_envelope(json!([3]))
25568                }
25569            },
25570            {
25571                "type": "SignalApplied",
25572                "payload": {
25573                    "sequence": 3,
25574                    "signal_id": "signal-3",
25575                    "signal_name": "increment",
25576                    "payload_codec": DEFAULT_CODEC,
25577                    "value": fixture_envelope(json!([3]))
25578                }
25579            }
25580        ]);
25581
25582        let running = worker
25583            .execute_query_task(replay_counter_query(
25584                "current",
25585                running_history.clone(),
25586                "running",
25587            ))
25588            .await
25589            .expect("running replay query");
25590        assert_eq!(
25591            running.clone().into_json().expect("query projection"),
25592            json!({"loaded": "loaded", "count": 3, "finished": false})
25593        );
25594
25595        let detached = worker
25596            .execute_query_task(replay_counter_query(
25597                "detached-mutation",
25598                running_history.clone(),
25599                "running",
25600            ))
25601            .await
25602            .expect("query mutates only its detached state clone");
25603        assert_eq!(detached.into_json().expect("query projection"), json!(999));
25604        let failed = worker
25605            .execute_query_task(replay_counter_query(
25606                "failed-mutation",
25607                running_history.clone(),
25608                "running",
25609            ))
25610            .await
25611            .expect_err("failed query");
25612        assert_eq!(failed.reason, "query_rejected");
25613        let unchanged = worker
25614            .execute_query_task(replay_counter_query("current", running_history, "running"))
25615            .await
25616            .expect("later query reconstructs unchanged state");
25617        assert_eq!(unchanged, running);
25618
25619        let restarted_worker = replay_counter_worker();
25620        let empty_arguments = fixture_envelope(json!([]));
25621        let loaded_result = fixture_envelope(json!("loaded"));
25622        let signal_three = fixture_blob(json!([3]));
25623        let signal_five = fixture_blob(json!([5]));
25624        let restarted_task: QueryTask = serde_json::from_value(json!({
25625            "query_task_id": "query-after-restart",
25626            "workflow_id": "counter-1",
25627            "run_id": "run-counter-1",
25628            "workflow_type": "replay-counter",
25629            "query_name": "current",
25630            "payload_codec": DEFAULT_CODEC,
25631            "workflow_arguments": empty_arguments.clone(),
25632            "query_arguments": empty_arguments,
25633            "history_events": [],
25634            "history_export": {
25635                "payloads": {"codec": DEFAULT_CODEC},
25636                "history_events": [
25637                    {
25638                        "type": "ActivityCompleted",
25639                        "payload": {
25640                            "sequence": 1,
25641                            "activity_type": "load-counter",
25642                            "payload_codec": DEFAULT_CODEC,
25643                            "result": null
25644                        }
25645                    },
25646                    {
25647                        "type": "SignalWaitOpened",
25648                        "payload": {
25649                            "sequence": 3,
25650                            "signal_name": "increment"
25651                        }
25652                    },
25653                    {
25654                        "type": "SignalReceived",
25655                        "payload": {
25656                            "signal_id": "signal-3",
25657                            "signal_name": "increment",
25658                            "workflow_sequence": 2
25659                        }
25660                    },
25661                    {
25662                        "type": "SignalApplied",
25663                        "payload": {
25664                            "sequence": 3,
25665                            "signal_id": "signal-3",
25666                            "signal_name": "increment"
25667                        }
25668                    },
25669                    {
25670                        "type": "SignalWaitOpened",
25671                        "payload": {
25672                            "sequence": 5,
25673                            "signal_name": "increment"
25674                        }
25675                    },
25676                    {
25677                        "type": "SignalReceived",
25678                        "payload": {
25679                            "signal_id": "signal-5",
25680                            "signal_name": "increment",
25681                            "workflow_sequence": 4
25682                        }
25683                    },
25684                    {
25685                        "type": "SignalApplied",
25686                        "payload": {
25687                            "sequence": 5,
25688                            "signal_id": "signal-5",
25689                            "signal_name": "increment"
25690                        }
25691                    }
25692                ],
25693                "activities": [{
25694                    "sequence": 1,
25695                    "activity_type": "load-counter",
25696                    "payload_codec": DEFAULT_CODEC,
25697                    "result": loaded_result
25698                }],
25699                "signals": [
25700                    {
25701                        "id": "signal-3",
25702                        "name": "increment",
25703                        "workflow_sequence": 2,
25704                        "payload_codec": DEFAULT_CODEC,
25705                        "arguments": signal_three
25706                    },
25707                    {
25708                        "id": "signal-5",
25709                        "name": "increment",
25710                        "workflow_sequence": 4,
25711                        "payload_codec": DEFAULT_CODEC,
25712                        "arguments": signal_five
25713                    }
25714                ]
25715            },
25716            "run_status": "completed"
25717        }))
25718        .expect("cold replay query task");
25719        let completed = restarted_worker
25720            .execute_query_task(restarted_task)
25721            .await
25722            .expect("completed cold replay query");
25723        assert_eq!(
25724            completed.into_json().expect("query projection"),
25725            json!({"loaded": "loaded", "count": 8, "finished": true})
25726        );
25727    }
25728
25729    #[tokio::test]
25730    async fn replayed_query_replay_failures_are_machine_readable() {
25731        let worker = replay_counter_worker();
25732        let task = replay_counter_query(
25733            "current",
25734            json!([{
25735                "type": "ActivityCompleted",
25736                "payload": {
25737                    "sequence": 1,
25738                    "payload_codec": DEFAULT_CODEC,
25739                    "result": {"codec": DEFAULT_CODEC, "blob": "{"}
25740                }
25741            }]),
25742            "running",
25743        );
25744        let failure = worker
25745            .execute_query_task(task)
25746            .await
25747            .expect_err("invalid replay history payload");
25748        assert_eq!(failure.reason, "query_payload_decode_failed");
25749        assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
25750        assert!(failure.message.contains("invalid_payload_framing"));
25751    }
25752
25753    #[tokio::test]
25754    async fn query_task_restores_compact_history_from_export() {
25755        let client = Client::new("http://127.0.0.1:8080").expect("client");
25756        let mut worker = Worker::new(client, "rust-workers");
25757        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25758        worker.register_query("counter", "current", |ctx, _args| async move {
25759            Ok(json!(ctx.signals("increment")[0][0]))
25760        });
25761        let empty_arguments = fixture_envelope(json!([]));
25762        let exported_signal = fixture_blob(json!([9]));
25763        let task: QueryTask = serde_json::from_value(json!({
25764            "query_task_id": "query-export",
25765            "workflow_type": "counter",
25766            "query_name": "current",
25767            "payload_codec": DEFAULT_CODEC,
25768            "workflow_arguments": empty_arguments.clone(),
25769            "query_arguments": empty_arguments,
25770            "history_events": [],
25771            "history_export": {
25772                "payloads": {"codec": DEFAULT_CODEC},
25773                "history_events": [{
25774                    "type": "SignalReceived",
25775                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
25776                }],
25777                "signals": [{
25778                    "id": "signal-export",
25779                    "name": "increment",
25780                    "status": "applied",
25781                    "workflow_sequence": 1,
25782                    "payload_codec": DEFAULT_CODEC,
25783                    "arguments": exported_signal
25784                }]
25785            }
25786        }))
25787        .expect("query task");
25788
25789        let result = worker.execute_query_task(task).await.expect("query result");
25790        assert_eq!(result.into_json().expect("query projection"), json!(9));
25791    }
25792
25793    #[tokio::test]
25794    async fn query_task_failures_have_stable_reasons() {
25795        let client = Client::new("http://127.0.0.1:8080").expect("client");
25796        let mut worker = Worker::new(client, "rust-workers");
25797        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25798        worker.register_query(
25799            "counter",
25800            "current",
25801            |_ctx, _args| async move { Ok(json!(0)) },
25802        );
25803
25804        let base_task = QueryTask {
25805            query_task_id: "query-errors".to_string(),
25806            query_task_attempt: 1,
25807            lease_owner: None,
25808            workflow_id: Some("counter-errors".to_string()),
25809            run_id: Some("run-errors".to_string()),
25810            workflow_type: "counter".to_string(),
25811            query_name: "missing".to_string(),
25812            payload_codec: DEFAULT_CODEC.to_string(),
25813            workflow_arguments: Some(fixture_envelope(json!([]))),
25814            query_arguments: Some(fixture_envelope(json!([]))),
25815            history_events: Vec::new(),
25816            history_export: None,
25817            run_status: Some("running".to_string()),
25818        };
25819
25820        let unknown = worker
25821            .execute_query_task(base_task.clone())
25822            .await
25823            .expect_err("unknown query");
25824        assert_eq!(unknown.reason, "rejected_unknown_query");
25825
25826        let mut malformed = base_task;
25827        malformed.query_name = "current".to_string();
25828        malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
25829        let malformed = worker
25830            .execute_query_task(malformed)
25831            .await
25832            .expect_err("malformed payload");
25833        assert_eq!(malformed.reason, "query_payload_decode_failed");
25834
25835        let client = Client::new("http://127.0.0.1:8080").expect("client");
25836        let mut unavailable_worker = Worker::new(client, "rust-workers");
25837        unavailable_worker
25838            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25839        let empty_arguments = fixture_envelope(json!([]));
25840        let unavailable_task: QueryTask = serde_json::from_value(json!({
25841            "query_task_id": "query-unavailable",
25842            "workflow_type": "counter",
25843            "query_name": "current",
25844            "payload_codec": DEFAULT_CODEC,
25845            "workflow_arguments": empty_arguments.clone(),
25846            "query_arguments": empty_arguments
25847        }))
25848        .expect("query task");
25849        let unavailable = unavailable_worker
25850            .execute_query_task(unavailable_task)
25851            .await
25852            .expect_err("query handler unavailable");
25853        assert_eq!(unavailable.reason, "query_handler_unavailable");
25854    }
25855
25856    #[tokio::test]
25857    async fn client_query_decodes_result_and_typed_failure() {
25858        let server = MockWorkerServer::start();
25859        let client = Client::builder(server.base_url())
25860            .timeout(Duration::from_secs(2))
25861            .build()
25862            .expect("client");
25863
25864        let result = client
25865            .query_workflow("counter-1", "current", json!([]))
25866            .await
25867            .expect("query result");
25868        assert_eq!(result, json!({"count": 8}));
25869
25870        let error = client
25871            .query_workflow("counter-1", "missing", json!([]))
25872            .await
25873            .expect_err("unknown query");
25874        let Error::QueryFailed(failure) = error else {
25875            panic!("expected typed query failure");
25876        };
25877        assert_eq!(failure.status, 404);
25878        assert_eq!(failure.reason, "rejected_unknown_query");
25879    }
25880
25881    #[tokio::test]
25882    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
25883        let server = MockWorkerServer::start();
25884        let client = Client::builder(server.base_url())
25885            .timeout(Duration::from_secs(2))
25886            .build()
25887            .expect("client");
25888        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
25889
25890        client
25891            .start_workflow(
25892                "typed.echo",
25893                "rust-workers",
25894                "typed-start",
25895                arguments.clone(),
25896            )
25897            .await
25898            .expect("typed workflow start");
25899        assert_eq!(
25900            decode_wire_avro_value(
25901                &server.request_body("/api/workflows")["input"],
25902                DEFAULT_CODEC,
25903            )
25904            .expect("typed start input"),
25905            arguments
25906        );
25907
25908        client
25909            .signal_workflow("typed-1", "changed", arguments.clone())
25910            .await
25911            .expect("typed signal");
25912        assert_eq!(
25913            decode_wire_avro_value(
25914                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
25915                DEFAULT_CODEC,
25916            )
25917            .expect("typed signal input"),
25918            arguments
25919        );
25920
25921        assert_eq!(
25922            client
25923                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
25924                .await
25925                .expect("typed query"),
25926            typed_fidelity_probe()
25927        );
25928        assert_eq!(
25929            decode_wire_avro_value(
25930                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
25931                DEFAULT_CODEC,
25932            )
25933            .expect("typed query input"),
25934            arguments
25935        );
25936
25937        assert_eq!(
25938            client
25939                .update_workflow_avro_value(
25940                    "typed-1",
25941                    "replace",
25942                    arguments.clone(),
25943                    Some("typed-request"),
25944                )
25945                .await
25946                .expect("typed update"),
25947            typed_fidelity_probe()
25948        );
25949        let update = server.request_body("/api/workflows/typed-1/update/replace");
25950        assert_eq!(update["request_id"], "typed-request");
25951        assert_eq!(
25952            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
25953            arguments
25954        );
25955
25956        let handle = WorkflowHandle {
25957            client: client.clone(),
25958            workflow_id: "typed-1".to_string(),
25959            run_id: Some("run-typed-1".to_string()),
25960            workflow_type: "typed.echo".to_string(),
25961        };
25962        assert_eq!(
25963            handle
25964                .result_avro_value(WorkflowResultOptions::default())
25965                .await
25966                .expect("typed workflow result"),
25967            typed_fidelity_probe()
25968        );
25969
25970        client
25971            .complete_activity_task(
25972                "activity-typed",
25973                "attempt-typed",
25974                "rust-worker",
25975                typed_fidelity_probe(),
25976                DEFAULT_CODEC,
25977            )
25978            .await
25979            .expect("typed activity completion");
25980        assert_eq!(
25981            decode_wire_avro_value(
25982                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
25983                    ["result"],
25984                DEFAULT_CODEC,
25985            )
25986            .expect("typed activity result"),
25987            typed_fidelity_probe()
25988        );
25989        client
25990            .fail_activity_task(
25991                "activity-typed",
25992                "attempt-typed",
25993                "rust-worker",
25994                "typed failure",
25995                true,
25996            )
25997            .await
25998            .expect("activity failure");
25999    }
26000
26001    #[tokio::test]
26002    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
26003        let server = MockWorkerServer::start();
26004        let client = Client::builder(server.base_url())
26005            .timeout(Duration::from_secs(2))
26006            .build()
26007            .expect("client");
26008
26009        let options = WorkflowCommandOptions::new()
26010            .reason("cleanup requested")
26011            .request_id("cancel-17");
26012        let cancelled = client
26013            .cancel_workflow("wf-lifecycle", options)
26014            .await
26015            .expect("instance cancellation");
26016        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
26017        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
26018        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
26019        assert_eq!(
26020            server.request_body("/api/workflows/wf-lifecycle/cancel"),
26021            json!({"reason":"cleanup requested","request_id":"cancel-17"})
26022        );
26023
26024        let terminated = client
26025            .terminate_workflow(
26026                "wf-lifecycle",
26027                WorkflowCommandOptions::new().reason("forced stop"),
26028            )
26029            .await
26030            .expect("instance termination");
26031        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
26032        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
26033
26034        client
26035            .cancel_workflow_run(
26036                "wf-lifecycle",
26037                "run-current",
26038                WorkflowCommandOptions::default(),
26039            )
26040            .await
26041            .expect("selected run cancellation");
26042        client
26043            .terminate_workflow_run(
26044                "wf-lifecycle",
26045                "run-current",
26046                WorkflowCommandOptions::default(),
26047            )
26048            .await
26049            .expect("selected run termination");
26050
26051        for (command, error) in [
26052            (
26053                WorkflowCommandKind::Cancel,
26054                client
26055                    .cancel_workflow_run(
26056                        "wf-lifecycle",
26057                        "run-stale",
26058                        WorkflowCommandOptions::default(),
26059                    )
26060                    .await
26061                    .expect_err("stale cancellation must be rejected"),
26062            ),
26063            (
26064                WorkflowCommandKind::Terminate,
26065                client
26066                    .terminate_workflow_run(
26067                        "wf-lifecycle",
26068                        "run-stale",
26069                        WorkflowCommandOptions::default(),
26070                    )
26071                    .await
26072                    .expect_err("stale termination must be rejected"),
26073            ),
26074        ] {
26075            let Error::WorkflowCommandRejected(rejection) = error else {
26076                panic!("expected typed command rejection");
26077            };
26078            assert_eq!(rejection.command, command);
26079            assert_eq!(rejection.status, 409);
26080            assert_eq!(rejection.reason, "historical_run_command_rejected");
26081            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
26082            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
26083        }
26084    }
26085
26086    #[tokio::test]
26087    async fn redrive_targets_a_failed_run_and_returns_successor_identity() {
26088        let server = MockWorkerServer::start();
26089        let client = Client::builder(server.base_url())
26090            .timeout(Duration::from_secs(2))
26091            .build()
26092            .expect("client");
26093
26094        let result = client
26095            .redrive_workflow_run("wf-lifecycle", "run-failed", Some("retry-1"))
26096            .await
26097            .expect("redrive accepted");
26098        assert_eq!(result.workflow_id, "wf-lifecycle");
26099        assert_eq!(result.source_run_id, "run-failed");
26100        assert_eq!(result.run_id, "run-successor");
26101        assert_eq!(result.resume_step_sequence, 2);
26102        assert_eq!(
26103            server.request_body("/api/workflows/wf-lifecycle/runs/run-failed/redrive"),
26104            json!({"request_id":"retry-1"})
26105        );
26106
26107        let repeated = client
26108            .redrive_workflow_run("wf-lifecycle", "run-failed-existing", Some("retry-1"))
26109            .await
26110            .expect("idempotent redrive response accepted");
26111        assert_eq!(repeated.run_id, "run-successor");
26112
26113        let error = client
26114            .redrive_workflow_run("wf-lifecycle", "run-completed", None)
26115            .await
26116            .expect_err("completed run must be rejected");
26117        let Error::Http { status, body } = error else {
26118            panic!("expected HTTP redrive rejection");
26119        };
26120        assert_eq!(status.as_u16(), 409);
26121        assert!(body.contains("run_not_failed"));
26122        assert_eq!(
26123            server.request_body("/api/workflows/wf-lifecycle/runs/run-completed/redrive"),
26124            json!({})
26125        );
26126
26127        let handle = WorkflowHandle {
26128            client: client.clone(),
26129            workflow_id: "wf-lifecycle".to_string(),
26130            run_id: Some("run-failed".to_string()),
26131            workflow_type: "test".to_string(),
26132        };
26133        assert_eq!(
26134            handle.redrive(None).await.expect("handle redrive").run_id,
26135            "run-successor"
26136        );
26137        let missing_run = WorkflowHandle {
26138            run_id: None,
26139            ..handle
26140        };
26141        assert!(matches!(
26142            missing_run.redrive(None).await,
26143            Err(Error::Codec(_))
26144        ));
26145    }
26146
26147    #[tokio::test]
26148    async fn workflow_start_options_send_server_enforced_deadlines() {
26149        let server = MockWorkerServer::start();
26150        let client = Client::builder(server.base_url())
26151            .timeout(Duration::from_secs(2))
26152            .build()
26153            .expect("client");
26154
26155        let handle = client
26156            .start_workflow_with_options(
26157                "rust.timeout",
26158                "rust-timeouts",
26159                "wf-start-options",
26160                WorkflowStartOptions::new()
26161                    .execution_timeout_seconds(30)
26162                    .run_timeout_seconds(1),
26163                json!([]),
26164            )
26165            .await
26166            .expect("workflow start");
26167
26168        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
26169        let body = server.request_body("/api/workflows");
26170        assert_eq!(body["execution_timeout_seconds"], 30);
26171        assert_eq!(body["run_timeout_seconds"], 1);
26172
26173        let invalid = client
26174            .start_workflow_with_options(
26175                "rust.timeout",
26176                "rust-timeouts",
26177                "wf-invalid-options",
26178                WorkflowStartOptions::new()
26179                    .execution_timeout_seconds(1)
26180                    .run_timeout_seconds(2),
26181                json!([]),
26182            )
26183            .await
26184            .expect_err("invalid deadline ordering");
26185        assert!(invalid
26186            .to_string()
26187            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
26188    }
26189
26190    #[tokio::test]
26191    async fn workflow_result_returns_each_typed_terminal_outcome() {
26192        let server = MockWorkerServer::start();
26193        let client = Client::builder(server.base_url())
26194            .timeout(Duration::from_secs(2))
26195            .build()
26196            .expect("client");
26197        let options = WorkflowResultOptions {
26198            poll_interval: Duration::ZERO,
26199            timeout: Duration::from_secs(1),
26200        };
26201
26202        let failed = WorkflowHandle {
26203            client: client.clone(),
26204            workflow_id: "wf-failed".to_string(),
26205            run_id: Some("run-failed".to_string()),
26206            workflow_type: "failure".to_string(),
26207        }
26208        .result(options)
26209        .await
26210        .expect_err("failed outcome");
26211        let Error::WorkflowFailed(failure) = failed else {
26212            panic!("expected WorkflowFailed");
26213        };
26214        assert_eq!(failure.workflow_id, "wf-failed");
26215        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
26216        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
26217        assert_eq!(failure.failure_category.as_deref(), Some("application"));
26218        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
26219        assert_eq!(
26220            failure.exception_class.as_deref(),
26221            Some("billing::PaymentError")
26222        );
26223        assert_eq!(failure.non_retryable, Some(true));
26224
26225        for (workflow_id, expected_kind, expected_reason) in [
26226            (
26227                "wf-cancelled",
26228                WorkflowTerminalKind::Cancelled,
26229                "cleanup requested",
26230            ),
26231            (
26232                "wf-terminated",
26233                WorkflowTerminalKind::Terminated,
26234                "forced stop",
26235            ),
26236            (
26237                "wf-timed-out",
26238                WorkflowTerminalKind::TimedOut,
26239                "run_timeout",
26240            ),
26241        ] {
26242            let error = WorkflowHandle {
26243                client: client.clone(),
26244                workflow_id: workflow_id.to_string(),
26245                run_id: None,
26246                workflow_type: "terminal".to_string(),
26247            }
26248            .result(options)
26249            .await
26250            .expect_err("typed terminal outcome");
26251            let outcome = match error {
26252                Error::WorkflowCancelled(outcome) => outcome,
26253                Error::WorkflowTerminated(outcome) => outcome,
26254                Error::WorkflowTimedOut(outcome) => outcome,
26255                other => panic!("unexpected terminal error: {other}"),
26256            };
26257            assert_eq!(outcome.kind, expected_kind);
26258            assert_eq!(outcome.workflow_id, workflow_id);
26259            assert_eq!(outcome.reason, expected_reason);
26260        }
26261
26262        let wait_timeout = WorkflowHandle {
26263            client,
26264            workflow_id: "wf-waiting".to_string(),
26265            run_id: Some("run-waiting".to_string()),
26266            workflow_type: "waiting".to_string(),
26267        }
26268        .result(WorkflowResultOptions {
26269            poll_interval: Duration::ZERO,
26270            timeout: Duration::ZERO,
26271        })
26272        .await
26273        .expect_err("client wait timeout");
26274        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
26275            panic!("expected typed client timeout");
26276        };
26277        assert_eq!(timeout.reason, "result_wait_timeout");
26278        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
26279        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
26280    }
26281
26282    #[tokio::test]
26283    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
26284        let server = MockWorkerServer::start();
26285        let client = Client::builder(server.base_url())
26286            .timeout(Duration::from_secs(2))
26287            .build()
26288            .expect("client");
26289
26290        let handle = WorkflowHandle {
26291            client,
26292            workflow_id: "wf-selected".to_string(),
26293            run_id: Some("run-selected".to_string()),
26294            workflow_type: "selected".to_string(),
26295        };
26296        let options = WorkflowResultOptions {
26297            poll_interval: Duration::ZERO,
26298            timeout: Duration::from_secs(1),
26299        };
26300
26301        let current = handle
26302            .result(options)
26303            .await
26304            .expect("instance result follows the current run");
26305        assert_eq!(current, json!("current run output"));
26306
26307        let error = handle
26308            .result_selected_run(options)
26309            .await
26310            .expect_err("the selected run is cancelled even though the current run completed");
26311
26312        let Error::WorkflowCancelled(outcome) = error else {
26313            panic!("expected selected run cancellation");
26314        };
26315        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
26316        assert_eq!(outcome.reason, "selected run cancelled");
26317        assert_eq!(
26318            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
26319            1
26320        );
26321        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
26322    }
26323
26324    #[tokio::test]
26325    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
26326        let server = MockWorkerServer::draining_polls();
26327        let client = Client::builder(server.base_url())
26328            .timeout(Duration::from_secs(2))
26329            .build()
26330            .expect("client");
26331
26332        let workflow = client
26333            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
26334            .await
26335            .expect("workflow drain response");
26336        let activity = client
26337            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
26338            .await
26339            .expect("activity drain response");
26340        let query = client
26341            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
26342            .await
26343            .expect("query drain response");
26344
26345        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
26346            assert_eq!(
26347                outcome,
26348                WorkerPollOutcome::Stop {
26349                    poll_status: Some("draining".to_string()),
26350                    reason: Some("worker_draining".to_string()),
26351                }
26352            );
26353        }
26354
26355        assert!(client
26356            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
26357            .await
26358            .expect("compatibility poll")
26359            .is_none());
26360    }
26361
26362    #[tokio::test]
26363    async fn managed_worker_honors_drain_stop_for_every_task_family() {
26364        let server = MockWorkerServer::draining_polls();
26365        let client = Client::builder(server.base_url())
26366            .timeout(Duration::from_secs(2))
26367            .build()
26368            .expect("client");
26369
26370        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
26371            .worker_id("draining-workflow-worker")
26372            .poll_timeout(Duration::ZERO);
26373        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
26374        workflow_worker
26375            .run()
26376            .await
26377            .expect("workflow drain is a clean stop");
26378
26379        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
26380            .worker_id("draining-activity-worker")
26381            .poll_timeout(Duration::ZERO);
26382        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
26383        activity_worker
26384            .run()
26385            .await
26386            .expect("activity drain is a clean stop");
26387
26388        let mut query_worker = Worker::new(client, "rust-workers")
26389            .worker_id("draining-query-worker")
26390            .poll_timeout(Duration::ZERO);
26391        query_worker.register_query("counter", "current", |_ctx, _args| async {
26392            Ok(Value::Null)
26393        });
26394        query_worker
26395            .run()
26396            .await
26397            .expect("query drain is a clean stop");
26398    }
26399
26400    #[tokio::test]
26401    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
26402        let server = MockWorkerServer::start();
26403        let client = Client::builder(server.base_url())
26404            .timeout(Duration::from_secs(2))
26405            .build()
26406            .expect("client");
26407
26408        let details: Value = serde_json::from_str(include_str!(
26409            "../tests/fixtures/activity-heartbeat-progress.json"
26410        ))
26411        .expect("heartbeat progress fixture");
26412        // Every individual field fits Server's progress contract, while the
26413        // obsolete encoded envelope exceeded its 191-character string limit.
26414        assert!(
26415            encode_typed_envelope(&AvroValue::from_serialize(&details).unwrap(), DEFAULT_CODEC)
26416                .unwrap()["blob"]
26417                .as_str()
26418                .unwrap()
26419                .len()
26420                > 191
26421        );
26422        let heartbeat = client
26423            .heartbeat_activity_task("activity-cancel", "attempt-cancel", "rust-worker", &details)
26424            .await
26425            .expect("cancellation heartbeat");
26426        assert!(heartbeat.cancel_requested);
26427        assert!(heartbeat.should_stop());
26428        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
26429        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
26430        let heartbeat_body =
26431            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
26432        assert_eq!(heartbeat_body["details"], details);
26433        assert_eq!(heartbeat_body["activity_attempt_id"], "attempt-cancel");
26434        assert_eq!(heartbeat_body["lease_owner"], "rust-worker");
26435
26436        let error = client
26437            .complete_activity_task(
26438                "activity-cancel",
26439                "attempt-cancel",
26440                "rust-worker",
26441                json!({"late":true}),
26442                DEFAULT_CODEC,
26443            )
26444            .await
26445            .expect_err("late completion must be refused");
26446        assert!(activity_task_rejection_is_final(&error));
26447        let Error::ActivityTaskRejected(rejection) = error else {
26448            panic!("expected typed activity rejection");
26449        };
26450        assert_eq!(rejection.status, 409);
26451        assert_eq!(rejection.reason, "run_cancelled");
26452        assert!(rejection.cancel_requested);
26453        assert_eq!(rejection.can_continue, Some(false));
26454    }
26455
26456    #[tokio::test]
26457    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
26458        let server = MockWorkerServer::cancelled_activity();
26459        let client = Client::builder(server.base_url())
26460            .timeout(Duration::from_secs(2))
26461            .build()
26462            .expect("client");
26463        let cancellation_observed = Arc::new(AtomicBool::new(false));
26464        let observed = Arc::clone(&cancellation_observed);
26465        let mut worker = Worker::new(client.clone(), "rust-workers")
26466            .worker_id("rust-cancel-worker")
26467            .poll_timeout(Duration::from_millis(10));
26468        worker.register_activity("cancel-aware", move |ctx, _args| {
26469            let observed = Arc::clone(&observed);
26470            async move {
26471                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
26472                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
26473                Ok(json!({"late":"completion"}))
26474            }
26475        });
26476
26477        assert_eq!(
26478            worker.run_once().await.expect("cancelled attempt handled"),
26479            1
26480        );
26481        assert!(cancellation_observed.load(Ordering::SeqCst));
26482        assert_eq!(
26483            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
26484            1
26485        );
26486
26487        let mut restarted = Worker::new(client, "rust-workers")
26488            .worker_id("rust-cancel-worker-restarted")
26489            .poll_timeout(Duration::from_millis(10));
26490        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
26491        assert_eq!(
26492            restarted
26493                .run_once()
26494                .await
26495                .expect("replacement worker continues polling"),
26496            0
26497        );
26498    }
26499
26500    #[tokio::test]
26501    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
26502        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
26503        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
26504        let client = Client::builder(server.base_url())
26505            .timeout(Duration::from_secs(2))
26506            .build()
26507            .expect("client");
26508
26509        let direct_error = client
26510            .complete_workflow_task(
26511                "workflow-timeout-task",
26512                "timeout-worker",
26513                3,
26514                vec![json!({
26515                    "type": "complete_workflow",
26516                    "result": fixture_envelope(Value::Null)
26517                })],
26518            )
26519            .await
26520            .expect_err("the low-level client preserves the completion rejection");
26521        let Error::Http { status, body } = direct_error else {
26522            panic!("expected the original HTTP completion rejection");
26523        };
26524        assert_eq!(status, reqwest::StatusCode::CONFLICT);
26525        assert_eq!(
26526            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
26527            "run_timed_out"
26528        );
26529
26530        let mut worker = Worker::new(client, "rust-workers")
26531            .worker_id("timeout-worker")
26532            .poll_timeout(Duration::from_millis(10));
26533        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
26534            Ok(json!({"late": "result"}))
26535        });
26536
26537        assert_eq!(
26538            worker
26539                .run_once()
26540                .await
26541                .expect("authoritative selected-run timeout settles the tick"),
26542            1
26543        );
26544        assert_eq!(
26545            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
26546            2,
26547            "both the direct client proof and managed worker must see the rejection"
26548        );
26549    }
26550
26551    #[tokio::test]
26552    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
26553        for (name, status, response) in [
26554            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
26555            (
26556                "command was recorded",
26557                "409 Conflict",
26558                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
26559            ),
26560            (
26561                "lease conflict",
26562                "409 Conflict",
26563                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
26564            ),
26565            (
26566                "nonterminal run",
26567                "409 Conflict",
26568                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
26569            ),
26570            (
26571                "different selected run",
26572                "409 Conflict",
26573                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
26574            ),
26575            (
26576                "different task attempt",
26577                "409 Conflict",
26578                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
26579            ),
26580            (
26581                "authentication failure",
26582                "401 Unauthorized",
26583                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
26584            ),
26585            (
26586                "authorization failure",
26587                "403 Forbidden",
26588                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
26589            ),
26590            (
26591                "protocol failure",
26592                "400 Bad Request",
26593                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
26594            ),
26595            (
26596                "malformed command",
26597                "422 Unprocessable Entity",
26598                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
26599            ),
26600            (
26601                "transient server failure",
26602                "503 Service Unavailable",
26603                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
26604            ),
26605        ] {
26606            let server = MockWorkerServer::workflow_completion(status, response);
26607            let client = Client::builder(server.base_url())
26608                .timeout(Duration::from_secs(2))
26609                .build()
26610                .expect("client");
26611            let mut worker = Worker::new(client, "rust-workers")
26612                .worker_id("timeout-worker")
26613                .poll_timeout(Duration::from_millis(10));
26614            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
26615                Ok(json!({"late": "result"}))
26616            });
26617
26618            let error = worker
26619                .run_once()
26620                .await
26621                .expect_err(&format!("{name} must remain an error"));
26622            assert!(
26623                matches!(error, Error::Http { .. } | Error::Protocol(_)),
26624                "{name} returned an unexpected error variant: {error}"
26625            );
26626        }
26627    }
26628
26629    #[tokio::test]
26630    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
26631        let server = MockWorkerServer::start();
26632        let client = Client::builder(server.base_url())
26633            .worker_token(Some("worker-secret".to_string()))
26634            .namespace("orders")
26635            .timeout(Duration::from_secs(2))
26636            .build()
26637            .expect("client");
26638        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
26639
26640        let result = client
26641            .deregister_worker_registration("worker/α space")
26642            .await
26643            .expect("deregister worker registration");
26644
26645        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
26646        assert_eq!(
26647            server.worker_protocol_for(path).as_deref(),
26648            Some(WORKER_PROTOCOL_VERSION)
26649        );
26650        assert_eq!(server.control_protocol_for(path), None);
26651        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
26652        assert_eq!(
26653            server.authorization_for(path).as_deref(),
26654            Some("Bearer worker-secret")
26655        );
26656        assert_eq!(
26657            result,
26658            WorkerDeregistrationEnvelope {
26659                worker_id: "deregistered-worker".to_string(),
26660                outcome: "deregistered".to_string(),
26661                recovered_workflow_task_count: 2,
26662            }
26663        );
26664    }
26665
26666    #[tokio::test]
26667    async fn low_level_registration_rejects_update_validators_before_transport() {
26668        let server = MockWorkerServer::start();
26669        let client = Client::builder(server.base_url())
26670            .timeout(Duration::from_secs(2))
26671            .build()
26672            .expect("client");
26673
26674        for update_validators in [json!(["approve"]), json!("approve")] {
26675            let error = client
26676                .register_worker_with_command_contracts(
26677                    "validator-claiming-worker",
26678                    "rust-workers",
26679                    vec!["orders".to_string()],
26680                    vec![],
26681                    1,
26682                    1,
26683                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
26684                    json!({
26685                        "orders": {
26686                            "queries": ["current"],
26687                            "updates": ["approve"],
26688                            "update_validators": update_validators,
26689                        },
26690                    }),
26691                )
26692                .await
26693                .expect_err("unsupported validator claims must fail before registration");
26694
26695            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
26696                panic!("expected typed unsupported-validator failure");
26697            };
26698            assert_eq!(workflow_type, "orders");
26699        }
26700        assert_eq!(server.request_count("/api/worker/register"), 0);
26701    }
26702
26703    #[tokio::test]
26704    async fn worker_registers_complete_json_and_lossless_handler_argument_contracts() {
26705        let server = MockWorkerServer::start();
26706        let client = Client::builder(server.base_url())
26707            .timeout(Duration::from_secs(2))
26708            .build()
26709            .expect("client");
26710        let mut worker = Worker::new(client, "rust-workers");
26711        worker.register_workflow("orders", |_ctx, input| async move { Ok(input) });
26712        worker.register_query("orders", "z-json", |_ctx, args| async move { Ok(args) });
26713        worker.register_query_avro_value("orders", "a-avro", |_ctx, args| async move { Ok(args) });
26714        worker.register_update("orders", "z-json", |_ctx, args| async move { Ok(args) });
26715        worker.register_update_avro_value("orders", "a-avro", |_ctx, args| async move { Ok(args) });
26716        worker
26717            .declare_workflow_signals("orders", &["z-json", "a-avro", "z-json"])
26718            .expect("declare workflow signals");
26719
26720        worker.register().await.expect("register handler contracts");
26721
26722        let declaration =
26723            &server.request_body("/api/worker/register")["workflow_command_contracts"]["orders"];
26724        let expected_contracts = json!([
26725            {
26726                "name": "a-avro",
26727                "parameters": [{
26728                    "name": "args", "position": 0, "required": false,
26729                    "variadic": true, "type": null, "allows_null": true,
26730                    "default_available": false, "default": null,
26731                }],
26732            },
26733            {
26734                "name": "z-json",
26735                "parameters": [{
26736                    "name": "args", "position": 0, "required": false,
26737                    "variadic": true, "type": null, "allows_null": true,
26738                    "default_available": false, "default": null,
26739                }],
26740            },
26741        ]);
26742        assert_eq!(declaration["queries"], json!(["a-avro", "z-json"]));
26743        assert_eq!(declaration["query_contracts"], expected_contracts);
26744        assert_eq!(declaration["updates"], json!(["a-avro", "z-json"]));
26745        assert_eq!(declaration["update_contracts"], expected_contracts);
26746        assert_eq!(declaration["signals"], json!(["a-avro", "z-json"]));
26747        assert_eq!(declaration["signal_contracts"], expected_contracts);
26748        assert_eq!(declaration["update_validators"], json!([]));
26749    }
26750
26751    #[tokio::test]
26752    async fn workflow_signal_declarations_reject_invalid_configuration_without_mutation() {
26753        let server = MockWorkerServer::start();
26754        let client = Client::new(server.base_url()).expect("client");
26755        let mut worker = Worker::new(client, "rust-workers");
26756        worker.register_workflow("orders", |_ctx, args| async move { Ok(args) });
26757        assert!(worker
26758            .declare_workflow_signals("missing", &["finish"])
26759            .is_err());
26760        worker
26761            .declare_workflow_signals("orders", &["finish"])
26762            .unwrap();
26763        for names in [&[""][..], &[" finish"][..], &["finish "][..]] {
26764            assert!(worker.declare_workflow_signals("orders", names).is_err());
26765        }
26766        worker.register().await.unwrap();
26767        assert_eq!(
26768            server.request_body("/api/worker/register")["workflow_command_contracts"]["orders"]
26769                ["signals"],
26770            json!(["finish"])
26771        );
26772        worker.declare_workflow_signals("orders", &[]).unwrap();
26773        worker.register().await.unwrap();
26774        let registrations = server.request_bodies("/api/worker/register");
26775        assert_eq!(
26776            registrations[1]["workflow_command_contracts"]["orders"]["signals"],
26777            json!([])
26778        );
26779        assert_eq!(
26780            registrations[1]["workflow_command_contracts"]["orders"]["signal_contracts"],
26781            json!([])
26782        );
26783    }
26784
26785    #[tokio::test]
26786    async fn low_level_registration_preserves_query_and_update_contracts() {
26787        let server = MockWorkerServer::start();
26788        let client = Client::builder(server.base_url())
26789            .timeout(Duration::from_secs(2))
26790            .build()
26791            .expect("client");
26792        let contracts = json!({
26793            "orders": {
26794                "queries": ["current"],
26795                "updates": ["approve"],
26796                "update_validators": [],
26797            },
26798            "payments": {
26799                "queries": ["status"],
26800                "updates": ["capture"],
26801            },
26802        });
26803
26804        client
26805            .register_worker_with_command_contracts(
26806                "command-worker",
26807                "rust-workers",
26808                vec!["orders".to_string(), "payments".to_string()],
26809                vec![],
26810                1,
26811                1,
26812                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
26813                contracts.clone(),
26814            )
26815            .await
26816            .expect("query and update contracts must remain supported");
26817
26818        assert_eq!(
26819            server.request_body("/api/worker/register")["workflow_command_contracts"],
26820            contracts
26821        );
26822        assert!(server
26823            .request_body("/api/worker/register")
26824            .get("workflow_definition_fingerprints")
26825            .is_none());
26826    }
26827
26828    #[tokio::test]
26829    async fn rust_worker_advertises_source_identity_without_reusing_it_for_a_new_handler() {
26830        let server = MockWorkerServer::start();
26831        let client = Client::builder(server.base_url())
26832            .timeout(Duration::from_secs(2))
26833            .build()
26834            .expect("client");
26835        let mut worker = Worker::new(client.clone(), "rust-workers");
26836        worker.register_workflow("orders", |_ctx, _input| async { Ok(Value::Null) });
26837
26838        assert!(worker
26839            .set_workflow_definition_sources("missing", &[include_str!("lib.rs")])
26840            .is_err());
26841        assert!(worker
26842            .set_workflow_definition_sources("orders", &[])
26843            .is_err());
26844        worker
26845            .set_workflow_definition_sources("orders", &[include_str!("lib.rs")])
26846            .expect("embed workflow source");
26847        worker.register().await.expect("register worker");
26848        let first = server.request_bodies("/api/worker/register")[0]
26849            ["workflow_definition_fingerprints"]["orders"]
26850            .as_str()
26851            .expect("source-backed fingerprint")
26852            .to_string();
26853        assert!(first.starts_with("sha256:"));
26854
26855        worker
26856            .set_workflow_definition_sources("orders", &[include_str!("lib.rs"), "changed helper"])
26857            .expect("change embedded helper source");
26858        worker.register().await.expect("register changed source");
26859        let changed = server.request_bodies("/api/worker/register")[1]
26860            ["workflow_definition_fingerprints"]["orders"]
26861            .as_str()
26862            .expect("changed fingerprint")
26863            .to_string();
26864        assert_ne!(first, changed);
26865
26866        let mut replacement =
26867            Worker::new(client, "rust-workers").worker_id(worker.worker_id.clone());
26868        replacement.register_workflow("orders", |_ctx, _input| async { Ok(Value::Null) });
26869        replacement
26870            .register()
26871            .await
26872            .expect("register replacement handler");
26873        assert_eq!(
26874            server.request_bodies("/api/worker/register")[2]["workflow_definition_fingerprints"],
26875            json!({})
26876        );
26877    }
26878
26879    #[tokio::test]
26880    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
26881        let server = MockWorkerServer::start();
26882        let control_only = Client::builder(server.base_url())
26883            .control_token(Some("control-secret".to_string()))
26884            .build()
26885            .expect("control client");
26886
26887        let error = control_only
26888            .register_worker("worker", "queue", vec![], vec![], 1, 1)
26889            .await
26890            .expect_err("control token must not authorize a worker request");
26891        assert!(matches!(
26892            error,
26893            Error::MissingRoleCredentials { role: "worker", .. }
26894        ));
26895        assert_eq!(server.request_count("/api/worker/register"), 0);
26896
26897        let worker_only = Client::builder(server.base_url())
26898            .worker_token(Some("worker-secret".to_string()))
26899            .build()
26900            .expect("worker client");
26901        let error = worker_only
26902            .health()
26903            .await
26904            .expect_err("worker token must not authorize a control request");
26905        assert!(matches!(
26906            error,
26907            Error::MissingRoleCredentials {
26908                role: "control",
26909                ..
26910            }
26911        ));
26912        assert_eq!(server.request_count("/api/health"), 0);
26913    }
26914
26915    #[tokio::test]
26916    async fn shared_token_supports_worker_and_control_planes() {
26917        let server = MockWorkerServer::start();
26918        let client = Client::builder(server.base_url())
26919            .token(Some("shared-secret".to_string()))
26920            .build()
26921            .expect("client");
26922
26923        client.health().await.expect("control request");
26924        client
26925            .register_worker("worker", "queue", vec![], vec![], 1, 1)
26926            .await
26927            .expect("worker request");
26928
26929        assert_eq!(
26930            server.authorization_for("/api/health").as_deref(),
26931            Some("Bearer shared-secret")
26932        );
26933        assert_eq!(
26934            server.control_protocol_for("/api/health").as_deref(),
26935            Some(CONTROL_PLANE_VERSION)
26936        );
26937        assert_eq!(
26938            server.authorization_for("/api/worker/register").as_deref(),
26939            Some("Bearer shared-secret")
26940        );
26941        assert_eq!(
26942            server
26943                .worker_protocol_for("/api/worker/register")
26944                .as_deref(),
26945            Some(WORKER_PROTOCOL_VERSION)
26946        );
26947    }
26948
26949    #[tokio::test]
26950    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
26951        let server = MockWorkerServer::start();
26952        let client = Client::builder(server.base_url())
26953            .timeout(Duration::from_secs(2))
26954            .build()
26955            .expect("client");
26956
26957        client
26958            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
26959            .await
26960            .expect("register");
26961        client
26962            .heartbeat_worker("capture-worker", 1, 1)
26963            .await
26964            .expect("heartbeat");
26965        client
26966            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
26967            .await
26968            .expect("workflow poll");
26969        client
26970            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
26971            .await
26972            .expect("activity poll");
26973
26974        for path in [
26975            "/api/worker/register",
26976            "/api/worker/heartbeat",
26977            "/api/worker/workflow-tasks/poll",
26978            "/api/worker/activity-tasks/poll",
26979        ] {
26980            assert_eq!(
26981                server.worker_protocol_for(path).as_deref(),
26982                Some(WORKER_PROTOCOL_VERSION),
26983                "unexpected protocol for {path}"
26984            );
26985        }
26986
26987        assert_eq!(
26988            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
26989            1
26990        );
26991        assert_eq!(
26992            server.request_body("/api/worker/workflow-tasks/poll")["history_page_size"],
26993            500
26994        );
26995        assert_eq!(
26996            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
26997            1
26998        );
26999        assert!(
27000            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
27001                .as_str()
27002                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
27003        );
27004        assert!(
27005            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
27006                .as_str()
27007                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
27008        );
27009    }
27010
27011    #[tokio::test]
27012    async fn query_task_endpoints_send_the_query_feature_protocol() {
27013        let server = MockWorkerServer::start();
27014        let client = Client::builder(server.base_url())
27015            .timeout(Duration::from_secs(2))
27016            .build()
27017            .expect("client");
27018
27019        client
27020            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
27021            .await
27022            .expect("query poll");
27023        client
27024            .complete_query_task(
27025                "query-capture",
27026                "capture-worker",
27027                1,
27028                json!(8),
27029                DEFAULT_CODEC,
27030            )
27031            .await
27032            .expect("query complete");
27033        client
27034            .fail_query_task(
27035                "query-capture",
27036                "capture-worker",
27037                1,
27038                "failed",
27039                "query_rejected",
27040                "QueryFailed",
27041            )
27042            .await
27043            .expect("query fail");
27044
27045        for path in [
27046            "/api/worker/query-tasks/poll",
27047            "/api/worker/query-tasks/query-capture/complete",
27048            "/api/worker/query-tasks/query-capture/fail",
27049        ] {
27050            assert_eq!(
27051                server.worker_protocol_for(path).as_deref(),
27052                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
27053                "unexpected protocol for {path}"
27054            );
27055        }
27056
27057        assert_eq!(
27058            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
27059            1
27060        );
27061        assert!(
27062            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
27063                .as_str()
27064                .is_some_and(|id| id.starts_with("rust-query-poll-"))
27065        );
27066    }
27067
27068    #[tokio::test]
27069    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
27070        let server = MockWorkerServer::transient_worker_failures();
27071        let client = Client::builder(server.base_url())
27072            .timeout(Duration::from_secs(2))
27073            .build()
27074            .expect("client");
27075
27076        client
27077            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
27078            .await
27079            .expect("workflow poll retry");
27080        client
27081            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
27082            .await
27083            .expect("activity poll retry");
27084        client
27085            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
27086            .await
27087            .expect("query poll retry");
27088
27089        for path in [
27090            "/api/worker/workflow-tasks/poll",
27091            "/api/worker/activity-tasks/poll",
27092            "/api/worker/query-tasks/poll",
27093        ] {
27094            let bodies = server.request_bodies(path);
27095            assert_eq!(bodies.len(), 2, "{path} must be retried once");
27096            assert_eq!(
27097                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
27098                "{path} must preserve the request binding across retry"
27099            );
27100        }
27101    }
27102
27103    #[tokio::test]
27104    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
27105        let server = MockWorkerServer::consecutive_poll_failures(2);
27106        let client = Client::builder(server.base_url())
27107            .timeout(Duration::from_secs(2))
27108            .build()
27109            .expect("client");
27110        let mut worker = Worker::new(client, "capture")
27111            .worker_id("capture-worker")
27112            .poll_timeout(Duration::from_millis(10))
27113            .retry_policy(WorkerRetryPolicy {
27114                max_retries: 2,
27115                initial_backoff: Duration::from_millis(1),
27116                max_backoff: Duration::from_millis(1),
27117            });
27118        worker.register_workflow(
27119            "capture.workflow",
27120            |_ctx, _input| async move { Ok(Value::Null) },
27121        );
27122        worker.register_activity(
27123            "capture.activity",
27124            |_ctx, _input| async move { Ok(Value::Null) },
27125        );
27126        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
27127            Ok(Value::Null)
27128        });
27129
27130        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
27131
27132        for path in [
27133            "/api/worker/workflow-tasks/poll",
27134            "/api/worker/activity-tasks/poll",
27135            "/api/worker/query-tasks/poll",
27136        ] {
27137            let bodies = server.request_bodies(path);
27138            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
27139            assert!(
27140                bodies
27141                    .iter()
27142                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
27143                "{path} must preserve one request binding across every retry"
27144            );
27145        }
27146    }
27147
27148    fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
27149        let reason = if unavailable {
27150            "storage_admission_unavailable"
27151        } else {
27152            "storage_pressure"
27153        };
27154        let mut body = json!({
27155            "reason": reason,
27156            "storage_state": if unavailable { "fenced" } else { "draining" },
27157            "retryable": true,
27158            "retry_after_seconds": 1,
27159        });
27160        if !mid_poll {
27161            body["request_admitted"] = json!(false);
27162        }
27163        if let Some(id) = poll_id {
27164            body["task"] = Value::Null;
27165            body["poll_status"] = json!(reason);
27166            body["poll_request_id"] = json!(id);
27167            body["retry_same_poll_request_id"] = json!(true);
27168            body["claim_admitted"] = json!(false);
27169        }
27170        body
27171    }
27172
27173    fn backend_refusal(path: &str, request: &str) -> Option<Value> {
27174        let operation = match path {
27175            "/api/worker/register" => "register_worker",
27176            "/api/worker/heartbeat" => "heartbeat_worker",
27177            "/api/worker/workflow-tasks/poll" => "poll_workflow_task",
27178            "/api/worker/activity-tasks/poll" => "poll_activity_task",
27179            "/api/worker/query-tasks/poll" => "poll_query_task",
27180            "/api/worker/update-validation-tasks/poll" => "poll_update_validation_task",
27181            _ => return None,
27182        };
27183        let request: Value = serde_json::from_str(request).ok()?;
27184        let mut response = json!({
27185            "reason": "backend_unavailable",
27186            "operation": operation,
27187            "outcome": "unknown",
27188            "retryable": true,
27189            "retry_after_seconds": 1,
27190            "worker_id": request["worker_id"],
27191            "task_queue": request.get("task_queue"),
27192        });
27193        if path.ends_with("/poll") {
27194            response["task"] = Value::Null;
27195            response["poll_status"] = json!("backend_unavailable");
27196            response["poll_request_id"] = request["poll_request_id"].clone();
27197            response["retry_same_poll_request_id"] = json!(true);
27198        }
27199        Some(response)
27200    }
27201
27202    fn backend_retry_override(
27203        path: &str,
27204        request: &str,
27205        number: usize,
27206    ) -> Option<(&'static str, String)> {
27207        if number > 8 {
27208            return None;
27209        }
27210        Some((
27211            "503 Service Unavailable",
27212            backend_refusal(path, request)?.to_string(),
27213        ))
27214    }
27215
27216    fn backend_unavailable_override(
27217        path: &str,
27218        request: &str,
27219        _number: usize,
27220    ) -> Option<(&'static str, String)> {
27221        Some((
27222            "503 Service Unavailable",
27223            backend_refusal(path, request)?.to_string(),
27224        ))
27225    }
27226
27227    fn storage_worker(server: &MockWorkerServer) -> Worker {
27228        Worker::new(Client::new(server.base_url()).expect("client"), "storage")
27229            .worker_id("storage-worker")
27230            .retry_policy(WorkerRetryPolicy {
27231                max_retries: 1,
27232                initial_backoff: Duration::from_millis(1),
27233                max_backoff: Duration::from_millis(1),
27234            })
27235    }
27236
27237    fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
27238        let requests = server.requests.lock().expect("requests");
27239        let bodies: Vec<_> = requests
27240            .iter()
27241            .filter(|request| request.path == path)
27242            .map(|request| &request.body)
27243            .collect();
27244        assert_eq!(bodies.len(), count, "{path}");
27245        assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
27246    }
27247
27248    #[test]
27249    fn backend_recovery_requires_the_explicit_worker_contract() {
27250        for (path, operation) in [
27251            ("/worker/register", "register_worker"),
27252            ("/worker/heartbeat", "heartbeat_worker"),
27253            ("/worker/workflow-tasks/poll", "poll_workflow_task"),
27254            ("/worker/activity-tasks/poll", "poll_activity_task"),
27255            ("/worker/query-tasks/poll", "poll_query_task"),
27256            (
27257                "/worker/update-validation-tasks/poll",
27258                "poll_update_validation_task",
27259            ),
27260        ] {
27261            let poll_id = path.ends_with("/poll").then_some("same-poll");
27262            let request = json!({"worker_id":"same-worker","task_queue":"same-queue","poll_request_id":poll_id});
27263            let body = backend_refusal(&format!("/api{path}"), &request.to_string())
27264                .expect("worker operation");
27265            assert_eq!(body["operation"], operation);
27266            let error = Error::Http {
27267                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27268                body: body.to_string(),
27269            };
27270            assert_eq!(
27271                worker_backend_unavailable_retry_after(&error, path, poll_id),
27272                Some(Duration::from_secs(1))
27273            );
27274            assert!(!worker_operation_is_retryable(&error));
27275            for (field, value) in [
27276                ("operation", json!("wrong_operation")),
27277                ("outcome", json!("completed")),
27278                ("retryable", json!(false)),
27279                ("retry_after_seconds", json!(0)),
27280                ("worker_id", json!("")),
27281            ] {
27282                let mut invalid = body.clone();
27283                invalid[field] = value;
27284                let error = Error::Http {
27285                    status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27286                    body: invalid.to_string(),
27287                };
27288                assert!(
27289                    worker_backend_unavailable_retry_after(&error, path, poll_id).is_none(),
27290                    "{path}: {field}"
27291                );
27292                assert!(!worker_operation_is_retryable(&error));
27293            }
27294            if let Some(poll_id) = poll_id {
27295                for (field, value) in [
27296                    ("poll_request_id", json!("wrong-poll")),
27297                    ("poll_status", json!("empty")),
27298                    ("task", json!({"task_id":"claimed"})),
27299                    ("retry_same_poll_request_id", json!(false)),
27300                ] {
27301                    let mut invalid = body.clone();
27302                    invalid[field] = value;
27303                    let error = Error::Http {
27304                        status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27305                        body: invalid.to_string(),
27306                    };
27307                    assert!(
27308                        worker_backend_unavailable_retry_after(&error, path, Some(poll_id))
27309                            .is_none(),
27310                        "{path}: {field}"
27311                    );
27312                }
27313            }
27314            let error = Error::Http {
27315                status: reqwest::StatusCode::UNAUTHORIZED,
27316                body: body.to_string(),
27317            };
27318            assert!(worker_backend_unavailable_retry_after(&error, path, poll_id).is_none());
27319            assert!(!worker_operation_is_retryable(&error));
27320        }
27321    }
27322
27323    #[tokio::test]
27324    async fn backend_recovery_outlives_generic_retry_budget_and_preserves_polls() {
27325        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27326            request_override: Some(backend_retry_override),
27327            ..MockWorkerBehavior::default()
27328        });
27329        let mut worker =
27330            storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
27331        worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
27332        worker.register_activity("backend.activity", |_, _| async { Ok(Value::Null) });
27333        worker.register_query("backend.workflow", "state", |_, _| async {
27334            Ok(Value::Null)
27335        });
27336        worker.register().await.expect("registration recovery");
27337        worker
27338            .client
27339            .heartbeat_worker("storage-worker", 1, 1)
27340            .await
27341            .expect("heartbeat recovery");
27342        assert_eq!(worker.run_once().await.expect("poll recovery"), 0);
27343        for path in [
27344            "/api/worker/register",
27345            "/api/worker/heartbeat",
27346            "/api/worker/workflow-tasks/poll",
27347            "/api/worker/activity-tasks/poll",
27348            "/api/worker/query-tasks/poll",
27349        ] {
27350            assert_identical_requests(&server, path, 9);
27351        }
27352    }
27353
27354    #[tokio::test]
27355    async fn backend_recovery_wait_stops_with_worker() {
27356        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27357            request_override: Some(backend_unavailable_override),
27358            ..MockWorkerBehavior::default()
27359        });
27360        let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy {
27361            max_retries: 1,
27362            initial_backoff: Duration::from_secs(2),
27363            max_backoff: Duration::from_secs(2),
27364        });
27365        worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
27366        let started = tokio::time::Instant::now();
27367        let _ = tokio::time::timeout(
27368            Duration::from_secs(1),
27369            worker.run_until(tokio::time::sleep(Duration::from_millis(100))),
27370        )
27371        .await
27372        .expect("shutdown interrupts backend wait");
27373        assert!(started.elapsed() < Duration::from_secs(1));
27374        assert!(server.request_count("/api/worker/register") >= 1);
27375    }
27376
27377    #[test]
27378    fn storage_admission_requires_an_explicit_identity_preserving_contract() {
27379        for unavailable in [false, true] {
27380            for mid_poll in [false, true] {
27381                let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
27382                let error = Error::Http {
27383                    status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27384                    body: body.to_string(),
27385                };
27386                assert_eq!(
27387                    worker_storage_admission_retry_after(&error, Some("same-poll")),
27388                    Some(Duration::from_secs(1))
27389                );
27390                assert!(
27391                    !worker_operation_is_retryable(&error),
27392                    "storage is not a bounded generic retry"
27393                );
27394                for (field, value) in [
27395                    ("poll_request_id", json!("wrong-poll")),
27396                    ("task", json!({"task_id":"claimed"})),
27397                    ("retryable", json!(false)),
27398                    ("retry_after_seconds", json!(0)),
27399                    ("retry_after_seconds", json!(true)),
27400                    ("retry_after_seconds", json!(1.0)),
27401                    ("storage_state", json!("normal")),
27402                    ("poll_status", json!("empty")),
27403                    ("claim_admitted", json!(true)),
27404                    ("retry_same_poll_request_id", json!(false)),
27405                    ("request_admitted", json!(true)),
27406                ] {
27407                    let mut invalid = body.clone();
27408                    invalid[field] = value;
27409                    let error = Error::Http {
27410                        status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27411                        body: invalid.to_string(),
27412                    };
27413                    assert!(
27414                        worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
27415                        "{field}"
27416                    );
27417                }
27418            }
27419        }
27420        let body = storage_refusal(None, false, false);
27421        let error = Error::Http {
27422            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27423            body: body.to_string(),
27424        };
27425        assert!(worker_storage_admission_retry_after(&error, None).is_some());
27426        assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
27427        let error = Error::Http {
27428            status: reqwest::StatusCode::FORBIDDEN,
27429            body: body.to_string(),
27430        };
27431        assert!(worker_storage_admission_retry_after(&error, None).is_none());
27432        let body = storage_refusal(None, false, true);
27433        let error = Error::Http {
27434            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27435            body: body.to_string(),
27436        };
27437        assert!(worker_storage_admission_retry_after(&error, None).is_none());
27438    }
27439
27440    #[tokio::test]
27441    async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
27442        for unavailable in [false, true] {
27443            for mid_poll in [false, true] {
27444                let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27445                    poll_failures_per_path: 1,
27446                    storage_refusals: 7,
27447                    storage_path: Some("/poll"),
27448                    storage_unavailable: unavailable,
27449                    storage_mid_poll: mid_poll,
27450                    ..MockWorkerBehavior::default()
27451                });
27452                let mut worker = storage_worker(&server);
27453                worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
27454                assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
27455                for path in [
27456                    "/api/worker/workflow-tasks/poll",
27457                    "/api/worker/activity-tasks/poll",
27458                    "/api/worker/query-tasks/poll",
27459                ] {
27460                    assert_identical_requests(&server, path, 9);
27461                }
27462            }
27463        }
27464    }
27465
27466    #[tokio::test]
27467    async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
27468        struct CountedBody(Arc<AtomicUsize>);
27469        impl Serialize for CountedBody {
27470            fn serialize<S: Serializer>(
27471                &self,
27472                serializer: S,
27473            ) -> std::result::Result<S::Ok, S::Error> {
27474                let count = self.0.fetch_add(1, Ordering::SeqCst);
27475                json!({"serialization":count,"lease_owner":"worker","attempt":7})
27476                    .serialize(serializer)
27477            }
27478        }
27479        for path in [
27480            "/api/worker/register",
27481            "/api/worker/heartbeat",
27482            "/api/worker/workflow-tasks/storage-task/complete",
27483            "/api/worker/workflow-tasks/storage-task/fail",
27484            "/api/worker/activity-tasks/storage-task/complete",
27485            "/api/worker/activity-tasks/storage-task/fail",
27486            "/api/worker/activity-tasks/storage-task/heartbeat",
27487            "/api/worker/query-tasks/storage-task/complete",
27488            "/api/worker/query-tasks/storage-task/fail",
27489        ] {
27490            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27491                storage_refusals: 7,
27492                storage_path: Some(path),
27493                ..MockWorkerBehavior::default()
27494            });
27495            let worker =
27496                storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
27497            let calls = Arc::new(AtomicUsize::new(0));
27498            let _: Value = worker
27499                .client
27500                .request_json(
27501                    reqwest::Method::POST,
27502                    &path[4..],
27503                    RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
27504                    Some(&CountedBody(Arc::clone(&calls))),
27505                )
27506                .await
27507                .expect("prepared request recovery");
27508            assert_eq!(calls.load(Ordering::SeqCst), 1);
27509            assert_identical_requests(&server, path, 8);
27510        }
27511        for worker_scope in [false, true] {
27512            let path = if worker_scope {
27513                "/api/health"
27514            } else {
27515                "/api/worker/register"
27516            };
27517            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27518                storage_refusals: usize::MAX,
27519                storage_path: Some(path),
27520                ..MockWorkerBehavior::default()
27521            });
27522            let worker = storage_worker(&server);
27523            let client = worker.client.clone();
27524            let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
27525            let error = if worker_scope {
27526                worker
27527                    .client
27528                    .health()
27529                    .await
27530                    .expect_err("control plane is not retried")
27531            } else {
27532                client
27533                    .request_json::<Value, Value>(
27534                        reqwest::Method::POST,
27535                        "/worker/register",
27536                        RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
27537                        Some(&json!({})),
27538                    )
27539                    .await
27540                    .expect_err("direct client is not retried")
27541            };
27542            assert!(worker_storage_admission_body(&error).is_some());
27543            assert_eq!(server.request_count(path), 1);
27544        }
27545    }
27546
27547    #[tokio::test]
27548    async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
27549        for fail in [false, true] {
27550            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27551                storage_activity: true,
27552                storage_refusals: 7,
27553                storage_path: Some("/storage-activity/"),
27554                ..MockWorkerBehavior::default()
27555            });
27556            let mut worker = storage_worker(&server);
27557            let calls = Arc::new(AtomicUsize::new(0));
27558            let observed = Arc::clone(&calls);
27559            worker.register_activity("storage.activity", move |ctx, _| {
27560                let calls = Arc::clone(&observed);
27561                async move {
27562                    calls.fetch_add(1, Ordering::SeqCst);
27563                    ctx.heartbeat(json!({"step":2})).await?;
27564                    if fail {
27565                        Err(Error::WorkerLoop("intentional handler failure".to_string()))
27566                    } else {
27567                        Ok(json!({"receipt":true}))
27568                    }
27569                }
27570            });
27571            assert_eq!(worker.run_once().await.expect("activity settled"), 1);
27572            assert_eq!(calls.load(Ordering::SeqCst), 1);
27573            assert_identical_requests(
27574                &server,
27575                "/api/worker/activity-tasks/storage-activity/heartbeat",
27576                8,
27577            );
27578            let suffix = if fail { "fail" } else { "complete" };
27579            assert_identical_requests(
27580                &server,
27581                &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
27582                8,
27583            );
27584            let other = if fail { "complete" } else { "fail" };
27585            assert_eq!(
27586                server.request_count(&format!(
27587                    "/api/worker/activity-tasks/storage-activity/{other}"
27588                )),
27589                0
27590            );
27591        }
27592    }
27593
27594    #[tokio::test]
27595    async fn storage_waits_are_interruptible_without_false_activity_failure() {
27596        for path in [
27597            "/api/worker/register",
27598            "/api/worker/heartbeat",
27599            "/api/worker/activity-tasks/poll",
27600            "/api/worker/activity-tasks/storage-activity/heartbeat",
27601            "/api/worker/activity-tasks/storage-activity/complete",
27602        ] {
27603            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27604                storage_activity: true,
27605                storage_refusals: usize::MAX,
27606                storage_path: Some(path),
27607                ..MockWorkerBehavior::default()
27608            });
27609            let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
27610            worker.register_activity("storage.activity", |ctx, _| async move {
27611                ctx.heartbeat(json!({"step":2})).await?;
27612                Ok(json!({"receipt":true}))
27613            });
27614            let shutdown = async {
27615                while server.request_count(path) == 0 {
27616                    tokio::time::sleep(Duration::from_millis(1)).await;
27617                }
27618            };
27619            let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
27620                .await
27621                .expect("shutdown interrupts admission");
27622            assert!(
27623                result.is_err(),
27624                "a refused operation must not appear acknowledged: {path}, {result:?}"
27625            );
27626            assert_eq!(server.request_count(path), 1);
27627            assert_eq!(
27628                server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
27629                0
27630            );
27631            assert_eq!(
27632                server.request_count("/api/worker/registrations/mock-worker"),
27633                usize::from(!path.ends_with("/register"))
27634            );
27635        }
27636    }
27637
27638    #[tokio::test]
27639    async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
27640        for fail in [false, true] {
27641            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27642                storage_query: true,
27643                storage_refusals: 7,
27644                storage_path: Some("/storage-query/"),
27645                ..MockWorkerBehavior::default()
27646            });
27647            let mut worker = storage_worker(&server);
27648            worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
27649            let calls = Arc::new(AtomicUsize::new(0));
27650            let observed = Arc::clone(&calls);
27651            worker.register_query("storage.workflow", "state", move |_, _| {
27652                let calls = Arc::clone(&observed);
27653                async move {
27654                    calls.fetch_add(1, Ordering::SeqCst);
27655                    if fail {
27656                        Err(Error::WorkerLoop("intentional query failure".to_string()))
27657                    } else {
27658                        Ok(json!({"state":"waiting"}))
27659                    }
27660                }
27661            });
27662            assert_eq!(worker.run_once().await.expect("query settled"), 1);
27663            assert_eq!(calls.load(Ordering::SeqCst), 1);
27664            let suffix = if fail { "fail" } else { "complete" };
27665            assert_identical_requests(
27666                &server,
27667                &format!("/api/worker/query-tasks/storage-query/{suffix}"),
27668                8,
27669            );
27670            let other = if fail { "complete" } else { "fail" };
27671            assert_eq!(
27672                server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
27673                0
27674            );
27675        }
27676    }
27677
27678    #[tokio::test]
27679    async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
27680        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27681            storage_refusals: 7,
27682            storage_path: Some("/poll"),
27683            unauthorized_polls: true,
27684            ..MockWorkerBehavior::default()
27685        });
27686        let error = storage_worker(&server)
27687            .run_once()
27688            .await
27689            .expect_err("auth remains terminal");
27690        assert!(matches!(
27691            error,
27692            Error::Http {
27693                status: reqwest::StatusCode::UNAUTHORIZED,
27694                ..
27695            }
27696        ));
27697        assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
27698
27699        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27700            storage_refusals: 7,
27701            storage_path: Some("/activity-cancel/complete"),
27702            ..MockWorkerBehavior::default()
27703        });
27704        let worker =
27705            storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
27706        let error = worker
27707            .client
27708            .complete_activity_task(
27709                "activity-cancel",
27710                "attempt-cancel",
27711                "worker",
27712                json!({}),
27713                DEFAULT_CODEC,
27714            )
27715            .await
27716            .expect_err("cancellation remains terminal");
27717        assert!(activity_task_rejection_is_final(&error));
27718        assert_identical_requests(
27719            &server,
27720            "/api/worker/activity-tasks/activity-cancel/complete",
27721            8,
27722        );
27723
27724        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27725            storage_refusals: usize::MAX,
27726            storage_path: Some("/poll"),
27727            storage_wrong_poll_id: true,
27728            ..MockWorkerBehavior::default()
27729        });
27730        assert!(storage_worker(&server).run_once().await.is_err());
27731        assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
27732    }
27733
27734    #[tokio::test]
27735    async fn storage_pollers_stop_when_the_run_future_is_aborted() {
27736        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27737            storage_refusals: usize::MAX,
27738            storage_path: Some("/poll"),
27739            ..MockWorkerBehavior::default()
27740        });
27741        let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
27742        worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
27743        let run = tokio::spawn(async move { worker.run().await });
27744        tokio::time::timeout(Duration::from_secs(2), async {
27745            while server.request_count("/api/worker/activity-tasks/poll") == 0 {
27746                tokio::time::sleep(Duration::from_millis(1)).await;
27747            }
27748        })
27749        .await
27750        .expect("poll started");
27751        run.abort();
27752        assert!(run.await.expect_err("cancelled run").is_cancelled());
27753        tokio::time::sleep(Duration::from_millis(250)).await;
27754        assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
27755    }
27756
27757    #[tokio::test]
27758    async fn query_protocol_rejection_from_older_server_is_typed() {
27759        let server = MockWorkerServer::reject_query_protocol();
27760        let client = Client::builder(server.base_url())
27761            .timeout(Duration::from_secs(2))
27762            .build()
27763            .expect("client");
27764
27765        let error = client
27766            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
27767            .await
27768            .expect_err("server below query protocol floor must reject");
27769        let Error::Protocol(failure) = error else {
27770            panic!("expected typed protocol failure");
27771        };
27772
27773        assert_eq!(failure.status, 400);
27774        assert_eq!(failure.reason, "unsupported_protocol_version");
27775        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
27776        assert_eq!(
27777            failure.requested_version.as_deref(),
27778            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
27779        );
27780        assert_eq!(
27781            server
27782                .worker_protocol_for("/api/worker/query-tasks/poll")
27783                .as_deref(),
27784            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
27785        );
27786    }
27787
27788    #[tokio::test]
27789    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
27790        let server = MockWorkerServer::reject_query_protocol();
27791        let client = Client::builder(server.base_url())
27792            .timeout(Duration::from_secs(2))
27793            .build()
27794            .expect("client");
27795        let mut worker = Worker::new(client, "rust-workers")
27796            .worker_id("baseline-worker")
27797            .poll_timeout(Duration::from_millis(10));
27798
27799        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
27800            Ok(Value::Null)
27801        });
27802
27803        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
27804        assert_eq!(
27805            server
27806                .worker_protocol_for("/api/worker/workflow-tasks/poll")
27807                .as_deref(),
27808            Some(WORKER_PROTOCOL_VERSION)
27809        );
27810        assert_eq!(
27811            server.worker_protocol_for("/api/worker/query-tasks/poll"),
27812            None,
27813            "a worker without query handlers must not use the query-task endpoint"
27814        );
27815    }
27816
27817    #[tokio::test]
27818    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
27819        let server = MockWorkerServer::reject_query_completion();
27820        let client = Client::builder(server.base_url())
27821            .timeout(Duration::from_secs(2))
27822            .build()
27823            .expect("client");
27824
27825        let error = client
27826            .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
27827            .await
27828            .expect_err("expired completion must be rejected");
27829        let Error::QueryFailed(failure) = error else {
27830            panic!("expected typed query failure");
27831        };
27832        assert_eq!(failure.status, 409);
27833        assert_eq!(failure.reason, "query_task_timed_out");
27834
27835        let mut worker = Worker::new(client, "rust-workers")
27836            .worker_id("late-worker")
27837            .poll_timeout(Duration::from_millis(10));
27838        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
27839        worker.register_query(
27840            "counter",
27841            "current",
27842            |_ctx, _args| async move { Ok(json!(8)) },
27843        );
27844
27845        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
27846        assert_eq!(
27847            worker
27848                .run_once()
27849                .await
27850                .expect("worker continues after late completion"),
27851            0
27852        );
27853        assert_eq!(
27854            server.request_count("/api/worker/query-tasks/query-late/complete"),
27855            2
27856        );
27857        assert_eq!(
27858            server.request_count("/api/worker/query-tasks/query-late/fail"),
27859            0,
27860            "a server completion rejection must not be reported as an encoding failure"
27861        );
27862    }
27863
27864    #[tokio::test]
27865    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
27866        let server = MockWorkerServer::start();
27867        let client = Client::builder(server.base_url())
27868            .timeout(Duration::from_secs(2))
27869            .build()
27870            .expect("client");
27871        let mut worker = Worker::new(client, "rust-workers")
27872            .worker_id("joined-worker")
27873            .poll_timeout(Duration::from_millis(10));
27874        worker.register_workflow(
27875            "joined.workflow",
27876            |_ctx, _input| async move { Ok(Value::Null) },
27877        );
27878        worker.register_activity(
27879            "joined.activity",
27880            |_ctx, _input| async move { Ok(Value::Null) },
27881        );
27882        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
27883            Ok(Value::Null)
27884        });
27885
27886        worker
27887            .run_until(tokio::time::sleep(Duration::from_millis(20)))
27888            .await
27889            .expect("normal shutdown");
27890
27891        let deregistration_path = "/api/worker/registrations/mock-worker";
27892        assert_eq!(server.request_count(deregistration_path), 1);
27893        for poll_path in [
27894            "/api/worker/workflow-tasks/poll",
27895            "/api/worker/activity-tasks/poll",
27896            "/api/worker/query-tasks/poll",
27897        ] {
27898            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
27899        }
27900        assert_eq!(
27901            server.captured_paths().last().map(String::as_str),
27902            Some(deregistration_path),
27903            "deregistration must start only after every poller has joined"
27904        );
27905    }
27906
27907    #[tokio::test]
27908    async fn registration_failure_does_not_deregister() {
27909        let server = MockWorkerServer::rejected_registration();
27910        let client = Client::builder(server.base_url())
27911            .timeout(Duration::from_secs(2))
27912            .build()
27913            .expect("client");
27914        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
27915
27916        let error = worker
27917            .run_until(async {})
27918            .await
27919            .expect_err("registration must fail");
27920        assert!(matches!(
27921            error,
27922            Error::Http {
27923                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27924                ..
27925            }
27926        ));
27927        assert!(server
27928            .captured_paths()
27929            .iter()
27930            .all(|path| !path.starts_with("/api/worker/registrations/")));
27931    }
27932
27933    #[tokio::test]
27934    async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
27935        let server = MockWorkerServer::rejected_registration_protocol();
27936        let client = Client::builder(server.base_url())
27937            .timeout(Duration::from_secs(2))
27938            .build()
27939            .expect("client");
27940        let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
27941
27942        let error = worker
27943            .run_until(async {})
27944            .await
27945            .expect_err("a protocol 1.16 server must reject this worker");
27946        let Error::Protocol(failure) = error else {
27947            panic!("expected typed protocol rejection");
27948        };
27949        assert_eq!(failure.reason, "unsupported_protocol_version");
27950        assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
27951        assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
27952        assert_eq!(
27953            server
27954                .worker_protocol_for("/api/worker/register")
27955                .as_deref(),
27956            Some(WORKER_PROTOCOL_VERSION)
27957        );
27958    }
27959
27960    #[tokio::test]
27961    async fn declined_registration_does_not_deregister() {
27962        let server = MockWorkerServer::declined_registration();
27963        let client = Client::builder(server.base_url())
27964            .timeout(Duration::from_secs(2))
27965            .build()
27966            .expect("client");
27967        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
27968
27969        let error = worker
27970            .run_until(async {})
27971            .await
27972            .expect_err("declined registration must fail");
27973        assert!(matches!(error, Error::WorkerLoop(_)));
27974        assert!(error.to_string().contains("was not accepted"));
27975        assert!(server
27976            .captured_paths()
27977            .iter()
27978            .all(|path| !path.starts_with("/api/worker/registrations/")));
27979    }
27980
27981    #[tokio::test]
27982    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
27983        let server = MockWorkerServer::rejected_deregistration();
27984        let client = Client::builder(server.base_url())
27985            .timeout(Duration::from_secs(2))
27986            .build()
27987            .expect("client");
27988        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
27989
27990        let error = worker
27991            .run_until(async {})
27992            .await
27993            .expect_err("deregistration must fail");
27994        assert!(matches!(
27995            error,
27996            Error::Http {
27997                status: reqwest::StatusCode::FORBIDDEN,
27998                ..
27999            }
28000        ));
28001        assert_eq!(
28002            server.request_count("/api/worker/registrations/mock-worker"),
28003            1
28004        );
28005    }
28006
28007    #[tokio::test]
28008    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
28009        let server = MockWorkerServer::rejected_deregistration_protocol();
28010        let client = Client::builder(server.base_url())
28011            .timeout(Duration::from_secs(2))
28012            .build()
28013            .expect("client");
28014        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
28015
28016        let error = worker
28017            .run_until(async {})
28018            .await
28019            .expect_err("protocol rejection must fail shutdown");
28020        let Error::Protocol(failure) = error else {
28021            panic!("expected typed protocol failure");
28022        };
28023        assert_eq!(failure.reason, "unsupported_protocol_version");
28024        assert_eq!(
28025            failure.requested_version.as_deref(),
28026            Some(WORKER_PROTOCOL_VERSION)
28027        );
28028        assert_eq!(
28029            server.request_count("/api/worker/registrations/mock-worker"),
28030            1
28031        );
28032    }
28033
28034    #[tokio::test]
28035    async fn primary_poller_error_retains_deregistration_failure_context() {
28036        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
28037        let client = Client::builder(server.base_url())
28038            .timeout(Duration::from_secs(2))
28039            .build()
28040            .expect("client");
28041        let mut worker = Worker::new(client, "rust-workers")
28042            .worker_id("combined-failure")
28043            .poll_timeout(Duration::from_millis(10));
28044        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
28045            Ok(Value::Null)
28046        });
28047
28048        let error = worker
28049            .run()
28050            .await
28051            .expect_err("worker and cleanup must fail");
28052        let summary = error.to_string();
28053        assert!(summary.contains("authentication_failed"));
28054        assert!(summary.contains("worker cannot deregister"));
28055        let Error::WorkerShutdown {
28056            primary,
28057            deregistration,
28058        } = error
28059        else {
28060            panic!("expected combined worker shutdown error");
28061        };
28062        assert!(matches!(
28063            *primary,
28064            Error::Http {
28065                status: reqwest::StatusCode::UNAUTHORIZED,
28066                ..
28067            }
28068        ));
28069        assert!(matches!(
28070            *deregistration,
28071            Error::Http {
28072                status: reqwest::StatusCode::FORBIDDEN,
28073                ..
28074            }
28075        ));
28076        assert_eq!(
28077            server.request_count("/api/worker/registrations/mock-worker"),
28078            1
28079        );
28080    }
28081
28082    #[tokio::test]
28083    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
28084        let server = MockWorkerServer::start();
28085        let client = Client::builder(server.base_url())
28086            .timeout(Duration::from_secs(2))
28087            .build()
28088            .expect("client");
28089        let mut worker = Worker::new(client, "rust-workers")
28090            .worker_id("activity-only-worker")
28091            .poll_timeout(Duration::from_millis(10));
28092
28093        worker.register_activity(
28094            "activity.only",
28095            |_ctx, _args| async move { Ok(Value::Null) },
28096        );
28097
28098        worker.run_until(async {}).await.expect("run worker");
28099    }
28100
28101    #[tokio::test]
28102    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
28103        let server = MockWorkerServer::start();
28104        let client = Client::builder(server.base_url())
28105            .timeout(Duration::from_secs(2))
28106            .build()
28107            .expect("client");
28108        let mut worker = Worker::new(client, "rust-workers")
28109            .worker_id("workflow-only-worker")
28110            .poll_timeout(Duration::from_millis(10));
28111
28112        worker.register_workflow(
28113            "workflow.only",
28114            |_ctx, _input| async move { Ok(Value::Null) },
28115        );
28116
28117        worker.run_until(async {}).await.expect("run worker");
28118    }
28119
28120    #[tokio::test]
28121    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
28122        let server = MockWorkerServer::start();
28123        let client = Client::builder(server.base_url())
28124            .timeout(Duration::from_secs(2))
28125            .build()
28126            .expect("client");
28127        let observations = Arc::new(Mutex::new(Vec::new()));
28128        let observed = Arc::clone(&observations);
28129        let mut worker = Worker::new(client, "rust-workers")
28130            .worker_id("observed-heartbeat-worker")
28131            .poll_timeout(Duration::from_millis(10))
28132            .on_worker_heartbeat(move |observation| {
28133                observed
28134                    .lock()
28135                    .expect("heartbeat observations")
28136                    .push(observation.clone());
28137            });
28138
28139        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
28140            Ok(Value::Null)
28141        });
28142        let acknowledged = Arc::clone(&observations);
28143        worker
28144            .run_until(async move {
28145                tokio::time::timeout(Duration::from_secs(2), async move {
28146                    loop {
28147                        if !acknowledged
28148                            .lock()
28149                            .expect("heartbeat observations")
28150                            .is_empty()
28151                        {
28152                            break;
28153                        }
28154                        tokio::time::sleep(Duration::from_millis(1)).await;
28155                    }
28156                })
28157                .await
28158                .expect("heartbeat acknowledgement within timeout");
28159            })
28160            .await
28161            .expect("run worker");
28162
28163        let observations = observations.lock().expect("heartbeat observations");
28164        let first = observations.first().expect("heartbeat acknowledgement");
28165        assert_eq!(first.worker_id, "observed-heartbeat-worker");
28166        assert_eq!(first.task_queue, "rust-workers");
28167        assert!(first.acknowledged_at_unix_millis > 0);
28168        assert_eq!(first.acknowledgement, json!({}));
28169    }
28170
28171    #[tokio::test]
28172    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
28173        let server = MockWorkerServer::delayed_heartbeat_worker();
28174        let client = Client::builder(server.base_url())
28175            .timeout(Duration::from_secs(3))
28176            .build()
28177            .expect("client");
28178        let observations = Arc::new(Mutex::new(Vec::new()));
28179        let observed = Arc::clone(&observations);
28180        let mut worker = Worker::new(client, "rust-snapshot-workers")
28181            .worker_id("rust-snapshot-worker")
28182            .poll_timeout(Duration::from_millis(10))
28183            .on_worker_heartbeat(move |observation| {
28184                observed
28185                    .lock()
28186                    .expect("heartbeat observations")
28187                    .push(observation.clone());
28188            });
28189
28190        worker.register_workflow("snapshot", |ctx, _input| async move {
28191            ctx.wait_signal("finish").await?;
28192            Ok(json!({"status": "finished"}))
28193        });
28194        worker.register_query("snapshot", "current", |ctx, _args| async move {
28195            Ok(json!(ctx
28196                .signals("increment")
28197                .iter()
28198                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
28199                .sum::<i64>()))
28200        });
28201        worker.register_activity("cancel-aware", |_ctx, _args| async move {
28202            Ok(json!({"late": "completion"}))
28203        });
28204
28205        worker
28206            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
28207            .await
28208            .expect("delayed heartbeat must allow a clean worker shutdown");
28209
28210        let observations = observations.lock().expect("heartbeat observations");
28211        assert!(
28212            observations.len() >= 3,
28213            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
28214        );
28215        assert!(
28216            observations.windows(2).all(|pair| {
28217                pair[1].acknowledged_at_unix_millis
28218                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
28219                    >= 850
28220            }),
28221            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
28222        );
28223        drop(observations);
28224
28225        let heartbeat_times = server.request_times("/api/worker/heartbeat");
28226        let delayed_request_at = *heartbeat_times
28227            .get(1)
28228            .expect("intentionally delayed heartbeat request");
28229        let delay_window_start = delayed_request_at + Duration::from_millis(100);
28230        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
28231        for path in [
28232            "/api/worker/workflow-tasks/poll",
28233            "/api/worker/activity-tasks/poll",
28234            "/api/worker/query-tasks/poll",
28235        ] {
28236            assert!(
28237                server
28238                    .request_times(path)
28239                    .iter()
28240                    .any(|received_at| *received_at >= delay_window_start
28241                        && *received_at <= delay_window_end),
28242                "{path} must keep polling while a heartbeat acknowledgement is delayed"
28243            );
28244        }
28245        assert!(
28246            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
28247            "workflow work must be settled"
28248        );
28249        assert!(
28250            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
28251            "activity work must be settled"
28252        );
28253        assert!(
28254            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
28255            "query work must be settled"
28256        );
28257    }
28258
28259    #[tokio::test]
28260    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
28261        let server = MockWorkerServer::heartbeat_retry_worker();
28262        let client = Client::builder(server.base_url())
28263            .timeout(Duration::from_secs(2))
28264            .build()
28265            .expect("client");
28266        let observations = Arc::new(Mutex::new(Vec::new()));
28267        let observed = Arc::clone(&observations);
28268        let worker = Worker::new(client, "rust-workers")
28269            .worker_id("heartbeat-retry-worker")
28270            .retry_policy(WorkerRetryPolicy {
28271                max_retries: 1,
28272                initial_backoff: Duration::from_millis(300),
28273                max_backoff: Duration::from_millis(300),
28274            })
28275            .on_worker_heartbeat(move |observation| {
28276                observed
28277                    .lock()
28278                    .expect("heartbeat observations")
28279                    .push(observation.clone());
28280            });
28281
28282        worker
28283            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
28284            .await
28285            .expect("retryable heartbeat failure must remain bounded and recover");
28286
28287        let observations = observations.lock().expect("heartbeat observations");
28288        assert!(observations.len() >= 3, "heartbeat retry must recover");
28289        assert!(
28290            observations.windows(2).all(|pair| {
28291                pair[1]
28292                    .acknowledged_at_unix_millis
28293                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
28294                    >= 850
28295            }),
28296            "a successful retry must start a fresh advertised cadence: {observations:?}"
28297        );
28298        assert_eq!(
28299            server.request_count("/api/worker/heartbeat"),
28300            observations.len() + 1,
28301            "one retryable failure must add exactly one bounded request"
28302        );
28303    }
28304
28305    #[tokio::test]
28306    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
28307        let server = MockWorkerServer::waiting_query_worker();
28308        let client = Client::builder(server.base_url())
28309            .timeout(Duration::from_secs(2))
28310            .build()
28311            .expect("client");
28312        let observations = Arc::new(Mutex::new(Vec::new()));
28313        let observed = Arc::clone(&observations);
28314        let mut worker = Worker::new(client, "rust-snapshot-workers")
28315            .worker_id("rust-snapshot-worker")
28316            .poll_timeout(Duration::from_millis(10))
28317            .on_worker_heartbeat(move |observation| {
28318                observed
28319                    .lock()
28320                    .expect("heartbeat observations")
28321                    .push(observation.clone());
28322            });
28323
28324        worker.register_workflow("snapshot", |ctx, _input| async move {
28325            ctx.wait_signal("finish").await?;
28326            Ok(json!({"status": "finished"}))
28327        });
28328        worker.register_query("snapshot", "current", |ctx, _args| async move {
28329            let current = ctx
28330                .signals("increment")
28331                .iter()
28332                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
28333                .sum::<i64>();
28334            Ok(json!(current))
28335        });
28336        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
28337        worker
28338            .declare_workflow_signals("snapshot", &["finish"])
28339            .expect("declare finish signal");
28340
28341        worker
28342            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
28343            .await
28344            .expect("pending workflow and query poller must remain live until shutdown");
28345
28346        assert!(
28347            observations.lock().expect("heartbeat observations").len() >= 4,
28348            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
28349        );
28350        assert!(
28351            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
28352            "workflow polling must continue after empty replay acknowledgements"
28353        );
28354        assert!(
28355            server.request_count("/api/worker/query-tasks/poll") >= 2,
28356            "query polling must continue after serving the current query"
28357        );
28358        assert_eq!(
28359            server.request_body("/api/worker/register")["capabilities"],
28360            json!([
28361                CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
28362                DURABLE_SELECTION_CAPABILITY,
28363                MEMO_UPSERTS_CAPABILITY,
28364                TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
28365                QUERY_TASKS_CAPABILITY,
28366                WORKFLOW_UPDATES_CAPABILITY,
28367                MESSAGE_STREAMS_CAPABILITY
28368            ])
28369        );
28370        assert_eq!(
28371            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
28372            json!({
28373                "queries": ["current"],
28374                "query_contracts": [{
28375                    "name": "current",
28376                    "parameters": [{
28377                        "name": "args", "position": 0, "required": false,
28378                        "variadic": true, "type": null, "allows_null": true,
28379                        "default_available": false, "default": null,
28380                    }],
28381                }],
28382                "signals": ["finish"],
28383                "signal_contracts": [{
28384                    "name": "finish",
28385                    "parameters": [{
28386                        "name": "args", "position": 0, "required": false,
28387                        "variadic": true, "type": null, "allows_null": true,
28388                        "default_available": false, "default": null,
28389                    }],
28390                }],
28391                "updates": ["replace"],
28392                "update_contracts": [{
28393                    "name": "replace",
28394                    "parameters": [{
28395                        "name": "args", "position": 0, "required": false,
28396                        "variadic": true, "type": null, "allows_null": true,
28397                        "default_available": false, "default": null,
28398                    }],
28399                }],
28400                "update_validators": [],
28401            })
28402        );
28403
28404        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
28405        assert_eq!(
28406            opened["commands"],
28407            json!([{
28408                "type": "open_signal_wait",
28409                "signal_name": "finish",
28410            }])
28411        );
28412
28413        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
28414            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
28415            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
28416            let failure = server.request_body(&fail_path);
28417            assert_eq!(
28418                failure["failure"]["type"],
28419                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
28420            );
28421            assert_eq!(server.request_count(&completion_path), 0);
28422        }
28423
28424        let query_completion =
28425            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
28426        assert_eq!(query_completion["result"], json!(8));
28427
28428        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
28429        assert_eq!(
28430            server.request_count(terminal_path),
28431            1,
28432            "the matching signal must settle the workflow exactly once"
28433        );
28434        let terminal = server.request_body(terminal_path);
28435        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
28436        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
28437        assert_eq!(
28438            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
28439                .expect("terminal workflow result"),
28440            json!({"status": "finished"})
28441        );
28442    }
28443
28444    #[tokio::test]
28445    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
28446        let server = MockWorkerServer::transient_worker_failures();
28447        let client = Client::builder(server.base_url())
28448            .timeout(Duration::from_secs(2))
28449            .build()
28450            .expect("client");
28451        let mut worker = Worker::new(client, "rust-workers")
28452            .worker_id("retry-worker")
28453            .poll_timeout(Duration::from_millis(10))
28454            .retry_policy(WorkerRetryPolicy {
28455                max_retries: 2,
28456                initial_backoff: Duration::from_millis(1),
28457                max_backoff: Duration::from_millis(1),
28458            });
28459        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28460        worker.register_activity(
28461            "counter.activity",
28462            |_ctx, _input| async move { Ok(Value::Null) },
28463        );
28464        worker.register_query(
28465            "counter",
28466            "current",
28467            |_ctx, _args| async move { Ok(json!(8)) },
28468        );
28469
28470        worker
28471            .run_until(tokio::time::sleep(Duration::from_millis(75)))
28472            .await
28473            .expect("transient failures must not stop the worker");
28474
28475        for path in [
28476            "/api/worker/heartbeat",
28477            "/api/worker/workflow-tasks/poll",
28478            "/api/worker/activity-tasks/poll",
28479            "/api/worker/query-tasks/poll",
28480        ] {
28481            assert!(
28482                server.request_count(path) >= 2,
28483                "{path} must continue after its transient failure"
28484            );
28485        }
28486    }
28487
28488    #[tokio::test]
28489    async fn worker_continues_after_long_poll_capacity_backpressure() {
28490        let server = MockWorkerServer::capacity_limited_activity_poll();
28491        let client = Client::builder(server.base_url())
28492            .timeout(Duration::from_secs(2))
28493            .build()
28494            .expect("client");
28495        let mut worker = Worker::new(client, "rust-workers")
28496            .worker_id("capacity-worker")
28497            .poll_timeout(Duration::from_millis(10))
28498            .retry_policy(WorkerRetryPolicy {
28499                max_retries: 0,
28500                initial_backoff: Duration::from_millis(1),
28501                max_backoff: Duration::from_millis(1),
28502            });
28503        worker.register_activity("capacity.activity", |_ctx, _input| async move {
28504            Ok(json!({"handled": true}))
28505        });
28506
28507        worker
28508            .run_until(tokio::time::sleep(Duration::from_millis(50)))
28509            .await
28510            .expect("capacity backpressure must not stop the worker");
28511
28512        assert!(
28513            server.request_count("/api/worker/activity-tasks/poll") >= 2,
28514            "the activity poller must continue after capacity backpressure"
28515        );
28516        assert_eq!(
28517            server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
28518            1,
28519            "the worker must complete work returned after capacity recovers"
28520        );
28521    }
28522
28523    #[test]
28524    fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
28525        let capacity = Error::Http {
28526            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
28527            body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
28528        };
28529        assert_eq!(
28530            worker_poll_capacity_retry_after(&capacity),
28531            Some(Duration::from_secs(3))
28532        );
28533
28534        let rejected_capacity = Error::Http {
28535            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
28536            body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
28537        };
28538        assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
28539        assert!(!worker_operation_is_retryable(&rejected_capacity));
28540
28541        let ordinary_rate_limit = Error::Http {
28542            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
28543            body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
28544                .to_string(),
28545        };
28546        assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
28547        assert!(worker_operation_is_retryable(&ordinary_rate_limit));
28548    }
28549
28550    #[tokio::test]
28551    async fn worker_recovers_all_poll_families_and_heartbeat_beyond_retry_budget() {
28552        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
28553            poll_failures_per_path: 8,
28554            heartbeat_failures: 8,
28555            ..MockWorkerBehavior::default()
28556        });
28557        let client = Client::builder(server.base_url())
28558            .timeout(Duration::from_secs(2))
28559            .build()
28560            .expect("client");
28561        let acknowledgements = Arc::new(AtomicUsize::new(0));
28562        let observed = Arc::clone(&acknowledgements);
28563        let mut worker = Worker::new(client, "rust-workers")
28564            .recover_transient_outages(true)
28565            .poll_timeout(Duration::from_millis(10))
28566            .retry_policy(WorkerRetryPolicy {
28567                max_retries: 2,
28568                initial_backoff: Duration::from_millis(2),
28569                max_backoff: Duration::from_millis(5),
28570            })
28571            .on_worker_heartbeat(move |_| {
28572                observed.fetch_add(1, Ordering::SeqCst);
28573            });
28574        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28575        worker.register_activity(
28576            "counter.activity",
28577            |_ctx, _input| async move { Ok(Value::Null) },
28578        );
28579        worker.register_query("counter", "current", |_ctx, _args| async move {
28580            Ok(Value::Null)
28581        });
28582
28583        worker
28584            .run_until(tokio::time::sleep(Duration::from_millis(500)))
28585            .await
28586            .expect("a recoverable outage must not stop the service worker");
28587
28588        assert_eq!(acknowledgements.load(Ordering::SeqCst), 1);
28589        assert_eq!(server.request_count("/api/worker/heartbeat"), 9);
28590        for path in [
28591            "/api/worker/workflow-tasks/poll",
28592            "/api/worker/activity-tasks/poll",
28593            "/api/worker/query-tasks/poll",
28594        ] {
28595            let bodies = server.request_bodies(path);
28596            assert!(bodies.len() >= 10, "{path} must resume after recovery");
28597            let first = &bodies[0]["poll_request_id"];
28598            assert!(first.as_str().is_some_and(|id| !id.is_empty()));
28599            for body in &bodies[..9] {
28600                assert_eq!(&body["poll_request_id"], first, "{path} retry identity");
28601            }
28602            assert_ne!(&bodies[9]["poll_request_id"], first);
28603            let times = server.request_times(path);
28604            for pair in times[..9].windows(2) {
28605                assert!(pair[1].duration_since(pair[0]) >= Duration::from_millis(2));
28606            }
28607        }
28608    }
28609
28610    #[tokio::test]
28611    async fn worker_outage_retry_waits_are_interrupted_by_shutdown() {
28612        for heartbeat_outage in [false, true] {
28613            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
28614                poll_failures_per_path: if heartbeat_outage { 0 } else { usize::MAX },
28615                heartbeat_failures: if heartbeat_outage { usize::MAX } else { 0 },
28616                ..MockWorkerBehavior::default()
28617            });
28618            let client = Client::builder(server.base_url()).build().expect("client");
28619            let mut worker = Worker::new(client, "rust-workers")
28620                .recover_transient_outages(true)
28621                .retry_policy(WorkerRetryPolicy {
28622                    max_retries: 1,
28623                    initial_backoff: Duration::from_secs(5),
28624                    max_backoff: Duration::from_secs(5),
28625                });
28626            worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28627            let started = Instant::now();
28628            worker
28629                .run_until(tokio::time::sleep(Duration::from_millis(200)))
28630                .await
28631                .expect("shutdown during an outage is a normal stop");
28632            assert!(started.elapsed() < Duration::from_secs(1));
28633            let path = if heartbeat_outage {
28634                "/api/worker/heartbeat"
28635            } else {
28636                "/api/worker/workflow-tasks/poll"
28637            };
28638            assert_eq!(server.request_count(path), 1);
28639            assert_eq!(
28640                server.request_count("/api/worker/registrations/mock-worker"),
28641                1
28642            );
28643        }
28644    }
28645
28646    #[tokio::test]
28647    async fn worker_outage_recovery_keeps_run_once_bounded() {
28648        let server = MockWorkerServer::unavailable_polls();
28649        let client = Client::builder(server.base_url()).build().expect("client");
28650        let mut worker = Worker::new(client, "rust-workers")
28651            .recover_transient_outages(true)
28652            .retry_policy(WorkerRetryPolicy {
28653                max_retries: 2,
28654                initial_backoff: Duration::from_millis(1),
28655                max_backoff: Duration::from_millis(1),
28656            });
28657        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28658        assert!(matches!(worker.run_once().await, Err(Error::Transport(_))));
28659        assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 3);
28660    }
28661
28662    #[tokio::test]
28663    async fn worker_outage_recovery_respects_disabled_retries() {
28664        let server = MockWorkerServer::unavailable_polls();
28665        let client = Client::builder(server.base_url()).build().expect("client");
28666        let mut worker = Worker::new(client, "rust-workers")
28667            .recover_transient_outages(true)
28668            .retry_policy(WorkerRetryPolicy {
28669                max_retries: 0,
28670                ..WorkerRetryPolicy::default()
28671            });
28672        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28673        assert!(matches!(worker.run().await, Err(Error::Transport(_))));
28674        assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
28675    }
28676
28677    #[tokio::test]
28678    async fn worker_fatal_poll_error_interrupts_outage_heartbeat_retries() {
28679        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
28680            heartbeat_failures: usize::MAX,
28681            unauthorized_polls: true,
28682            ..MockWorkerBehavior::default()
28683        });
28684        let client = Client::builder(server.base_url()).build().expect("client");
28685        let mut worker = Worker::new(client, "rust-workers")
28686            .recover_transient_outages(true)
28687            .retry_policy(WorkerRetryPolicy {
28688                max_retries: 1,
28689                initial_backoff: Duration::from_secs(5),
28690                max_backoff: Duration::from_secs(5),
28691            });
28692        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28693        let error = tokio::time::timeout(Duration::from_secs(1), worker.run())
28694            .await
28695            .expect("fatal poll errors must interrupt heartbeat recovery")
28696            .expect_err("authentication failure is terminal");
28697        assert!(
28698            matches!(error, Error::Http { status, .. } if status == reqwest::StatusCode::UNAUTHORIZED)
28699        );
28700        assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
28701    }
28702
28703    #[tokio::test]
28704    async fn worker_bounds_transport_retries() {
28705        let server = MockWorkerServer::unavailable_polls();
28706        let client = Client::builder(server.base_url())
28707            .timeout(Duration::from_secs(2))
28708            .build()
28709            .expect("client");
28710        let mut worker = Worker::new(client, "rust-workers")
28711            .worker_id("bounded-retry-worker")
28712            .poll_timeout(Duration::from_millis(10))
28713            .retry_policy(WorkerRetryPolicy {
28714                max_retries: 2,
28715                initial_backoff: Duration::from_millis(1),
28716                max_backoff: Duration::from_millis(1),
28717            });
28718        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28719
28720        let error = worker.run().await.expect_err("retry bound must terminate");
28721        assert!(matches!(error, Error::Transport(_)));
28722        assert_eq!(
28723            server.request_count("/api/worker/workflow-tasks/poll"),
28724            3,
28725            "one initial request plus exactly two retries"
28726        );
28727    }
28728
28729    #[tokio::test]
28730    async fn worker_retry_policy_can_disable_poll_retries() {
28731        let server = MockWorkerServer::unavailable_polls();
28732        let client = Client::builder(server.base_url())
28733            .timeout(Duration::from_secs(2))
28734            .build()
28735            .expect("client");
28736        let mut worker = Worker::new(client, "rust-workers")
28737            .worker_id("no-retry-worker")
28738            .poll_timeout(Duration::from_millis(10))
28739            .retry_policy(WorkerRetryPolicy {
28740                max_retries: 0,
28741                initial_backoff: Duration::from_millis(1),
28742                max_backoff: Duration::from_millis(1),
28743            });
28744        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28745
28746        let error = worker
28747            .run_once()
28748            .await
28749            .expect_err("disabled retries must return the first transport failure");
28750        assert!(matches!(error, Error::Transport(_)));
28751        assert_eq!(
28752            server.request_count("/api/worker/workflow-tasks/poll"),
28753            1,
28754            "max_retries=0 must send only the initial request"
28755        );
28756    }
28757
28758    #[tokio::test]
28759    async fn worker_does_not_retry_authentication_failures() {
28760        let server = MockWorkerServer::unauthorized_polls();
28761        let client = Client::builder(server.base_url())
28762            .timeout(Duration::from_secs(2))
28763            .build()
28764            .expect("client");
28765        let mut worker = Worker::new(client, "rust-workers")
28766            .worker_id("unauthorized-worker")
28767            .recover_transient_outages(true)
28768            .poll_timeout(Duration::from_millis(10));
28769        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28770
28771        let error = worker
28772            .run()
28773            .await
28774            .expect_err("authentication must terminate");
28775        let Error::Http { status, body } = error else {
28776            panic!("expected stable HTTP authentication error");
28777        };
28778        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
28779        assert!(body.contains("authentication_failed"));
28780        assert_eq!(
28781            server.request_count("/api/worker/workflow-tasks/poll"),
28782            1,
28783            "authentication failures must not be retried"
28784        );
28785    }
28786
28787    #[test]
28788    fn duplicate_registration_keeps_the_original_workflow() {
28789        let client = Client::new("http://127.0.0.1:9").expect("client");
28790        let mut worker = Worker::new(client, "registration-tests");
28791        worker.register_workflow("same", |_ctx, _input| async { Ok(json!("first")) });
28792        let original = Arc::clone(&worker.workflows["same"].execute);
28793        worker
28794            .set_workflow_definition_sources("same", &["first workflow source"])
28795            .expect("source identity");
28796        let fingerprint = worker.workflows["same"].definition_fingerprint.clone();
28797        worker.register_typed_workflow("same", |_ctx, _input: Value| async { Ok(json!("second")) });
28798        assert!(Arc::ptr_eq(&original, &worker.workflows["same"].execute));
28799        assert_eq!(worker.workflows["same"].definition_fingerprint, fingerprint);
28800    }
28801
28802    #[test]
28803    fn duplicate_registration_keeps_the_original_activity() {
28804        let client = Client::new("http://127.0.0.1:9").expect("client");
28805        let mut worker = Worker::new(client, "registration-tests");
28806        worker.register_activity("same", |_ctx, _input| async { Ok(json!("first")) });
28807        let original = Arc::clone(&worker.activities["same"]);
28808        worker.register_activity_avro_value("same", |_ctx, _input| async {
28809            Ok(AvroValue::String("second".into()))
28810        });
28811        assert!(Arc::ptr_eq(&original, &worker.activities["same"]));
28812    }
28813
28814    #[test]
28815    fn duplicate_registration_keeps_original_query_and_update_handlers() {
28816        let client = Client::new("http://127.0.0.1:9").expect("client");
28817        let mut worker = Worker::new(client, "registration-tests");
28818        worker.register_query("orders", "same", |_ctx, _input| async {
28819            Ok(json!("first"))
28820        });
28821        let RegisteredQuery::Snapshot(original) = worker.queries["orders"]["same"].clone() else {
28822            panic!("snapshot query");
28823        };
28824        worker.register_replayed_query("orders", "same", |_ctx, _state: Arc<()>, _input| async {
28825            Ok(json!("second"))
28826        });
28827        let RegisteredQuery::Snapshot(retained) = &worker.queries["orders"]["same"] else {
28828            panic!("duplicate replaced the query adapter");
28829        };
28830        assert!(Arc::ptr_eq(&original, retained));
28831        worker.register_update("orders", "same", |_ctx, _input| async {
28832            Ok(json!("first"))
28833        });
28834        let original = Arc::clone(&worker.updates["orders"]["same"]);
28835        worker.register_update_avro_value("orders", "same", |_ctx, _input| async {
28836            Ok(AvroValue::Null)
28837        });
28838        assert!(Arc::ptr_eq(&original, &worker.updates["orders"]["same"]));
28839    }
28840
28841    #[tokio::test]
28842    async fn duplicate_registration_fails_before_server_contact() {
28843        let server = MockWorkerServer::start();
28844        let client = Client::builder(server.base_url()).build().expect("client");
28845        let mut worker =
28846            Worker::new(client, "registration-tests").poll_timeout(Duration::from_millis(10));
28847        worker.register_workflow("same", |_ctx, _input| async { Ok(json!("first")) });
28848        worker.register_workflow("same", |_ctx, _input| async { Ok(json!("second")) });
28849        let error = worker
28850            .run_once()
28851            .await
28852            .expect_err("ambiguous worker must not poll");
28853        assert!(
28854            error.to_string().contains("duplicate_registration"),
28855            "{error}"
28856        );
28857        assert!(server.requests.lock().expect("requests").is_empty());
28858    }
28859
28860    #[derive(Clone, Debug)]
28861    struct CapturedRequest {
28862        headers: String,
28863        method: String,
28864        path: String,
28865        authorization: Option<String>,
28866        namespace: Option<String>,
28867        worker_protocol: Option<String>,
28868        control_protocol: Option<String>,
28869        body: String,
28870        received_at: Instant,
28871    }
28872
28873    struct MockWorkerServer {
28874        addr: SocketAddr,
28875        stop: Arc<AtomicBool>,
28876        requests: Arc<Mutex<Vec<CapturedRequest>>>,
28877        thread: Option<thread::JoinHandle<()>>,
28878    }
28879
28880    type RequestOverride = fn(&str, &str, usize) -> Option<(&'static str, String)>;
28881
28882    #[derive(Clone, Copy, Default)]
28883    struct MockWorkerBehavior {
28884        response_override: Option<fn(&str) -> Option<(&'static str, String)>>,
28885        request_override: Option<RequestOverride>,
28886        storage_refusals: usize,
28887        storage_path: Option<&'static str>,
28888        storage_unavailable: bool,
28889        storage_mid_poll: bool,
28890        storage_activity: bool,
28891        storage_query: bool,
28892        storage_wrong_poll_id: bool,
28893        reject_query_protocol: bool,
28894        reject_query_completion: bool,
28895        waiting_query_worker: bool,
28896        decline_registration: bool,
28897        complete_named_signal: bool,
28898        poll_failures_per_path: usize,
28899        long_poll_capacity_responses_per_path: usize,
28900        heartbeat_failures: usize,
28901        heartbeat_failure_request: Option<usize>,
28902        delayed_heartbeat_request: Option<usize>,
28903        heartbeat_response_delay: Duration,
28904        concurrent_requests: bool,
28905        unauthorized_polls: bool,
28906        reject_registration: bool,
28907        reject_registration_protocol: bool,
28908        reject_deregistration: bool,
28909        reject_deregistration_protocol: bool,
28910        cancelled_activity: bool,
28911        draining_polls: bool,
28912        invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
28913        workflow_completion_status: Option<&'static str>,
28914        workflow_completion_body: Option<&'static str>,
28915    }
28916
28917    impl MockWorkerServer {
28918        fn start() -> Self {
28919            Self::start_with_behavior(MockWorkerBehavior::default())
28920        }
28921
28922        fn reject_query_protocol() -> Self {
28923            Self::start_with_behavior(MockWorkerBehavior {
28924                reject_query_protocol: true,
28925                ..MockWorkerBehavior::default()
28926            })
28927        }
28928
28929        fn reject_query_completion() -> Self {
28930            Self::start_with_behavior(MockWorkerBehavior {
28931                reject_query_completion: true,
28932                ..MockWorkerBehavior::default()
28933            })
28934        }
28935
28936        fn waiting_query_worker() -> Self {
28937            Self::start_with_behavior(MockWorkerBehavior {
28938                waiting_query_worker: true,
28939                complete_named_signal: true,
28940                ..MockWorkerBehavior::default()
28941            })
28942        }
28943
28944        fn transient_worker_failures() -> Self {
28945            Self::start_with_behavior(MockWorkerBehavior {
28946                poll_failures_per_path: 1,
28947                heartbeat_failures: 1,
28948                ..MockWorkerBehavior::default()
28949            })
28950        }
28951
28952        fn consecutive_poll_failures(count: usize) -> Self {
28953            Self::start_with_behavior(MockWorkerBehavior {
28954                poll_failures_per_path: count,
28955                ..MockWorkerBehavior::default()
28956            })
28957        }
28958
28959        fn capacity_limited_activity_poll() -> Self {
28960            Self::start_with_behavior(MockWorkerBehavior {
28961                long_poll_capacity_responses_per_path: 1,
28962                ..MockWorkerBehavior::default()
28963            })
28964        }
28965
28966        fn delayed_heartbeat_worker() -> Self {
28967            Self::start_with_behavior(MockWorkerBehavior {
28968                waiting_query_worker: true,
28969                delayed_heartbeat_request: Some(2),
28970                heartbeat_response_delay: Duration::from_millis(1_500),
28971                concurrent_requests: true,
28972                cancelled_activity: true,
28973                ..MockWorkerBehavior::default()
28974            })
28975        }
28976
28977        fn heartbeat_retry_worker() -> Self {
28978            Self::start_with_behavior(MockWorkerBehavior {
28979                waiting_query_worker: true,
28980                heartbeat_failure_request: Some(2),
28981                concurrent_requests: true,
28982                ..MockWorkerBehavior::default()
28983            })
28984        }
28985
28986        fn unavailable_polls() -> Self {
28987            Self::start_with_behavior(MockWorkerBehavior {
28988                poll_failures_per_path: usize::MAX,
28989                ..MockWorkerBehavior::default()
28990            })
28991        }
28992
28993        fn unauthorized_polls() -> Self {
28994            Self::start_with_behavior(MockWorkerBehavior {
28995                unauthorized_polls: true,
28996                ..MockWorkerBehavior::default()
28997            })
28998        }
28999
29000        fn rejected_registration() -> Self {
29001            Self::start_with_behavior(MockWorkerBehavior {
29002                reject_registration: true,
29003                ..MockWorkerBehavior::default()
29004            })
29005        }
29006
29007        fn rejected_registration_protocol() -> Self {
29008            Self::start_with_behavior(MockWorkerBehavior {
29009                reject_registration_protocol: true,
29010                ..MockWorkerBehavior::default()
29011            })
29012        }
29013
29014        fn declined_registration() -> Self {
29015            Self::start_with_behavior(MockWorkerBehavior {
29016                decline_registration: true,
29017                ..MockWorkerBehavior::default()
29018            })
29019        }
29020
29021        fn rejected_deregistration() -> Self {
29022            Self::start_with_behavior(MockWorkerBehavior {
29023                reject_deregistration: true,
29024                ..MockWorkerBehavior::default()
29025            })
29026        }
29027
29028        fn rejected_deregistration_protocol() -> Self {
29029            Self::start_with_behavior(MockWorkerBehavior {
29030                reject_deregistration_protocol: true,
29031                ..MockWorkerBehavior::default()
29032            })
29033        }
29034
29035        fn unauthorized_polls_and_rejected_deregistration() -> Self {
29036            Self::start_with_behavior(MockWorkerBehavior {
29037                unauthorized_polls: true,
29038                reject_deregistration: true,
29039                ..MockWorkerBehavior::default()
29040            })
29041        }
29042
29043        fn cancelled_activity() -> Self {
29044            Self::start_with_behavior(MockWorkerBehavior {
29045                cancelled_activity: true,
29046                ..MockWorkerBehavior::default()
29047            })
29048        }
29049
29050        fn draining_polls() -> Self {
29051            Self::start_with_behavior(MockWorkerBehavior {
29052                draining_polls: true,
29053                ..MockWorkerBehavior::default()
29054            })
29055        }
29056
29057        fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
29058            Self::start_with_behavior(MockWorkerBehavior {
29059                invalid_task_payload_codec: Some(codec),
29060                ..MockWorkerBehavior::default()
29061            })
29062        }
29063
29064        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
29065            Self::start_with_behavior(MockWorkerBehavior {
29066                workflow_completion_status: Some(status),
29067                workflow_completion_body: Some(body),
29068                ..MockWorkerBehavior::default()
29069            })
29070        }
29071
29072        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
29073            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
29074            listener
29075                .set_nonblocking(true)
29076                .expect("configure mock listener");
29077            let addr = listener.local_addr().expect("mock server address");
29078            let stop = Arc::new(AtomicBool::new(false));
29079            let server_stop = Arc::clone(&stop);
29080            let requests = Arc::new(Mutex::new(Vec::new()));
29081            let server_requests = Arc::clone(&requests);
29082            let thread = thread::spawn(move || {
29083                let mut request_threads = Vec::new();
29084                while !server_stop.load(Ordering::SeqCst) {
29085                    match listener.accept() {
29086                        Ok((mut stream, _)) => {
29087                            if behavior.concurrent_requests {
29088                                let requests = Arc::clone(&server_requests);
29089                                request_threads.push(thread::spawn(move || {
29090                                    handle_mock_worker_request(&mut stream, &requests, behavior)
29091                                }));
29092                            } else {
29093                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
29094                            }
29095                        }
29096                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
29097                            let mut index = 0;
29098                            while index < request_threads.len() {
29099                                if request_threads[index].is_finished() {
29100                                    request_threads
29101                                        .swap_remove(index)
29102                                        .join()
29103                                        .expect("join mock request");
29104                                } else {
29105                                    index += 1;
29106                                }
29107                            }
29108                            thread::sleep(Duration::from_millis(5));
29109                        }
29110                        Err(_) => break,
29111                    }
29112                }
29113                for request_thread in request_threads {
29114                    request_thread.join().expect("join mock request");
29115                }
29116            });
29117
29118            Self {
29119                addr,
29120                stop,
29121                requests,
29122                thread: Some(thread),
29123            }
29124        }
29125
29126        fn base_url(&self) -> String {
29127            format!("http://{}", self.addr)
29128        }
29129
29130        fn worker_protocol_for(&self, path: &str) -> Option<String> {
29131            self.requests
29132                .lock()
29133                .expect("captured requests")
29134                .iter()
29135                .find(|request| request.path == path)
29136                .and_then(|request| request.worker_protocol.clone())
29137        }
29138
29139        fn control_protocol_for(&self, path: &str) -> Option<String> {
29140            self.requests
29141                .lock()
29142                .expect("captured requests")
29143                .iter()
29144                .find(|request| request.path == path)
29145                .and_then(|request| request.control_protocol.clone())
29146        }
29147
29148        fn method_for(&self, path: &str) -> Option<String> {
29149            self.requests
29150                .lock()
29151                .expect("captured requests")
29152                .iter()
29153                .find(|request| request.path == path)
29154                .map(|request| request.method.clone())
29155        }
29156
29157        fn authorization_for(&self, path: &str) -> Option<String> {
29158            self.requests
29159                .lock()
29160                .expect("captured requests")
29161                .iter()
29162                .find(|request| request.path == path)
29163                .and_then(|request| request.authorization.clone())
29164        }
29165
29166        fn namespace_for(&self, path: &str) -> Option<String> {
29167            self.requests
29168                .lock()
29169                .expect("captured requests")
29170                .iter()
29171                .find(|request| request.path == path)
29172                .and_then(|request| request.namespace.clone())
29173        }
29174
29175        fn request_count(&self, path: &str) -> usize {
29176            self.requests
29177                .lock()
29178                .expect("captured requests")
29179                .iter()
29180                .filter(|request| request.path == path)
29181                .count()
29182        }
29183
29184        fn captured_paths(&self) -> Vec<String> {
29185            self.requests
29186                .lock()
29187                .expect("captured requests")
29188                .iter()
29189                .map(|request| request.path.clone())
29190                .collect()
29191        }
29192
29193        fn request_times(&self, path: &str) -> Vec<Instant> {
29194            self.requests
29195                .lock()
29196                .expect("captured requests")
29197                .iter()
29198                .filter(|request| request.path == path)
29199                .map(|request| request.received_at)
29200                .collect()
29201        }
29202
29203        fn request_body(&self, path: &str) -> Value {
29204            let requests = self.requests.lock().expect("captured requests");
29205            let body = &requests
29206                .iter()
29207                .find(|request| request.path == path)
29208                .unwrap_or_else(|| panic!("missing request for {path}"))
29209                .body;
29210            serde_json::from_str(body).unwrap_or_else(|error| {
29211                panic!("invalid JSON request body for {path}: {error}: {body:?}")
29212            })
29213        }
29214
29215        fn request_bodies(&self, path: &str) -> Vec<Value> {
29216            self.requests
29217                .lock()
29218                .expect("captured requests")
29219                .iter()
29220                .filter(|request| request.path == path)
29221                .map(|request| {
29222                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
29223                        panic!(
29224                            "invalid JSON request body for {path}: {error}: {:?}",
29225                            request.body
29226                        )
29227                    })
29228                })
29229                .collect()
29230        }
29231    }
29232
29233    impl Drop for MockWorkerServer {
29234        fn drop(&mut self) {
29235            self.stop.store(true, Ordering::SeqCst);
29236            let _ = TcpStream::connect(self.addr);
29237
29238            if let Some(thread) = self.thread.take() {
29239                thread.join().expect("join mock server");
29240            }
29241        }
29242    }
29243
29244    fn handle_mock_worker_request(
29245        stream: &mut TcpStream,
29246        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
29247        behavior: MockWorkerBehavior,
29248    ) {
29249        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
29250        let mut buffer = [0_u8; 8192];
29251        let mut request = Vec::new();
29252
29253        loop {
29254            match stream.read(&mut buffer) {
29255                Ok(0) => break,
29256                Ok(read) => {
29257                    request.extend_from_slice(&buffer[..read]);
29258                    if mock_request_is_complete(&request) {
29259                        break;
29260                    }
29261                }
29262                Err(error)
29263                    if matches!(
29264                        error.kind(),
29265                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
29266                    ) =>
29267                {
29268                    break;
29269                }
29270                Err(_) => return,
29271            }
29272        }
29273
29274        let request = String::from_utf8_lossy(&request);
29275        let body = request
29276            .split_once("\r\n\r\n")
29277            .map(|(_, body)| body)
29278            .unwrap_or_default();
29279        let path = request
29280            .lines()
29281            .next()
29282            .and_then(|line| line.split_whitespace().nth(1))
29283            .unwrap_or_default();
29284        let method = request
29285            .lines()
29286            .next()
29287            .and_then(|line| line.split_whitespace().next())
29288            .unwrap_or_default();
29289        let authorization = request.lines().find_map(|line| {
29290            let (name, value) = line.split_once(':')?;
29291            name.eq_ignore_ascii_case("Authorization")
29292                .then(|| value.trim().to_string())
29293        });
29294        let namespace = request.lines().find_map(|line| {
29295            let (name, value) = line.split_once(':')?;
29296            name.eq_ignore_ascii_case("X-Namespace")
29297                .then(|| value.trim().to_string())
29298        });
29299        let worker_protocol = request.lines().find_map(|line| {
29300            let (name, value) = line.split_once(':')?;
29301            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
29302                .then(|| value.trim().to_string())
29303        });
29304        let control_protocol = request.lines().find_map(|line| {
29305            let (name, value) = line.split_once(':')?;
29306            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
29307                .then(|| value.trim().to_string())
29308        });
29309        let request_number = {
29310            let mut requests = requests.lock().expect("captured requests");
29311            requests.push(CapturedRequest {
29312                headers: request
29313                    .split_once("\r\n\r\n")
29314                    .map_or("", |(headers, _)| headers)
29315                    .to_owned(),
29316                method: method.to_string(),
29317                path: path.to_string(),
29318                authorization,
29319                namespace,
29320                worker_protocol: worker_protocol.clone(),
29321                control_protocol,
29322                body: body.to_string(),
29323                received_at: Instant::now(),
29324            });
29325            requests
29326                .iter()
29327                .filter(|request| request.path == path)
29328                .count()
29329        };
29330
29331        if let Some(response) = behavior
29332            .request_override
29333            .and_then(|handler| handler(path, body, request_number))
29334        {
29335            write_mock_response(stream, response.0, &response.1);
29336            return;
29337        }
29338        if let Some(response) = behavior.response_override.and_then(|handler| handler(path)) {
29339            write_mock_response(stream, response.0, &response.1);
29340            return;
29341        }
29342        if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
29343            return;
29344        }
29345        let pressure_path = behavior
29346            .storage_path
29347            .is_some_and(|part| path.contains(part));
29348        let prior_failures = if path.ends_with("/poll") {
29349            behavior.poll_failures_per_path
29350        } else {
29351            0
29352        };
29353        if pressure_path
29354            && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
29355        {
29356            let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
29357            let poll_id = path
29358                .ends_with("/poll")
29359                .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
29360            let mut refusal = storage_refusal(
29361                poll_id,
29362                behavior.storage_unavailable,
29363                behavior.storage_mid_poll,
29364            );
29365            if behavior.storage_wrong_poll_id {
29366                refusal["poll_request_id"] = json!("wrong-poll");
29367            }
29368            write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
29369            return;
29370        }
29371        if path.contains("/storage-task/")
29372            || path.contains("/storage-activity/")
29373            || path.contains("/storage-query/")
29374        {
29375            write_mock_response(stream, "200 OK", "{}");
29376            return;
29377        }
29378        if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
29379            write_mock_response(stream, "200 OK", &json!({"task":{
29380                "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
29381                "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
29382                "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
29383                "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
29384                "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
29385            }}).to_string());
29386            return;
29387        }
29388        if behavior.storage_activity
29389            && path == "/api/worker/activity-tasks/poll"
29390            && request_number == 1
29391        {
29392            write_mock_response(stream, "200 OK", &json!({"task":{
29393                "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
29394                "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
29395                "attempt_number":7, "lease_owner":"storage-worker"
29396            }}).to_string());
29397            return;
29398        }
29399
29400        if path == "/api/worker/register" {
29401            if behavior.reject_registration_protocol {
29402                write_mock_response(
29403                    stream,
29404                    "400 Bad Request",
29405                    r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
29406                );
29407                return;
29408            }
29409            if behavior.reject_registration {
29410                write_mock_response(
29411                    stream,
29412                    "503 Service Unavailable",
29413                    r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
29414                );
29415                return;
29416            }
29417        }
29418
29419        if path.starts_with("/api/worker/registrations/") {
29420            if behavior.reject_deregistration_protocol {
29421                write_mock_response(
29422                    stream,
29423                    "400 Bad Request",
29424                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
29425                );
29426            } else if behavior.reject_deregistration {
29427                write_mock_response(
29428                    stream,
29429                    "403 Forbidden",
29430                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
29431                );
29432            } else {
29433                write_mock_response(
29434                    stream,
29435                    "200 OK",
29436                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
29437                );
29438            }
29439            return;
29440        }
29441
29442        let is_poll = matches!(
29443            path,
29444            "/api/worker/workflow-tasks/poll"
29445                | "/api/worker/activity-tasks/poll"
29446                | "/api/worker/query-tasks/poll"
29447        );
29448        if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
29449            write_mock_response(
29450                stream,
29451                "429 Too Many Requests",
29452                r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
29453            );
29454            return;
29455        }
29456        if is_poll && request_number <= behavior.poll_failures_per_path {
29457            return;
29458        }
29459        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
29460            return;
29461        }
29462        if path == "/api/worker/heartbeat"
29463            && behavior.heartbeat_failure_request == Some(request_number)
29464        {
29465            return;
29466        }
29467        if path == "/api/worker/heartbeat"
29468            && behavior.delayed_heartbeat_request == Some(request_number)
29469        {
29470            thread::sleep(behavior.heartbeat_response_delay);
29471        }
29472        if behavior.unauthorized_polls && is_poll {
29473            write_mock_response(
29474                stream,
29475                "401 Unauthorized",
29476                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
29477            );
29478            return;
29479        }
29480        if behavior.draining_polls && is_poll {
29481            write_mock_response(
29482                stream,
29483                "409 Conflict",
29484                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
29485            );
29486            return;
29487        }
29488
29489        if let Some(codec_case) = behavior.invalid_task_payload_codec {
29490            if is_poll && request_number == 1 {
29491                let mut task = match path {
29492                    "/api/worker/workflow-tasks/poll" => json!({
29493                        "task_id": "codec-workflow",
29494                        "workflow_type": "codec.workflow",
29495                        "payload_codec": DEFAULT_CODEC,
29496                        "workflow_task_attempt": 1,
29497                        "lease_owner": "codec-worker"
29498                    }),
29499                    "/api/worker/activity-tasks/poll" => json!({
29500                        "task_id": "codec-activity",
29501                        "activity_attempt_id": "codec-activity-attempt",
29502                        "activity_type": "codec.activity",
29503                        "payload_codec": DEFAULT_CODEC,
29504                        "attempt_number": 1,
29505                        "lease_owner": "codec-worker"
29506                    }),
29507                    "/api/worker/query-tasks/poll" => json!({
29508                        "query_task_id": "codec-query",
29509                        "query_task_attempt": 1,
29510                        "workflow_type": "codec.workflow",
29511                        "query_name": "known",
29512                        "payload_codec": DEFAULT_CODEC,
29513                        "lease_owner": "codec-worker"
29514                    }),
29515                    _ => unreachable!("is_poll limits task codec probe paths"),
29516                };
29517                codec_case.apply(&mut task);
29518                write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
29519                return;
29520            }
29521
29522            if matches!(
29523                path,
29524                "/api/worker/workflow-tasks/codec-workflow/fail"
29525                    | "/api/worker/activity-tasks/codec-activity/fail"
29526                    | "/api/worker/query-tasks/codec-query/fail"
29527            ) {
29528                write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
29529                return;
29530            }
29531        }
29532
29533        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
29534            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
29535            let body = format!(
29536                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
29537            );
29538            write_mock_response(stream, "400 Bad Request", &body);
29539            return;
29540        }
29541
29542        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
29543        {
29544            write_mock_response(
29545                stream,
29546                "409 Conflict",
29547                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
29548            );
29549            return;
29550        }
29551
29552        if behavior.workflow_completion_status.is_some()
29553            && path == "/api/worker/workflow-tasks/poll"
29554            && request_number == 1
29555        {
29556            write_mock_response(
29557                stream,
29558                "200 OK",
29559                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
29560            );
29561            return;
29562        }
29563
29564        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
29565            if let (Some(status), Some(body)) = (
29566                behavior.workflow_completion_status,
29567                behavior.workflow_completion_body,
29568            ) {
29569                write_mock_response(stream, status, body);
29570                return;
29571            }
29572        }
29573
29574        if behavior.waiting_query_worker {
29575            if behavior.complete_named_signal
29576                && path == "/api/worker/workflow-tasks/poll"
29577                && request_number == 1
29578            {
29579                let body = json!({
29580                    "task": {
29581                        "task_id": "snapshot-open",
29582                        "workflow_id": "snapshot-1",
29583                        "run_id": "snapshot-run-1",
29584                        "workflow_type": "snapshot",
29585                        "payload_codec": DEFAULT_CODEC,
29586                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
29587                            .expect("Avro workflow arguments"),
29588                        "history_events": [],
29589                        "workflow_task_attempt": 1,
29590                        "lease_owner": "rust-snapshot-worker"
29591                    }
29592                })
29593                .to_string();
29594                write_mock_response(stream, "200 OK", &body);
29595                return;
29596            }
29597
29598            let signal_request = request_number - usize::from(behavior.complete_named_signal);
29599            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
29600            if path == "/api/worker/workflow-tasks/poll"
29601                && signal_request >= 1
29602                && signal_request <= signal_request_limit
29603            {
29604                let finish = behavior.complete_named_signal && signal_request == 3;
29605                let amounts = if signal_request == 1 {
29606                    vec![3]
29607                } else {
29608                    vec![3, 5]
29609                };
29610                let task_id = if signal_request == 1 {
29611                    "snapshot-wait-3"
29612                } else if finish {
29613                    "snapshot-finish"
29614                } else {
29615                    "snapshot-wait-5"
29616                };
29617                let mut history_events = std::iter::once(json!({
29618                    "event_type": "SignalWaitOpened",
29619                    "payload": {"sequence": 1, "signal_name": "finish"}
29620                }))
29621                .chain(amounts.iter().enumerate().map(|(index, amount)| {
29622                    json!({
29623                        "event_type": "SignalReceived",
29624                        "payload": {
29625                            "signal_id": format!("increment-{amount}"),
29626                            "signal_name": "increment",
29627                            "workflow_sequence": index + 2,
29628                            "payload_codec": DEFAULT_CODEC,
29629                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
29630                                .expect("Avro signal envelope")
29631                        }
29632                    })
29633                }))
29634                .collect::<Vec<_>>();
29635                let (resume_id, resume_name, resume_arguments) = if finish {
29636                    history_events.push(json!({
29637                        "event_type": "SignalReceived",
29638                        "payload": {
29639                            "signal_id": "finish",
29640                            "signal_name": "finish",
29641                            "workflow_sequence": 4,
29642                            "payload_codec": DEFAULT_CODEC,
29643                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
29644                                .expect("Avro finish signal envelope")
29645                        }
29646                    }));
29647                    (
29648                        "finish".to_string(),
29649                        "finish".to_string(),
29650                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
29651                            .expect("Avro finish resume signal"),
29652                    )
29653                } else {
29654                    let amount = amounts.last().expect("amount");
29655                    (
29656                        format!("increment-{amount}"),
29657                        "increment".to_string(),
29658                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
29659                            .expect("Avro increment resume signal"),
29660                    )
29661                };
29662                let body = json!({
29663                    "task": {
29664                        "task_id": task_id,
29665                        "workflow_id": "snapshot-1",
29666                        "run_id": "snapshot-run-1",
29667                        "workflow_type": "snapshot",
29668                        "payload_codec": DEFAULT_CODEC,
29669                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
29670                            .expect("Avro workflow arguments"),
29671                        "history_events": history_events,
29672                        "workflow_task_attempt": 1,
29673                        "workflow_signal_id": resume_id,
29674                        "signal_name": resume_name,
29675                        "signal_arguments": resume_arguments,
29676                        "lease_owner": "rust-snapshot-worker"
29677                    }
29678                })
29679                .to_string();
29680                write_mock_response(stream, "200 OK", &body);
29681                return;
29682            }
29683
29684            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
29685                let history_events = [3, 5]
29686                    .into_iter()
29687                    .enumerate()
29688                    .map(|(index, amount)| {
29689                        json!({
29690                            "event_type": "SignalReceived",
29691                            "payload": {
29692                                "signal_id": format!("increment-{amount}"),
29693                                "signal_name": "increment",
29694                                "workflow_sequence": index + 2,
29695                                "payload_codec": DEFAULT_CODEC,
29696                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
29697                                    .expect("Avro query signal envelope")
29698                            }
29699                        })
29700                    })
29701                    .collect::<Vec<_>>();
29702                let body = json!({
29703                    "task": {
29704                        "query_task_id": "snapshot-current",
29705                        "query_task_attempt": 1,
29706                        "lease_owner": "rust-snapshot-worker",
29707                        "workflow_id": "snapshot-1",
29708                        "run_id": "snapshot-run-1",
29709                        "workflow_type": "snapshot",
29710                        "query_name": "current",
29711                        "payload_codec": DEFAULT_CODEC,
29712                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
29713                            .expect("Avro workflow arguments"),
29714                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
29715                            .expect("Avro query arguments"),
29716                        "history_events": history_events,
29717                        "run_status": "waiting"
29718                    }
29719                })
29720                .to_string();
29721                write_mock_response(stream, "200 OK", &body);
29722                return;
29723            }
29724
29725            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
29726                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
29727            {
29728                write_mock_response(
29729                    stream,
29730                    "200 OK",
29731                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
29732                );
29733                return;
29734            }
29735
29736            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
29737                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
29738                return;
29739            }
29740
29741            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
29742                write_mock_response(
29743                    stream,
29744                    "200 OK",
29745                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
29746                );
29747                return;
29748            }
29749
29750            if path == "/api/worker/query-tasks/snapshot-current/complete" {
29751                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
29752                return;
29753            }
29754        }
29755
29756        if matches!(
29757            path,
29758            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
29759        ) {
29760            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
29761                .expect("typed mock result");
29762            let body = json!({
29763                "result": typed_fidelity_probe().into_json().expect("result projection"),
29764                "result_envelope": result,
29765            })
29766            .to_string();
29767            write_mock_response(stream, "200 OK", &body);
29768            return;
29769        }
29770
29771        if path == "/api/workflows/typed-1" {
29772            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
29773                .expect("typed mock result");
29774            let body = json!({
29775                "workflow_id": "typed-1",
29776                "run_id": "run-typed-1",
29777                "workflow_type": "typed.echo",
29778                "status": "completed",
29779                "output": typed_fidelity_probe().into_json().expect("output projection"),
29780                "output_envelope": result,
29781            })
29782            .to_string();
29783            write_mock_response(stream, "200 OK", &body);
29784            return;
29785        }
29786
29787        let (status, body) = match path {
29788            "/api/cluster/info" => ("200 OK", r#"{"limits":{"max_payload_bytes":2097152}}"#),
29789            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
29790            "/api/workflows" => (
29791                "201 Created",
29792                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
29793            ),
29794            "/api/worker/register" if behavior.decline_registration => (
29795                "200 OK",
29796                r#"{"worker_id":"declined-worker","registered":false}"#,
29797            ),
29798            "/api/worker/register" if behavior.waiting_query_worker => (
29799                "200 OK",
29800                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
29801            ),
29802            "/api/worker/register" => (
29803                "200 OK",
29804                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
29805            ),
29806            "/api/worker/heartbeat" => ("200 OK", "{}"),
29807            "/api/worker/activity-tasks/poll"
29808                if behavior.cancelled_activity && request_number == 1 =>
29809            {
29810                (
29811                    "200 OK",
29812                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
29813                )
29814            }
29815            "/api/worker/activity-tasks/poll"
29816                if behavior.long_poll_capacity_responses_per_path > 0
29817                    && request_number
29818                        == behavior
29819                            .long_poll_capacity_responses_per_path
29820                            .saturating_add(1) =>
29821            {
29822                (
29823                    "200 OK",
29824                    r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
29825                )
29826            }
29827            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
29828                ("200 OK", r#"{"task":null}"#)
29829            }
29830            "/api/worker/query-tasks/poll"
29831                if behavior.reject_query_completion && request_number == 1 =>
29832            {
29833                (
29834                    "200 OK",
29835                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
29836                )
29837            }
29838            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
29839            "/api/worker/query-tasks/query-capture/complete"
29840            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
29841            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
29842                "200 OK",
29843                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
29844            ),
29845            "/api/worker/activity-tasks/activity-cancel/complete" => (
29846                "409 Conflict",
29847                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
29848            ),
29849            "/api/worker/activity-tasks/activity-typed/complete"
29850            | "/api/worker/activity-tasks/activity-typed/fail"
29851            | "/api/worker/activity-tasks/capacity-activity/complete"
29852            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
29853            "/api/workflows/counter-1/query/current" => (
29854                "200 OK",
29855                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
29856            ),
29857            "/api/workflows/counter-1/query/missing" => (
29858                "404 Not Found",
29859                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
29860            ),
29861            "/api/workflows/wf-lifecycle/cancel" => (
29862                "200 OK",
29863                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
29864            ),
29865            "/api/workflows/wf-lifecycle/terminate" => (
29866                "200 OK",
29867                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
29868            ),
29869            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
29870                "200 OK",
29871                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
29872            ),
29873            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
29874                "200 OK",
29875                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
29876            ),
29877            "/api/workflows/wf-lifecycle/runs/run-failed/redrive" => (
29878                "202 Accepted",
29879                r#"{"workflow_id":"wf-lifecycle","continued_from_run_id":"run-failed","run_id":"run-successor","outcome":"redriven","command_status":"accepted","resume_step_sequence":2}"#,
29880            ),
29881            "/api/workflows/wf-lifecycle/runs/run-failed-existing/redrive" => (
29882                "200 OK",
29883                r#"{"workflow_id":"wf-lifecycle","continued_from_run_id":"run-failed-existing","run_id":"run-successor","outcome":"redriven","command_status":"accepted","resume_step_sequence":2}"#,
29884            ),
29885            "/api/workflows/wf-lifecycle/runs/run-completed/redrive" => (
29886                "409 Conflict",
29887                r#"{"workflow_id":"wf-lifecycle","run_id":"run-completed","reason":"run_not_failed","message":"Run is not failed."}"#,
29888            ),
29889            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
29890            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
29891                "409 Conflict",
29892                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
29893            ),
29894            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
29895                "200 OK",
29896                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
29897            ),
29898            "/api/workflows/wf-cancelled" => (
29899                "200 OK",
29900                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
29901            ),
29902            "/api/workflows/wf-terminated" => (
29903                "200 OK",
29904                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
29905            ),
29906            "/api/workflows/wf-timed-out" => (
29907                "200 OK",
29908                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
29909            ),
29910            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
29911                "200 OK",
29912                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
29913            ),
29914            "/api/workflows/wf-selected" => (
29915                "200 OK",
29916                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
29917            ),
29918            "/api/workflows/wf-selected/runs/run-selected" => (
29919                "200 OK",
29920                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
29921            ),
29922            _ => ("404 Not Found", r#"{"message":"not found"}"#),
29923        };
29924        write_mock_response(stream, status, body);
29925    }
29926
29927    fn mock_request_is_complete(request: &[u8]) -> bool {
29928        let Some(header_end) = request
29929            .windows(4)
29930            .position(|window| window == b"\r\n\r\n")
29931            .map(|position| position + 4)
29932        else {
29933            return false;
29934        };
29935        let headers = String::from_utf8_lossy(&request[..header_end]);
29936        let content_length = headers.lines().find_map(|line| {
29937            let (name, value) = line.split_once(':')?;
29938            name.eq_ignore_ascii_case("content-length")
29939                .then(|| value.trim().parse::<usize>().ok())
29940                .flatten()
29941        });
29942
29943        request.len() >= header_end + content_length.unwrap_or(0)
29944    }
29945
29946    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
29947        let response = format!(
29948            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
29949            body.len()
29950        );
29951
29952        let _ = stream.write_all(response.as_bytes());
29953        let _ = stream.flush();
29954    }
29955
29956    fn activity_completion_backend_recovery(
29957        path: &str,
29958        _body: &str,
29959        request_number: usize,
29960    ) -> Option<(&'static str, String)> {
29961        match path {
29962            "/api/worker/activity-tasks/poll" if request_number == 1 => Some((
29963                "200 OK",
29964                r#"{"task":{"task_id":"recover-activity","activity_attempt_id":"recover-attempt","activity_type":"recover.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"recover-worker"}}"#.to_string(),
29965            )),
29966            "/api/worker/activity-tasks/recover-activity/complete" if request_number == 1 => Some((
29967                "503 Service Unavailable",
29968                r#"{"reason":"backend_unavailable","operation":"complete_activity_task","outcome":"unknown","worker_id":"recover-worker","task_queue":null,"retryable":true,"retry_after_seconds":1,"task_id":"recover-activity","lease_owner":"recover-worker","activity_attempt_id":"recover-attempt"}"#.to_string(),
29969            )),
29970            "/api/worker/activity-tasks/recover-activity/complete" => Some((
29971                "200 OK",
29972                r#"{"outcome":"completed","recorded":true}"#.to_string(),
29973            )),
29974            _ => None,
29975        }
29976    }
29977
29978    fn workflow_completion_backend_recovery(
29979        path: &str,
29980        _body: &str,
29981        request_number: usize,
29982    ) -> Option<(&'static str, String)> {
29983        match path {
29984            "/api/worker/workflow-tasks/poll" if request_number == 1 => Some((
29985                "200 OK",
29986                include_str!("../tests/fixtures/worker-completion-recovery.json").to_string(),
29987            )),
29988            "/api/worker/workflow-tasks/recover-workflow/complete" if request_number == 1 => Some((
29989                "503 Service Unavailable",
29990                r#"{"reason":"backend_unavailable","operation":"complete_workflow_task","outcome":"unknown","worker_id":"recover-worker","task_queue":null,"retryable":true,"retry_after_seconds":1,"task_id":"recover-workflow","lease_owner":"recover-worker","workflow_task_attempt":3}"#.to_string(),
29991            )),
29992            "/api/worker/workflow-tasks/recover-workflow/complete" => Some((
29993                "200 OK",
29994                r#"{"outcome":"completed","recorded":true}"#.to_string(),
29995            )),
29996            _ => None,
29997        }
29998    }
29999
30000    fn activity_completion_stale_after_backend_recovery(
30001        path: &str,
30002        body: &str,
30003        request_number: usize,
30004    ) -> Option<(&'static str, String)> {
30005        if path == "/api/worker/activity-tasks/recover-activity/complete" && request_number > 1 {
30006            return Some((
30007                "409 Conflict",
30008                r#"{"task_id":"recover-activity","activity_attempt_id":"recover-attempt","lease_owner":"recover-worker","reason":"stale_attempt","outcome":"completed","recorded":false,"activity_status":"completed","attempt_status":"completed","task_status":"completed"}"#.to_string(),
30009            ));
30010        }
30011        activity_completion_backend_recovery(path, body, request_number)
30012    }
30013
30014    fn activity_completion_remains_unavailable(
30015        path: &str,
30016        body: &str,
30017        request_number: usize,
30018    ) -> Option<(&'static str, String)> {
30019        if path == "/api/worker/activity-tasks/recover-activity/complete" {
30020            return activity_completion_backend_recovery(path, body, 1);
30021        }
30022        activity_completion_backend_recovery(path, body, request_number)
30023    }
30024
30025    #[tokio::test]
30026    async fn managed_activity_retries_same_fenced_completion_after_backend_loss() {
30027        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
30028            request_override: Some(activity_completion_backend_recovery),
30029            ..MockWorkerBehavior::default()
30030        });
30031        let client = Client::builder(server.base_url())
30032            .timeout(Duration::from_secs(2))
30033            .build()
30034            .expect("client");
30035        let handler_calls = Arc::new(AtomicUsize::new(0));
30036        let mut worker = Worker::new(client, "recover-queue")
30037            .worker_id("recover-worker")
30038            .poll_timeout(Duration::from_millis(10));
30039        worker.register_activity("recover.activity", {
30040            let handler_calls = Arc::clone(&handler_calls);
30041            move |_ctx, _args| {
30042                let handler_calls = Arc::clone(&handler_calls);
30043                async move {
30044                    handler_calls.fetch_add(1, Ordering::SeqCst);
30045                    Ok(json!("done"))
30046                }
30047            }
30048        });
30049
30050        assert_eq!(worker.run_once().await.expect("activity settled"), 1);
30051        assert_eq!(handler_calls.load(Ordering::SeqCst), 1);
30052        let bodies = server.request_bodies("/api/worker/activity-tasks/recover-activity/complete");
30053        assert_eq!(bodies.len(), 2);
30054        assert_eq!(bodies[0], bodies[1]);
30055    }
30056
30057    #[tokio::test]
30058    async fn managed_workflow_retries_same_fenced_completion_after_backend_loss() {
30059        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
30060            request_override: Some(workflow_completion_backend_recovery),
30061            ..MockWorkerBehavior::default()
30062        });
30063        let client = Client::builder(server.base_url())
30064            .timeout(Duration::from_secs(2))
30065            .build()
30066            .expect("client");
30067        let handler_calls = Arc::new(AtomicUsize::new(0));
30068        let mut worker = Worker::new(client, "recover-queue")
30069            .worker_id("recover-worker")
30070            .poll_timeout(Duration::from_millis(10));
30071        worker.register_workflow("recover.workflow", {
30072            let handler_calls = Arc::clone(&handler_calls);
30073            move |_ctx, _args| {
30074                let handler_calls = Arc::clone(&handler_calls);
30075                async move {
30076                    handler_calls.fetch_add(1, Ordering::SeqCst);
30077                    Ok(json!("done"))
30078                }
30079            }
30080        });
30081
30082        assert_eq!(worker.run_once().await.expect("workflow settled"), 1);
30083        assert_eq!(handler_calls.load(Ordering::SeqCst), 1);
30084        let bodies = server.request_bodies("/api/worker/workflow-tasks/recover-workflow/complete");
30085        assert_eq!(bodies.len(), 2);
30086        assert_eq!(bodies[0], bodies[1]);
30087    }
30088
30089    #[tokio::test]
30090    async fn managed_activity_accepts_authoritative_stale_outcome_after_unknown_completion() {
30091        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
30092            request_override: Some(activity_completion_stale_after_backend_recovery),
30093            ..MockWorkerBehavior::default()
30094        });
30095        let client = Client::builder(server.base_url())
30096            .timeout(Duration::from_secs(2))
30097            .build()
30098            .expect("client");
30099        let handler_calls = Arc::new(AtomicUsize::new(0));
30100        let mut worker = Worker::new(client, "recover-queue")
30101            .worker_id("recover-worker")
30102            .poll_timeout(Duration::from_millis(10));
30103        worker.register_activity("recover.activity", {
30104            let handler_calls = Arc::clone(&handler_calls);
30105            move |_ctx, _args| {
30106                let handler_calls = Arc::clone(&handler_calls);
30107                async move {
30108                    handler_calls.fetch_add(1, Ordering::SeqCst);
30109                    Ok(json!("done"))
30110                }
30111            }
30112        });
30113
30114        assert_eq!(
30115            worker
30116                .run_once()
30117                .await
30118                .expect("authoritative stale outcome"),
30119            1
30120        );
30121        assert_eq!(handler_calls.load(Ordering::SeqCst), 1);
30122        let bodies = server.request_bodies("/api/worker/activity-tasks/recover-activity/complete");
30123        assert_eq!(bodies.len(), 2);
30124        assert_eq!(bodies[0], bodies[1]);
30125    }
30126
30127    #[test]
30128    fn fenced_completion_retry_rejects_mismatched_or_incomplete_identity() {
30129        let request = json!({"lease_owner":"recover-worker","activity_attempt_id":"recover-attempt","result":{"codec":"avro","blob":"value"}});
30130        let response = json!({
30131            "reason":"backend_unavailable", "operation":"complete_activity_task",
30132            "outcome":"unknown", "worker_id":"recover-worker", "task_queue":null,
30133            "retryable":true, "retry_after_seconds":1, "task_id":"recover-activity",
30134            "lease_owner":"recover-worker", "activity_attempt_id":"recover-attempt"
30135        });
30136        let path = "/worker/activity-tasks/recover-activity/complete";
30137        let error = |body: Value| Error::Http {
30138            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
30139            body: body.to_string(),
30140        };
30141        assert_eq!(
30142            worker_fenced_completion_retry_after(&error(response.clone()), path, Some(&request)),
30143            Some(Duration::from_secs(1))
30144        );
30145
30146        for (field, invalid) in [
30147            ("operation", json!("poll_activity_task")),
30148            ("outcome", json!("rejected")),
30149            ("worker_id", json!("another-worker")),
30150            ("task_queue", json!("recover-queue")),
30151            ("retryable", json!(false)),
30152            ("retry_after_seconds", json!(0)),
30153            ("task_id", json!("another-task")),
30154            ("lease_owner", json!("another-worker")),
30155            ("activity_attempt_id", json!("another-attempt")),
30156        ] {
30157            let mut invalid_response = response.clone();
30158            invalid_response[field] = invalid;
30159            assert_eq!(
30160                worker_fenced_completion_retry_after(
30161                    &error(invalid_response),
30162                    path,
30163                    Some(&request)
30164                ),
30165                None,
30166                "must not retry mismatched {field}"
30167            );
30168        }
30169        assert_eq!(
30170            worker_fenced_completion_retry_after(&error(response), path, None),
30171            None
30172        );
30173    }
30174
30175    #[tokio::test]
30176    async fn shutdown_interrupts_fenced_completion_backend_wait() {
30177        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
30178            request_override: Some(activity_completion_remains_unavailable),
30179            ..MockWorkerBehavior::default()
30180        });
30181        let client = Client::builder(server.base_url())
30182            .timeout(Duration::from_secs(2))
30183            .build()
30184            .expect("client");
30185        let stop = Arc::new(AtomicBool::new(false));
30186        let worker = Worker::new(client, "recover-queue")
30187            .worker_id("recover-worker")
30188            .with_storage_admission(Arc::clone(&stop));
30189        let completion = tokio::spawn(async move {
30190            worker
30191                .client
30192                .complete_activity_task(
30193                    "recover-activity",
30194                    "recover-attempt",
30195                    "recover-worker",
30196                    json!("done"),
30197                    DEFAULT_CODEC,
30198                )
30199                .await
30200        });
30201        let path = "/api/worker/activity-tasks/recover-activity/complete";
30202        tokio::time::timeout(Duration::from_secs(2), async {
30203            while server.request_count(path) == 0 {
30204                tokio::time::sleep(Duration::from_millis(10)).await;
30205            }
30206        })
30207        .await
30208        .expect("first completion request");
30209        stop.store(true, Ordering::SeqCst);
30210        let error = tokio::time::timeout(Duration::from_secs(1), completion)
30211            .await
30212            .expect("shutdown must interrupt retry wait")
30213            .expect("completion task")
30214            .expect_err("backend is still unavailable");
30215        assert!(matches!(error, Error::ActivityTaskRejected(rejection) if rejection.status == 503));
30216        assert_eq!(server.request_count(path), 1);
30217    }
30218}