durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{type_name, Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    io::{self, Read},
8    pin::Pin,
9    sync::{
10        atomic::{AtomicBool, Ordering},
11        Arc, Mutex, OnceLock,
12    },
13    task::{Context as TaskContext, Poll},
14    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use chrono::DateTime;
20use futures_util::{future::OptionFuture, task::noop_waker_ref};
21use serde::{
22    de::DeserializeOwned,
23    ser::{SerializeMap, SerializeSeq},
24    Deserialize, Deserializer, Serialize, Serializer,
25};
26pub use serde_json::{json, Value};
27use sha2::{Digest, Sha256};
28use thiserror::Error;
29pub use uuid::Uuid;
30
31pub const WORKER_PROTOCOL_VERSION: &str = "1.17";
32pub const CONTROL_PLANE_VERSION: &str = "2";
33pub const DEFAULT_CODEC: &str = "avro";
34pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
35/// Worker-registration capability for authored condition-wait occurrence identity.
36pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
37    "condition_wait_occurrence_identity";
38/// Worker-registration capability for portable memo upserts.
39pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
40/// Worker-registration capability for server-routed read-only queries.
41pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
42/// Worker-registration capability for canonical typed search attributes.
43pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
44/// Worker-registration capability for synchronous workflow updates.
45pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
46/// Worker-registration capability for durable named input streams.
47pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
48pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
49pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
50pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
51pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
52pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
53/// First additive worker protocol that defines query-task transport.
54pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
55/// First additive worker protocol that defines typed search-attribute upserts.
56pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
57/// First additive worker protocol that defines portable memo upserts.
58pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
59/// First additive worker protocol that preserves declared search-attribute types.
60pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
61/// First additive worker protocol that defines external durable condition waits.
62pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
63/// First additive worker protocol that preserves authored condition-wait occurrences.
64pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
65
66pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
67    let Some((major, minor)) = version.split_once('.') else {
68        return false;
69    };
70    major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
71}
72
73fn validate_user_signal_name(signal_name: &str) -> Result<()> {
74    if signal_name == MESSAGE_STREAM_SIGNAL {
75        return Err(Error::Codec(format!(
76            "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
77        )));
78    }
79    Ok(())
80}
81
82const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
83const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
84    "Workflow task waiting for scheduled history.";
85const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
86const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
87const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
88const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
89const MAX_MEMO_ENTRIES: usize = 100;
90const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
91const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
92
93const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
94    "lease_expired",
95    "query_task_not_found",
96    "query_task_not_leased",
97    "query_task_timed_out",
98];
99
100/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
101pub const AVRO_VALUE_SCHEMA_JSON: &str =
102    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
103pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
104pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
105const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
106
107static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
108static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
109    OnceLock::new();
110
111#[derive(Clone, Copy)]
112enum RequestProtocol {
113    ControlPlane,
114    Worker(&'static str),
115}
116
117pub type Result<T> = std::result::Result<T, Error>;
118
119#[derive(Debug, Error)]
120pub enum Error {
121    #[error("transport error: {0}")]
122    Transport(#[from] reqwest::Error),
123    #[error(
124        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
125    )]
126    InvalidBaseUrl,
127    #[error("json error: {0}")]
128    Json(#[from] serde_json::Error),
129    #[error("http {status}: {body}")]
130    Http {
131        status: reqwest::StatusCode,
132        body: String,
133    },
134    #[error("codec error: {0}")]
135    Codec(String),
136    #[error(transparent)]
137    QueryFailed(QueryFailure),
138    #[error(transparent)]
139    Protocol(ProtocolFailure),
140    #[error(transparent)]
141    NonDeterministicReplay(ReplayFailure),
142    #[error(transparent)]
143    ChildWorkflowFailed(ChildWorkflowFailure),
144    #[error(transparent)]
145    ActivityFailed(ActivityFailure),
146    #[error(transparent)]
147    ParallelFailed(ParallelFailure),
148    #[error(transparent)]
149    SagaCompensationFailed(SagaCompensationFailure),
150    #[error(transparent)]
151    InvalidParallelGroup(ParallelGroupError),
152    #[error(transparent)]
153    WorkflowCancellationRequested(WorkflowCancellationRequested),
154    #[error(transparent)]
155    WorkflowCommandRejected(WorkflowCommandRejection),
156    #[error(transparent)]
157    WorkflowFailed(WorkflowTerminalOutcome),
158    #[error(transparent)]
159    WorkflowCancelled(WorkflowTerminalOutcome),
160    #[error(transparent)]
161    WorkflowTerminated(WorkflowTerminalOutcome),
162    #[error(transparent)]
163    WorkflowTimedOut(WorkflowTerminalOutcome),
164    #[error(transparent)]
165    ActivityTaskRejected(ActivityTaskRejection),
166    #[error("workflow handler {0:?} is not registered")]
167    WorkflowNotRegistered(String),
168    #[error("activity handler {0:?} is not registered")]
169    ActivityNotRegistered(String),
170    #[error(
171        "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
172    )]
173    HandlerType {
174        handler_kind: HandlerKind,
175        handler_name: String,
176        value_kind: HandlerValueKind,
177        rust_type: &'static str,
178        message: String,
179    },
180    #[error("workflow future yielded without emitting a durable command")]
181    WorkflowYieldedWithoutCommand,
182    #[error(
183        "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
184    )]
185    MissingWorkflowCommandIdentity,
186    #[error("workflow state lock is poisoned")]
187    WorkflowStatePoisoned,
188    #[error("timer duration is too large for the worker protocol")]
189    TimerDurationOverflow,
190    #[error(transparent)]
191    InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
192    #[error(transparent)]
193    InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
194    #[error("operation timed out")]
195    Timeout,
196    #[error(
197        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
198    )]
199    MissingRoleCredentials {
200        role: &'static str,
201        opposite_role: &'static str,
202    },
203    #[error("worker loop error: {0}")]
204    WorkerLoop(String),
205    #[error(
206        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
207    )]
208    UnsupportedUpdateValidators { workflow_type: String },
209    #[error("{primary}; worker deregistration also failed: {deregistration}")]
210    WorkerShutdown {
211        primary: Box<Error>,
212        deregistration: Box<Error>,
213    },
214    #[error("invalid child workflow options: {0}")]
215    InvalidChildWorkflowOptions(String),
216    #[error("invalid workflow memo update: {0}")]
217    InvalidMemoUpdate(String),
218    #[error(
219        "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
220    )]
221    WorkflowMemoUpdatesUnavailable,
222    #[error(transparent)]
223    InvalidActivityOptions(ActivityOptionsError),
224    #[error(transparent)]
225    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
226    #[doc(hidden)]
227    #[error("workflow requested continue as new")]
228    ContinueAsNew(ContinueAsNewRequest),
229}
230
231/// Validation failure for a durable condition-wait definition.
232#[derive(Clone, Debug, Error, PartialEq, Eq)]
233pub enum ConditionWaitOptionsError {
234    #[error("condition_key must be non-empty")]
235    EmptyKey,
236    #[error("condition_definition_fingerprint must be non-empty")]
237    EmptyPredicateIdentity,
238    #[error("condition timeout is too large for the worker protocol")]
239    TimeoutOverflow,
240}
241
242/// Stable identity and optional durable timeout for a condition wait.
243///
244/// `predicate_identity` is recorded as the worker protocol's
245/// `condition_definition_fingerprint` and must change whenever predicate
246/// behavior changes. Prefer the [`wait_condition!`] macro when the predicate
247/// is written inline; it derives this identity from the predicate tokens.
248#[derive(Clone, Debug, PartialEq, Eq)]
249pub struct ConditionWaitOptions {
250    condition_key: String,
251    predicate_identity: String,
252    timeout: Option<Duration>,
253}
254
255impl ConditionWaitOptions {
256    pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
257        Self {
258            condition_key: condition_key.into(),
259            predicate_identity: predicate_identity.into(),
260            timeout: None,
261        }
262    }
263
264    pub fn timeout(mut self, timeout: Duration) -> Self {
265        self.timeout = Some(timeout);
266        self
267    }
268
269    fn validate(
270        &self,
271    ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
272        let condition_key = self.condition_key.trim();
273        if condition_key.is_empty() {
274            return Err(ConditionWaitOptionsError::EmptyKey);
275        }
276        let predicate_identity = self.predicate_identity.trim();
277        if predicate_identity.is_empty() {
278            return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
279        }
280        let timeout_seconds = self
281            .timeout
282            .map(|timeout| {
283                timeout
284                    .as_secs()
285                    .checked_add(u64::from(timeout.subsec_nanos() > 0))
286                    .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
287            })
288            .transpose()?;
289
290        Ok(ValidatedConditionWaitOptions {
291            condition_key: condition_key.to_string(),
292            predicate_identity: predicate_identity.to_string(),
293            timeout_seconds,
294        })
295    }
296}
297
298#[derive(Clone, Debug, PartialEq, Eq)]
299struct ValidatedConditionWaitOptions {
300    condition_key: String,
301    predicate_identity: String,
302    timeout_seconds: Option<u64>,
303}
304
305const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
306
307/// Unambiguous terminal result of a durable condition wait.
308#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
309#[serde(rename_all = "snake_case")]
310pub enum ConditionWaitResult {
311    Satisfied,
312    TimedOut,
313}
314
315impl ConditionWaitResult {
316    pub fn is_satisfied(self) -> bool {
317        self == Self::Satisfied
318    }
319
320    pub fn is_timed_out(self) -> bool {
321        self == Self::TimedOut
322    }
323}
324
325/// Build the stable condition definition identity used by [`wait_condition!`].
326#[doc(hidden)]
327pub fn __condition_definition_fingerprint(source: &str) -> String {
328    let mut digest = Sha256::new();
329    digest.update(b"durable-workflow-rust.wait-condition.v1\0");
330    digest.update(source.as_bytes());
331    format!("sha256:{:x}", digest.finalize())
332}
333
334/// Create a durable condition wait whose predicate definition is fingerprinted
335/// from its inline Rust tokens.
336///
337/// The returned [`ConditionWaitCall`] must be awaited. The timeout form is
338/// `wait_condition!(ctx, "approval", timeout: duration, || predicate)`.
339#[macro_export]
340macro_rules! wait_condition {
341    ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
342        $ctx.wait_condition(
343            $crate::ConditionWaitOptions::new(
344                $key,
345                $crate::__condition_definition_fingerprint(concat!(
346                    module_path!(),
347                    "\0",
348                    stringify!($predicate)
349                )),
350            )
351            .timeout($timeout),
352            $predicate,
353        )
354    }};
355    ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
356        $ctx.wait_condition(
357            $crate::ConditionWaitOptions::new(
358                $key,
359                $crate::__condition_definition_fingerprint(concat!(
360                    module_path!(),
361                    "\0",
362                    stringify!($predicate)
363                )),
364            ),
365            $predicate,
366        )
367    }};
368}
369
370const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
371const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
372const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 255;
373const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
374const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
375
376/// Validation failure for a typed workflow search-attribute update.
377#[derive(Clone, Debug, Error, PartialEq, Eq)]
378pub enum SearchAttributeUpdateError {
379    #[error("search-attribute update requires at least one attribute")]
380    Empty,
381    #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
382    InvalidKey(String),
383    #[error("search-attribute update exceeds the limit of 100 attributes")]
384    TooManyAttributes,
385    #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
386    ValueTooLong {
387        key: String,
388        kind: &'static str,
389        limit: usize,
390    },
391    #[error(
392        "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
393    )]
394    EmptyString(String),
395    #[error("search attribute {0:?} has a non-finite float value")]
396    NonFiniteFloat(String),
397    #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
398    InvalidDateTime(String),
399    #[error("search-attribute update exceeds the 65536-byte protocol limit")]
400    PayloadTooLarge,
401}
402
403/// One public typed search-attribute value.
404#[derive(Clone, Debug, PartialEq)]
405pub enum SearchAttributeValue {
406    String(String),
407    Keyword(String),
408    KeywordList(Vec<String>),
409    Int(i64),
410    Float(f64),
411    Bool(bool),
412    DateTime(String),
413    Delete,
414}
415
416impl SearchAttributeValue {
417    fn type_name(&self) -> Option<&'static str> {
418        match self {
419            Self::String(_) => Some("string"),
420            Self::Keyword(_) => Some("keyword"),
421            Self::KeywordList(_) => Some("keyword_list"),
422            Self::Int(_) => Some("int"),
423            Self::Float(_) => Some("float"),
424            Self::Bool(_) => Some("bool"),
425            Self::DateTime(_) => Some("datetime"),
426            Self::Delete => None,
427        }
428    }
429
430    fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
431        let normalize_string = |value: String, kind: &'static str, limit: usize| {
432            let value = value.trim().to_string();
433            if value.is_empty() {
434                return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
435            }
436            if value.len() > limit {
437                return Err(SearchAttributeUpdateError::ValueTooLong {
438                    key: key.to_string(),
439                    kind,
440                    limit,
441                });
442            }
443            Ok(value)
444        };
445
446        match self {
447            Self::String(value) => Ok(Self::String(normalize_string(
448                value,
449                "string",
450                MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
451            )?)),
452            Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
453                value,
454                "keyword",
455                MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
456            )?)),
457            Self::KeywordList(values) => {
458                let values = values
459                    .into_iter()
460                    .map(|value| {
461                        let value = value.trim().to_string();
462                        if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
463                            return Err(SearchAttributeUpdateError::ValueTooLong {
464                                key: key.to_string(),
465                                kind: "keyword-list entry",
466                                limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
467                            });
468                        }
469                        Ok(value)
470                    })
471                    .collect::<std::result::Result<Vec<_>, _>>()?;
472                Ok(Self::KeywordList(values))
473            }
474            Self::Float(value) if !value.is_finite() => {
475                Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
476            }
477            Self::DateTime(value) => {
478                let value =
479                    normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
480                if DateTime::parse_from_rfc3339(&value).is_err() {
481                    return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
482                }
483                Ok(Self::DateTime(value))
484            }
485            value => Ok(value),
486        }
487    }
488
489    fn into_json(self) -> Value {
490        match self {
491            Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
492                Value::String(value)
493            }
494            Self::KeywordList(values) => {
495                Value::Array(values.into_iter().map(Value::String).collect())
496            }
497            Self::Int(value) => json!(value),
498            Self::Float(value) => json!(value),
499            Self::Bool(value) => json!(value),
500            Self::Delete => Value::Null,
501        }
502    }
503}
504
505/// Validated typed workflow-side search-attribute mutation.
506#[derive(Clone, Debug, Default, PartialEq)]
507pub struct SearchAttributeUpdate {
508    attributes: BTreeMap<String, SearchAttributeValue>,
509}
510
511impl SearchAttributeUpdate {
512    pub fn new() -> Self {
513        Self::default()
514    }
515
516    pub fn set(
517        mut self,
518        key: impl Into<String>,
519        value: SearchAttributeValue,
520    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
521        let key = key.into();
522        validate_search_attribute_key(&key)?;
523        if !self.attributes.contains_key(&key)
524            && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
525        {
526            return Err(SearchAttributeUpdateError::TooManyAttributes);
527        }
528        self.attributes.insert(key.clone(), value.normalized(&key)?);
529        self.validate_size()?;
530        Ok(self)
531    }
532
533    pub fn string(
534        self,
535        key: impl Into<String>,
536        value: impl Into<String>,
537    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
538        self.set(key, SearchAttributeValue::String(value.into()))
539    }
540
541    pub fn keyword(
542        self,
543        key: impl Into<String>,
544        value: impl Into<String>,
545    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
546        self.set(key, SearchAttributeValue::Keyword(value.into()))
547    }
548
549    pub fn keyword_list<I, V>(
550        self,
551        key: impl Into<String>,
552        values: I,
553    ) -> std::result::Result<Self, SearchAttributeUpdateError>
554    where
555        I: IntoIterator<Item = V>,
556        V: Into<String>,
557    {
558        self.set(
559            key,
560            SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
561        )
562    }
563
564    pub fn int(
565        self,
566        key: impl Into<String>,
567        value: i64,
568    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
569        self.set(key, SearchAttributeValue::Int(value))
570    }
571
572    pub fn float(
573        self,
574        key: impl Into<String>,
575        value: f64,
576    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
577        self.set(key, SearchAttributeValue::Float(value))
578    }
579
580    pub fn bool(
581        self,
582        key: impl Into<String>,
583        value: bool,
584    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
585        self.set(key, SearchAttributeValue::Bool(value))
586    }
587
588    pub fn datetime(
589        self,
590        key: impl Into<String>,
591        value: impl Into<String>,
592    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
593        self.set(key, SearchAttributeValue::DateTime(value.into()))
594    }
595
596    pub fn delete(
597        self,
598        key: impl Into<String>,
599    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
600        self.set(key, SearchAttributeValue::Delete)
601    }
602
603    fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
604        let (attributes, _) = self.clone().into_wire_parts();
605        if serde_json::to_vec(&attributes)
606            .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
607            .unwrap_or(true)
608        {
609            return Err(SearchAttributeUpdateError::PayloadTooLarge);
610        }
611        Ok(())
612    }
613
614    fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
615        let mut attributes = serde_json::Map::new();
616        let mut attribute_types = BTreeMap::new();
617        for (key, value) in self.attributes {
618            if let Some(type_name) = value.type_name() {
619                attribute_types.insert(key.clone(), type_name.to_string());
620            }
621            attributes.insert(key, value.into_json());
622        }
623        (Value::Object(attributes), attribute_types)
624    }
625
626    fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
627        if self.attributes.is_empty() {
628            return Err(SearchAttributeUpdateError::Empty);
629        }
630        self.validate_size()
631    }
632}
633
634fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
635    let valid = !key.is_empty()
636        && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
637        && key
638            .bytes()
639            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
640    if valid {
641        Ok(())
642    } else {
643        Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
644    }
645}
646
647/// The registered handler family reported by [`Error::HandlerType`].
648#[derive(Clone, Copy, Debug, PartialEq, Eq)]
649pub enum HandlerKind {
650    Workflow,
651    Activity,
652}
653
654impl std::fmt::Display for HandlerKind {
655    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
656        formatter.write_str(match self {
657            Self::Workflow => "workflow",
658            Self::Activity => "activity",
659        })
660    }
661}
662
663/// Whether a typed handler failed to adapt its input or result.
664#[derive(Clone, Copy, Debug, PartialEq, Eq)]
665pub enum HandlerValueKind {
666    Input,
667    Result,
668}
669
670impl std::fmt::Display for HandlerValueKind {
671    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
672        formatter.write_str(match self {
673            Self::Input => "input",
674            Self::Result => "result",
675        })
676    }
677}
678
679/// The lifecycle command sent to a workflow execution.
680#[derive(Clone, Copy, Debug, PartialEq, Eq)]
681pub enum WorkflowCommandKind {
682    Cancel,
683    Terminate,
684}
685
686impl WorkflowCommandKind {
687    fn as_str(self) -> &'static str {
688        match self {
689            Self::Cancel => "cancel",
690            Self::Terminate => "terminate",
691        }
692    }
693}
694
695/// Optional structured fields for a cancellation or termination request.
696#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
697pub struct WorkflowCommandOptions {
698    #[serde(skip_serializing_if = "Option::is_none")]
699    pub reason: Option<String>,
700    #[serde(skip_serializing_if = "Option::is_none")]
701    pub request_id: Option<String>,
702}
703
704/// Server-enforced timeout policy for a workflow start.
705///
706/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
707/// only bounds how long the caller waits. A server deadline produces a terminal
708/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
709/// `run_timeout`.
710#[derive(Clone, Debug, PartialEq, Eq)]
711pub struct WorkflowStartOptions {
712    pub execution_timeout_seconds: u64,
713    pub run_timeout_seconds: u64,
714}
715
716impl Default for WorkflowStartOptions {
717    fn default() -> Self {
718        Self {
719            execution_timeout_seconds: 3600,
720            run_timeout_seconds: 600,
721        }
722    }
723}
724
725impl WorkflowStartOptions {
726    pub fn new() -> Self {
727        Self::default()
728    }
729
730    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
731        self.execution_timeout_seconds = seconds;
732        self
733    }
734
735    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
736        self.run_timeout_seconds = seconds;
737        self
738    }
739
740    fn validate(&self) -> Result<()> {
741        if self.execution_timeout_seconds == 0 {
742            return Err(Error::Codec(
743                "execution_timeout_seconds must be at least 1".to_string(),
744            ));
745        }
746        if self.run_timeout_seconds == 0 {
747            return Err(Error::Codec(
748                "run_timeout_seconds must be at least 1".to_string(),
749            ));
750        }
751        if self.run_timeout_seconds > self.execution_timeout_seconds {
752            return Err(Error::Codec(
753                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
754            ));
755        }
756
757        Ok(())
758    }
759}
760
761/// Optional routing overrides for a continue-as-new transition.
762///
763/// Omitted values retain the current workflow type and task queue. Server-owned
764/// instance metadata is not accepted here and is carried by the server.
765#[derive(Clone, Debug, Default, PartialEq, Eq)]
766pub struct ContinueAsNewOptions {
767    pub workflow_type: Option<String>,
768    pub task_queue: Option<String>,
769}
770
771impl ContinueAsNewOptions {
772    pub fn new() -> Self {
773        Self::default()
774    }
775
776    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
777        self.workflow_type = Some(workflow_type.into());
778        self
779    }
780
781    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
782        self.task_queue = Some(task_queue.into());
783        self
784    }
785
786    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
787        for (field, value) in [
788            ("workflow_type", self.workflow_type.as_deref()),
789            ("task_queue", self.task_queue.as_deref()),
790        ] {
791            if value.is_some_and(|value| value.trim().is_empty()) {
792                return Err(ContinueAsNewOptionsError {
793                    field,
794                    message: format!("{field} must not be empty"),
795                });
796            }
797        }
798        Ok(())
799    }
800}
801
802/// A stable validation error raised before a continue-as-new command is emitted.
803#[derive(Clone, Debug, Error, PartialEq, Eq)]
804#[error("invalid continue-as-new option {field}: {message}")]
805pub struct ContinueAsNewOptionsError {
806    pub field: &'static str,
807    pub message: String,
808}
809
810/// Public history-budget information attached to the current workflow task.
811#[derive(Clone, Debug, Default, PartialEq, Eq)]
812pub struct WorkflowHistoryBudget {
813    pub event_count: u64,
814    pub size_bytes: Option<u64>,
815    pub continue_as_new_recommended: bool,
816    pub pressure: Option<String>,
817}
818
819#[doc(hidden)]
820#[derive(Clone, Debug)]
821pub struct ContinueAsNewRequest {
822    arguments: AvroValue,
823    options: ContinueAsNewOptions,
824}
825
826impl WorkflowCommandOptions {
827    pub fn new() -> Self {
828        Self::default()
829    }
830
831    pub fn reason(mut self, reason: impl Into<String>) -> Self {
832        self.reason = Some(reason.into());
833        self
834    }
835
836    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
837        self.request_id = Some(request_id.into());
838        self
839    }
840}
841
842/// The accepted, machine-readable result of a lifecycle command.
843#[derive(Clone, Debug, PartialEq)]
844pub struct WorkflowCommandResult {
845    pub command: WorkflowCommandKind,
846    pub workflow_id: String,
847    pub run_id: Option<String>,
848    pub outcome: Option<String>,
849    pub reason: Option<String>,
850    pub command_status: Option<String>,
851    pub raw: Value,
852}
853
854/// A stable rejection returned by instance- or selected-run lifecycle commands.
855#[derive(Clone, Debug, Error)]
856#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
857pub struct WorkflowCommandRejection {
858    pub command: WorkflowCommandKind,
859    pub status: u16,
860    pub reason: String,
861    pub message: String,
862    pub workflow_id: String,
863    pub run_id: Option<String>,
864    pub target_scope: Option<String>,
865    pub body: Value,
866}
867
868/// Stable terminal categories returned by [`WorkflowHandle::result`].
869#[derive(Clone, Copy, Debug, PartialEq, Eq)]
870pub enum WorkflowTerminalKind {
871    Failed,
872    Cancelled,
873    Terminated,
874    TimedOut,
875}
876
877/// A typed terminal workflow outcome with durable identity and failure metadata.
878///
879/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
880/// of parsing its display representation. Fields remain `None` when an older
881/// server did not publish that metadata.
882#[derive(Clone, Debug, Error)]
883#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
884pub struct WorkflowTerminalOutcome {
885    pub kind: WorkflowTerminalKind,
886    pub workflow_id: String,
887    pub run_id: Option<String>,
888    pub reason: String,
889    pub failure_category: Option<String>,
890    pub failure_id: Option<String>,
891    pub exception_type: Option<String>,
892    pub exception_class: Option<String>,
893    pub non_retryable: Option<bool>,
894    pub message: Option<String>,
895    pub exception: Option<Value>,
896    pub raw: Value,
897}
898
899/// A worker-side activity settlement or heartbeat rejected by durable state.
900#[derive(Clone, Debug, Error)]
901#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
902pub struct ActivityTaskRejection {
903    pub operation: String,
904    pub status: u16,
905    pub reason: String,
906    pub task_id: String,
907    pub activity_attempt_id: String,
908    pub cancel_requested: bool,
909    pub can_continue: Option<bool>,
910    pub run_closed_reason: Option<String>,
911    pub body: Value,
912}
913
914/// Stable validation categories for [`ActivityOptions`].
915#[derive(Clone, Copy, Debug, PartialEq, Eq)]
916pub enum ActivityOptionsErrorKind {
917    EmptyTaskQueue,
918    EmptyRetryPolicy,
919    InvalidMaxAttempts,
920    BackoffWithoutRetryBudget,
921    TooManyBackoffIntervals,
922    InvalidBackoffCoefficient,
923    BackoffGenerationTooLarge,
924    BackoffOverflow,
925    EmptyNonRetryableErrorType,
926    TimeoutNotPositive,
927    TimeoutOverflow,
928    TimeoutOrder,
929}
930
931/// A machine-readable activity-options validation failure.
932#[derive(Clone, Debug, Error, PartialEq, Eq)]
933#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
934pub struct ActivityOptionsError {
935    pub kind: ActivityOptionsErrorKind,
936    pub field: Option<&'static str>,
937    pub message: String,
938}
939
940impl ActivityOptionsError {
941    fn new(
942        kind: ActivityOptionsErrorKind,
943        field: Option<&'static str>,
944        message: impl Into<String>,
945    ) -> Self {
946        Self {
947            kind,
948            field,
949            message: message.into(),
950        }
951    }
952}
953
954/// Stable terminal categories returned when an awaited activity does not succeed.
955#[derive(Clone, Copy, Debug, PartialEq, Eq)]
956pub enum ActivityFailureKind {
957    Failed,
958    Cancelled,
959    TimedOut,
960}
961
962/// A stable, machine-readable terminal activity failure.
963///
964/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
965/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
966#[derive(Clone, Debug, Error)]
967#[error("activity failed ({reason}): {message}")]
968pub struct ActivityFailure {
969    pub kind: ActivityFailureKind,
970    pub reason: String,
971    pub message: String,
972    pub activity_execution_id: Option<String>,
973    pub activity_attempt_id: Option<String>,
974    pub activity_type: Option<String>,
975    pub activity_class: Option<String>,
976    pub attempt_number: Option<u64>,
977    pub failure_id: Option<String>,
978    pub failure_category: Option<String>,
979    pub timeout_kind: Option<String>,
980    pub non_retryable: bool,
981    pub exception_type: Option<String>,
982    pub exception_class: Option<String>,
983    pub code: Option<Value>,
984    pub exception: Option<Value>,
985}
986
987/// Stable terminal categories returned when an awaited child does not succeed.
988#[derive(Clone, Copy, Debug, PartialEq, Eq)]
989pub enum ChildWorkflowFailureKind {
990    Failed,
991    Cancelled,
992    Terminated,
993}
994
995/// A stable, machine-readable child workflow failure delivered to its parent.
996///
997/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
998/// of parsing the display message. Child and parent identifiers retain the
999/// relationship recorded in durable history across worker restarts.
1000#[derive(Clone, Debug, Error)]
1001#[error("child workflow failed ({reason}): {message}")]
1002pub struct ChildWorkflowFailure {
1003    pub kind: ChildWorkflowFailureKind,
1004    pub reason: String,
1005    pub message: String,
1006    pub parent_workflow_id: Option<String>,
1007    pub parent_workflow_run_id: Option<String>,
1008    pub child_workflow_id: Option<String>,
1009    pub child_workflow_run_id: Option<String>,
1010    pub child_workflow_type: Option<String>,
1011    pub failure_id: Option<String>,
1012    pub failure_category: Option<String>,
1013    pub exception_type: Option<String>,
1014    pub exception_class: Option<String>,
1015    pub non_retryable: bool,
1016    pub code: Option<Value>,
1017    pub exception: Option<Value>,
1018}
1019
1020/// The identity of one durable workflow execution.
1021#[derive(Clone, Debug, PartialEq, Eq)]
1022pub struct WorkflowIdentity {
1023    pub workflow_id: Option<String>,
1024    pub run_id: Option<String>,
1025}
1026
1027/// A successful child result together with its durable parent-child identity.
1028#[derive(Clone, Debug, PartialEq)]
1029pub struct ChildWorkflowResult {
1030    pub parent: WorkflowIdentity,
1031    pub child: WorkflowIdentity,
1032    pub child_workflow_type: Option<String>,
1033    pub result: Value,
1034}
1035
1036/// Lossless successful child result for fixed Avro Value workflows.
1037#[derive(Clone, Debug, PartialEq)]
1038pub struct ChildWorkflowAvroResult {
1039    pub parent: WorkflowIdentity,
1040    pub child: WorkflowIdentity,
1041    pub child_workflow_type: Option<String>,
1042    pub result: AvroValue,
1043}
1044
1045/// Stable identity for one enclosing deterministic parallel group.
1046///
1047/// The same fields are attached to every ordinary activity, timer, or child
1048/// workflow command in the group. Nested leaves carry an outer-to-inner path;
1049/// no Rust-specific wire command is introduced.
1050#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1051pub struct ParallelGroupMetadata {
1052    pub parallel_group_id: String,
1053    pub parallel_group_kind: String,
1054    pub parallel_group_base_sequence: u64,
1055    pub parallel_group_size: usize,
1056    pub parallel_group_index: usize,
1057}
1058
1059/// One input-ordered result returned by [`WorkflowContext::parallel`].
1060#[derive(Clone, Debug, PartialEq)]
1061pub enum ParallelResult {
1062    Activity(Value),
1063    ChildWorkflow(ChildWorkflowResult),
1064    Timer,
1065    Group(Vec<ParallelResult>),
1066}
1067
1068/// Lossless fixed-Avro counterpart to [`ParallelResult`].
1069#[derive(Clone, Debug, PartialEq)]
1070pub enum ParallelAvroResult {
1071    Activity(AvroValue),
1072    ChildWorkflow(ChildWorkflowAvroResult),
1073    Timer,
1074    Group(Vec<ParallelAvroResult>),
1075}
1076
1077impl ParallelAvroResult {
1078    fn into_json_result(self) -> Result<ParallelResult> {
1079        match self {
1080            Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1081            Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1082                parent: result.parent,
1083                child: result.child,
1084                child_workflow_type: result.child_workflow_type,
1085                result: result.result.into_json()?,
1086            })),
1087            Self::Timer => Ok(ParallelResult::Timer),
1088            Self::Group(results) => Ok(ParallelResult::Group(
1089                results
1090                    .into_iter()
1091                    .map(Self::into_json_result)
1092                    .collect::<Result<Vec<_>>>()?,
1093            )),
1094        }
1095    }
1096}
1097
1098/// One successful leaf retained when another parallel member failed.
1099#[derive(Clone, Debug, PartialEq)]
1100pub struct ParallelCompletion {
1101    pub member_path: Vec<usize>,
1102    pub result: ParallelResult,
1103}
1104
1105/// A deterministic join failed after some siblings had already completed.
1106///
1107/// `cause` retains the typed activity, child-workflow, cancellation, or codec
1108/// error. `completed` is declaration ordered and contains only durable
1109/// successes observed in the same replay. Late sibling completions can add
1110/// entries on a later replay without changing `member_path` or the selected
1111/// positional failure.
1112#[derive(Debug, Error)]
1113#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1114pub struct ParallelFailure {
1115    pub group_id: String,
1116    pub member_path: Vec<usize>,
1117    pub group_path: Vec<ParallelGroupMetadata>,
1118    pub completed: Vec<ParallelCompletion>,
1119    #[source]
1120    pub cause: Box<Error>,
1121}
1122
1123/// Stable validation error returned before an invalid group emits commands.
1124#[derive(Clone, Debug, Error, PartialEq, Eq)]
1125#[error("invalid deterministic parallel group ({reason}): {message}")]
1126pub struct ParallelGroupError {
1127    pub reason: &'static str,
1128    pub member_path: Vec<usize>,
1129    pub message: String,
1130}
1131
1132/// Cooperative workflow cancellation observed at an author-controlled point.
1133#[derive(Clone, Debug, Error, PartialEq, Eq)]
1134#[error("workflow cancellation was requested")]
1135pub struct WorkflowCancellationRequested;
1136
1137/// A forward saga failure followed by a terminal compensation failure.
1138#[derive(Debug, Error)]
1139#[error(
1140    "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1141)]
1142pub struct SagaCompensationFailure {
1143    pub initiating_failure: Box<Error>,
1144    pub compensation_failure: Box<Error>,
1145    pub compensation_activity_type: String,
1146    pub compensation_registration_order: usize,
1147}
1148
1149/// Server behavior when a parent closes while its child is still open.
1150#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1151pub enum ParentClosePolicy {
1152    #[default]
1153    Abandon,
1154    RequestCancel,
1155    Terminate,
1156}
1157
1158impl ParentClosePolicy {
1159    fn as_str(self) -> &'static str {
1160        match self {
1161            Self::Abandon => "abandon",
1162            Self::RequestCancel => "request_cancel",
1163            Self::Terminate => "terminate",
1164        }
1165    }
1166}
1167
1168/// Durable retry policy for one child workflow invocation.
1169#[derive(Clone, Debug, Default, PartialEq, Eq)]
1170pub struct ChildWorkflowRetryPolicy {
1171    pub max_attempts: Option<u32>,
1172    pub backoff_seconds: Vec<u64>,
1173    pub non_retryable_error_types: Vec<String>,
1174}
1175
1176/// Options recorded with a child-workflow command.
1177///
1178/// The task queue is mandatory so routing is explicit and replay-stable.
1179#[derive(Clone, Debug, PartialEq, Eq)]
1180pub struct ChildWorkflowOptions {
1181    pub task_queue: String,
1182    pub parent_close_policy: ParentClosePolicy,
1183    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1184    pub execution_timeout_seconds: Option<u64>,
1185    pub run_timeout_seconds: Option<u64>,
1186}
1187
1188impl ChildWorkflowOptions {
1189    pub fn new(task_queue: impl Into<String>) -> Self {
1190        Self {
1191            task_queue: task_queue.into(),
1192            parent_close_policy: ParentClosePolicy::Abandon,
1193            retry_policy: None,
1194            execution_timeout_seconds: None,
1195            run_timeout_seconds: None,
1196        }
1197    }
1198
1199    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1200        self.parent_close_policy = policy;
1201        self
1202    }
1203
1204    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1205        self.retry_policy = Some(policy);
1206        self
1207    }
1208
1209    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1210        self.execution_timeout_seconds = Some(seconds);
1211        self
1212    }
1213
1214    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1215        self.run_timeout_seconds = Some(seconds);
1216        self
1217    }
1218}
1219
1220/// Backoff intervals for one durable activity retry policy.
1221#[derive(Clone, Debug, PartialEq, Eq)]
1222pub enum ActivityBackoff {
1223    /// Use these intervals between attempts. The server repeats the final
1224    /// interval if the retry budget contains more attempts than entries.
1225    Explicit(Vec<Duration>),
1226    /// Generate one interval for every retry using integer exponential growth.
1227    Exponential {
1228        initial_interval: Duration,
1229        coefficient: u32,
1230        maximum_interval: Option<Duration>,
1231    },
1232}
1233
1234/// Durable server-side retry policy for one activity execution.
1235#[derive(Clone, Debug, Default, PartialEq, Eq)]
1236pub struct ActivityRetryPolicy {
1237    pub max_attempts: Option<u32>,
1238    pub backoff: Option<ActivityBackoff>,
1239    pub non_retryable_error_types: Vec<String>,
1240}
1241
1242impl ActivityRetryPolicy {
1243    /// Start a policy with a finite attempt budget, including the first attempt.
1244    pub fn new(max_attempts: u32) -> Self {
1245        Self {
1246            max_attempts: Some(max_attempts),
1247            ..Self::default()
1248        }
1249    }
1250
1251    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1252        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1253        self
1254    }
1255
1256    pub fn exponential_backoff(
1257        mut self,
1258        initial_interval: Duration,
1259        coefficient: u32,
1260        maximum_interval: Option<Duration>,
1261    ) -> Self {
1262        self.backoff = Some(ActivityBackoff::Exponential {
1263            initial_interval,
1264            coefficient,
1265            maximum_interval,
1266        });
1267        self
1268    }
1269
1270    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1271        self.non_retryable_error_types.push(error_type.into());
1272        self
1273    }
1274
1275    pub fn non_retryable_error_types(
1276        mut self,
1277        error_types: impl IntoIterator<Item = impl Into<String>>,
1278    ) -> Self {
1279        self.non_retryable_error_types
1280            .extend(error_types.into_iter().map(Into::into));
1281        self
1282    }
1283}
1284
1285/// Options recorded atomically on one deterministic `schedule_activity` command.
1286///
1287/// Durations are rounded up to whole seconds when encoded, so the server never
1288/// receives a shorter timeout or backoff than the caller requested.
1289#[derive(Clone, Debug, Default, PartialEq, Eq)]
1290pub struct ActivityOptions {
1291    pub task_queue: Option<String>,
1292    pub retry_policy: Option<ActivityRetryPolicy>,
1293    pub start_to_close_timeout: Option<Duration>,
1294    pub schedule_to_start_timeout: Option<Duration>,
1295    pub schedule_to_close_timeout: Option<Duration>,
1296    pub heartbeat_timeout: Option<Duration>,
1297}
1298
1299impl ActivityOptions {
1300    pub fn new() -> Self {
1301        Self::default()
1302    }
1303
1304    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1305        self.task_queue = Some(task_queue.into());
1306        self
1307    }
1308
1309    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1310        self.retry_policy = Some(policy);
1311        self
1312    }
1313
1314    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1315        self.start_to_close_timeout = Some(timeout);
1316        self
1317    }
1318
1319    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1320        self.schedule_to_start_timeout = Some(timeout);
1321        self
1322    }
1323
1324    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1325        self.schedule_to_close_timeout = Some(timeout);
1326        self
1327    }
1328
1329    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1330        self.heartbeat_timeout = Some(timeout);
1331        self
1332    }
1333
1334    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1335        if self
1336            .task_queue
1337            .as_deref()
1338            .is_some_and(|queue| queue.trim().is_empty())
1339        {
1340            return Err(ActivityOptionsError::new(
1341                ActivityOptionsErrorKind::EmptyTaskQueue,
1342                Some("task_queue"),
1343                "task_queue must not be empty",
1344            ));
1345        }
1346
1347        for (field, value) in [
1348            ("start_to_close_timeout", self.start_to_close_timeout),
1349            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1350            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1351            ("heartbeat_timeout", self.heartbeat_timeout),
1352        ] {
1353            if value.is_some_and(|value| value.is_zero()) {
1354                return Err(ActivityOptionsError::new(
1355                    ActivityOptionsErrorKind::TimeoutNotPositive,
1356                    Some(field),
1357                    format!("{field} must be positive"),
1358                ));
1359            }
1360        }
1361
1362        validate_timeout_order(
1363            "heartbeat_timeout",
1364            self.heartbeat_timeout,
1365            "start_to_close_timeout",
1366            self.start_to_close_timeout,
1367        )?;
1368        validate_timeout_order(
1369            "start_to_close_timeout",
1370            self.start_to_close_timeout,
1371            "schedule_to_close_timeout",
1372            self.schedule_to_close_timeout,
1373        )?;
1374        validate_timeout_order(
1375            "schedule_to_start_timeout",
1376            self.schedule_to_start_timeout,
1377            "schedule_to_close_timeout",
1378            self.schedule_to_close_timeout,
1379        )?;
1380
1381        Ok(ValidatedActivityOptions {
1382            task_queue: self.task_queue.clone(),
1383            retry_policy: self
1384                .retry_policy
1385                .as_ref()
1386                .map(validate_activity_retry_policy)
1387                .transpose()?,
1388            start_to_close_timeout: timeout_seconds(
1389                "start_to_close_timeout",
1390                self.start_to_close_timeout,
1391            )?,
1392            schedule_to_start_timeout: timeout_seconds(
1393                "schedule_to_start_timeout",
1394                self.schedule_to_start_timeout,
1395            )?,
1396            schedule_to_close_timeout: timeout_seconds(
1397                "schedule_to_close_timeout",
1398                self.schedule_to_close_timeout,
1399            )?,
1400            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1401        })
1402    }
1403}
1404
1405/// A deferred durable leaf or nested group for [`WorkflowContext::parallel`].
1406///
1407/// Constructors capture arguments but perform no I/O. The join validates the
1408/// complete tree, attaches the existing parallel-group metadata to every
1409/// ordinary command, schedules all leaves, and then suspends.
1410pub enum ParallelOperation {
1411    Activity {
1412        activity_type: String,
1413        options: ActivityOptions,
1414        arguments: Result<AvroValue>,
1415    },
1416    ChildWorkflow {
1417        workflow_type: String,
1418        options: ChildWorkflowOptions,
1419        arguments: Result<AvroValue>,
1420    },
1421    Timer(Duration),
1422    Group(Vec<ParallelOperation>),
1423}
1424
1425impl ParallelOperation {
1426    pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1427        Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1428    }
1429
1430    pub fn activity_with_options<T: Serialize>(
1431        activity_type: impl Into<String>,
1432        options: ActivityOptions,
1433        args: T,
1434    ) -> Self {
1435        Self::Activity {
1436            activity_type: activity_type.into(),
1437            options,
1438            arguments: AvroValue::from_serialize(&args),
1439        }
1440    }
1441
1442    pub fn child_workflow<T: Serialize>(
1443        workflow_type: impl Into<String>,
1444        options: ChildWorkflowOptions,
1445        args: T,
1446    ) -> Self {
1447        Self::ChildWorkflow {
1448            workflow_type: workflow_type.into(),
1449            options,
1450            arguments: AvroValue::from_serialize(&args),
1451        }
1452    }
1453
1454    pub fn timer(duration: Duration) -> Self {
1455        Self::Timer(duration)
1456    }
1457
1458    pub fn group(operations: Vec<ParallelOperation>) -> Self {
1459        Self::Group(operations)
1460    }
1461}
1462
1463#[derive(Clone, Debug)]
1464struct ValidatedActivityOptions {
1465    task_queue: Option<String>,
1466    retry_policy: Option<Value>,
1467    start_to_close_timeout: Option<u64>,
1468    schedule_to_start_timeout: Option<u64>,
1469    schedule_to_close_timeout: Option<u64>,
1470    heartbeat_timeout: Option<u64>,
1471}
1472
1473fn validate_timeout_order(
1474    smaller_name: &'static str,
1475    smaller: Option<Duration>,
1476    larger_name: &'static str,
1477    larger: Option<Duration>,
1478) -> std::result::Result<(), ActivityOptionsError> {
1479    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1480        return Err(ActivityOptionsError::new(
1481            ActivityOptionsErrorKind::TimeoutOrder,
1482            Some(smaller_name),
1483            format!("{smaller_name} must be <= {larger_name}"),
1484        ));
1485    }
1486    Ok(())
1487}
1488
1489fn timeout_seconds(
1490    field: &'static str,
1491    value: Option<Duration>,
1492) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1493    value
1494        .map(|value| {
1495            activity_protocol_seconds(value).ok_or_else(|| {
1496                ActivityOptionsError::new(
1497                    ActivityOptionsErrorKind::TimeoutOverflow,
1498                    Some(field),
1499                    format!("{field} is too large for the worker protocol"),
1500                )
1501            })
1502        })
1503        .transpose()
1504}
1505
1506fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1507    value
1508        .as_secs()
1509        .checked_add(u64::from(value.subsec_nanos() > 0))
1510}
1511
1512fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1513    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1514}
1515
1516fn validate_activity_retry_policy(
1517    policy: &ActivityRetryPolicy,
1518) -> std::result::Result<Value, ActivityOptionsError> {
1519    if policy.max_attempts.is_none()
1520        && policy.backoff.is_none()
1521        && policy.non_retryable_error_types.is_empty()
1522    {
1523        return Err(ActivityOptionsError::new(
1524            ActivityOptionsErrorKind::EmptyRetryPolicy,
1525            Some("retry_policy"),
1526            "retry_policy must configure at least one field",
1527        ));
1528    }
1529    if policy.max_attempts == Some(0) {
1530        return Err(ActivityOptionsError::new(
1531            ActivityOptionsErrorKind::InvalidMaxAttempts,
1532            Some("retry_policy.max_attempts"),
1533            "max_attempts must be >= 1",
1534        ));
1535    }
1536    if policy
1537        .non_retryable_error_types
1538        .iter()
1539        .any(|error_type| error_type.trim().is_empty())
1540    {
1541        return Err(ActivityOptionsError::new(
1542            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1543            Some("retry_policy.non_retryable_error_types"),
1544            "non_retryable_error_types must not contain empty values",
1545        ));
1546    }
1547
1548    let backoff_seconds = match &policy.backoff {
1549        None => None,
1550        Some(backoff) => {
1551            let max_attempts = policy.max_attempts.ok_or_else(|| {
1552                ActivityOptionsError::new(
1553                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1554                    Some("retry_policy.backoff"),
1555                    "backoff requires max_attempts",
1556                )
1557            })?;
1558            let retry_count = max_attempts.saturating_sub(1) as usize;
1559            let intervals = match backoff {
1560                ActivityBackoff::Explicit(intervals) => {
1561                    if intervals.len() > retry_count {
1562                        return Err(ActivityOptionsError::new(
1563                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
1564                            Some("retry_policy.backoff"),
1565                            "backoff interval count must not exceed max_attempts - 1",
1566                        ));
1567                    }
1568                    intervals.clone()
1569                }
1570                ActivityBackoff::Exponential {
1571                    initial_interval,
1572                    coefficient,
1573                    maximum_interval,
1574                } => {
1575                    if *coefficient < 1 {
1576                        return Err(ActivityOptionsError::new(
1577                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1578                            Some("retry_policy.backoff.coefficient"),
1579                            "backoff coefficient must be >= 1",
1580                        ));
1581                    }
1582                    if retry_count > 10_000 {
1583                        return Err(ActivityOptionsError::new(
1584                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1585                            Some("retry_policy.max_attempts"),
1586                            "generated backoff supports at most 10000 retry intervals",
1587                        ));
1588                    }
1589                    let mut current = *initial_interval;
1590                    let mut intervals = Vec::with_capacity(retry_count);
1591                    for _ in 0..retry_count {
1592                        let interval = maximum_interval
1593                            .map(|maximum| current.min(maximum))
1594                            .unwrap_or(current);
1595                        intervals.push(interval);
1596                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
1597                            break;
1598                        }
1599                        current = current.checked_mul(*coefficient).ok_or_else(|| {
1600                            ActivityOptionsError::new(
1601                                ActivityOptionsErrorKind::BackoffOverflow,
1602                                Some("retry_policy.backoff"),
1603                                "generated backoff interval overflowed",
1604                            )
1605                        })?;
1606                    }
1607                    intervals
1608                }
1609            };
1610            Some(
1611                intervals
1612                    .into_iter()
1613                    .map(|interval| {
1614                        activity_protocol_seconds(interval).ok_or_else(|| {
1615                            ActivityOptionsError::new(
1616                                ActivityOptionsErrorKind::BackoffOverflow,
1617                                Some("retry_policy.backoff"),
1618                                "backoff interval is too large for the worker protocol",
1619                            )
1620                        })
1621                    })
1622                    .collect::<std::result::Result<Vec<_>, _>>()?,
1623            )
1624        }
1625    };
1626
1627    let mut encoded = serde_json::Map::new();
1628    if let Some(max_attempts) = policy.max_attempts {
1629        encoded.insert("max_attempts".to_string(), json!(max_attempts));
1630    }
1631    if let Some(backoff_seconds) = backoff_seconds {
1632        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1633    }
1634    if !policy.non_retryable_error_types.is_empty() {
1635        let mut canonical_error_types = Vec::new();
1636        for error_type in policy
1637            .non_retryable_error_types
1638            .iter()
1639            .map(|error_type| error_type.trim())
1640        {
1641            if !canonical_error_types.contains(&error_type) {
1642                canonical_error_types.push(error_type);
1643            }
1644        }
1645        encoded.insert(
1646            "non_retryable_error_types".to_string(),
1647            json!(canonical_error_types),
1648        );
1649    }
1650    Ok(Value::Object(encoded))
1651}
1652
1653/// A stable, machine-readable failure raised when workflow code no longer
1654/// reconstructs the durable command stream recorded in history.
1655#[derive(Clone, Debug, Error)]
1656#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1657pub struct ReplayFailure {
1658    pub reason: String,
1659    pub sequence: Option<u64>,
1660    pub expected: Option<String>,
1661    pub actual: Option<String>,
1662    pub message: String,
1663}
1664
1665impl ReplayFailure {
1666    fn new(
1667        reason: impl Into<String>,
1668        sequence: Option<u64>,
1669        expected: Option<String>,
1670        actual: Option<String>,
1671        message: impl Into<String>,
1672    ) -> Self {
1673        Self {
1674            reason: reason.into(),
1675            sequence,
1676            expected,
1677            actual,
1678            message: message.into(),
1679        }
1680    }
1681}
1682
1683/// A stable, machine-readable workflow query or query-task settlement failure.
1684#[derive(Clone, Debug, Error)]
1685#[error("query failed ({reason}, HTTP {status}): {message}")]
1686pub struct QueryFailure {
1687    pub status: u16,
1688    pub reason: String,
1689    pub message: String,
1690    pub body: Value,
1691}
1692
1693/// A stable failure returned when a server rejects an SDK protocol version.
1694#[derive(Clone, Debug, Error)]
1695#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1696pub struct ProtocolFailure {
1697    pub status: u16,
1698    pub reason: String,
1699    pub message: String,
1700    pub supported_version: Option<String>,
1701    pub requested_version: Option<String>,
1702    pub body: Value,
1703}
1704
1705#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1706pub struct PayloadEnvelope {
1707    pub codec: String,
1708    pub blob: String,
1709}
1710
1711impl PayloadEnvelope {
1712    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1713        encode_payload(value, DEFAULT_CODEC)
1714    }
1715
1716    /// Encode an explicit typed value, including the bytes branch that JSON
1717    /// serialization cannot represent.
1718    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1719        encode_avro_value(value)
1720    }
1721}
1722
1723/// Native adapter for the fixed language-neutral Avro Value schema.
1724#[derive(Clone, Debug)]
1725pub enum AvroValue {
1726    Null,
1727    Boolean(bool),
1728    Long(i64),
1729    Double(f64),
1730    Bytes(Vec<u8>),
1731    String(String),
1732    Array(Vec<AvroValue>),
1733    Map(BTreeMap<String, AvroValue>),
1734}
1735
1736impl PartialEq for AvroValue {
1737    fn eq(&self, other: &Self) -> bool {
1738        match (self, other) {
1739            (Self::Null, Self::Null) => true,
1740            (Self::Boolean(left), Self::Boolean(right)) => left == right,
1741            (Self::Long(left), Self::Long(right)) => left == right,
1742            (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1743            (Self::Bytes(left), Self::Bytes(right)) => left == right,
1744            (Self::String(left), Self::String(right)) => left == right,
1745            (Self::Array(left), Self::Array(right)) => left == right,
1746            (Self::Map(left), Self::Map(right)) => left == right,
1747            _ => false,
1748        }
1749    }
1750}
1751
1752impl AvroValue {
1753    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1754        Self::from_serde_value(
1755            serde_value::to_value(value).map_err(|error| {
1756                Error::Codec(format!("could not adapt value for Avro: {error}"))
1757            })?,
1758        )
1759    }
1760
1761    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1762        use serde_value::Value as SerdeValue;
1763
1764        match value {
1765            SerdeValue::Unit => Ok(Self::Null),
1766            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1767            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1768            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1769            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1770            SerdeValue::I64(value) => Ok(Self::Long(value)),
1771            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1772            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1773            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1774            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1775                Error::Codec(
1776                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1777                        .to_string(),
1778                )
1779            }),
1780            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1781            SerdeValue::F64(value) => Self::finite_double(value),
1782            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1783            SerdeValue::String(value) => Ok(Self::String(value)),
1784            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1785            SerdeValue::Option(None) => Ok(Self::Null),
1786            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1787                Self::from_serde_value(*value)
1788            }
1789            SerdeValue::Seq(values) => values
1790                .into_iter()
1791                .map(Self::from_serde_value)
1792                .collect::<Result<Vec<_>>>()
1793                .map(Self::Array),
1794            SerdeValue::Map(values) => values
1795                .into_iter()
1796                .map(|(key, value)| {
1797                    let SerdeValue::String(key) = key else {
1798                        return Err(Error::Codec(
1799                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1800                        ));
1801                    };
1802
1803                    Ok((key, Self::from_serde_value(value)?))
1804                })
1805                .collect::<Result<BTreeMap<_, _>>>()
1806                .map(Self::Map),
1807        }
1808    }
1809
1810    fn finite_double(value: f64) -> Result<Self> {
1811        if !value.is_finite() {
1812            return Err(Error::Codec(
1813                "non_finite_float: Avro Value doubles must be finite".to_string(),
1814            ));
1815        }
1816
1817        Ok(Self::Double(value))
1818    }
1819
1820    fn into_json(self) -> Result<Value> {
1821        match self {
1822            Self::Null => Ok(Value::Null),
1823            Self::Boolean(value) => Ok(Value::Bool(value)),
1824            Self::Long(value) => Ok(Value::Number(value.into())),
1825            Self::Double(value) => serde_json::Number::from_f64(value)
1826                .map(Value::Number)
1827                .ok_or_else(|| {
1828                    Error::Codec(
1829                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1830                    )
1831                }),
1832            Self::Bytes(value) => Ok(json!({
1833                "$type": "bytes",
1834                "base64": BASE64.encode(value),
1835            })),
1836            Self::String(value) => Ok(Value::String(value)),
1837            Self::Array(values) => values
1838                .into_iter()
1839                .map(Self::into_json)
1840                .collect::<Result<Vec<_>>>()
1841                .map(Value::Array),
1842            Self::Map(values) => values
1843                .into_iter()
1844                .map(|(key, value)| Ok((key, value.into_json()?)))
1845                .collect::<Result<serde_json::Map<_, _>>>()
1846                .map(Value::Object),
1847        }
1848    }
1849
1850    fn into_serde_value(self) -> serde_value::Value {
1851        use serde_value::Value as SerdeValue;
1852
1853        match self {
1854            Self::Null => SerdeValue::Unit,
1855            Self::Boolean(value) => SerdeValue::Bool(value),
1856            Self::Long(value) => SerdeValue::I64(value),
1857            Self::Double(value) => SerdeValue::F64(value),
1858            Self::Bytes(value) => SerdeValue::Bytes(value),
1859            Self::String(value) => SerdeValue::String(value),
1860            Self::Array(values) => {
1861                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1862            }
1863            Self::Map(values) => SerdeValue::Map(
1864                values
1865                    .into_iter()
1866                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1867                    .collect(),
1868            ),
1869        }
1870    }
1871
1872    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1873        self.into_serde_value().deserialize_into().map_err(|error| {
1874            Error::Codec(format!(
1875                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1876            ))
1877        })
1878    }
1879}
1880
1881impl Serialize for AvroValue {
1882    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1883    where
1884        S: Serializer,
1885    {
1886        match self {
1887            Self::Null => serializer.serialize_unit(),
1888            Self::Boolean(value) => serializer.serialize_bool(*value),
1889            Self::Long(value) => serializer.serialize_i64(*value),
1890            Self::Double(value) => serializer.serialize_f64(*value),
1891            Self::Bytes(value) => serializer.serialize_bytes(value),
1892            Self::String(value) => serializer.serialize_str(value),
1893            Self::Array(values) => {
1894                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1895                for value in values {
1896                    sequence.serialize_element(value)?;
1897                }
1898                sequence.end()
1899            }
1900            Self::Map(values) => {
1901                let mut map = serializer.serialize_map(Some(values.len()))?;
1902                for (key, value) in values {
1903                    map.serialize_entry(key, value)?;
1904                }
1905                map.end()
1906            }
1907        }
1908    }
1909}
1910
1911pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1912    let datum = avro_value_to_datum(value)?;
1913    let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
1914        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1915    let mut bytes = Vec::with_capacity(datum.len() + 10);
1916    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1917    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1918    bytes.extend_from_slice(&datum);
1919    Ok(PayloadEnvelope {
1920        codec: DEFAULT_CODEC.to_string(),
1921        blob: BASE64.encode(bytes),
1922    })
1923}
1924
1925pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1926    if envelope.codec != DEFAULT_CODEC {
1927        return Err(unsupported_payload_codec(&envelope.codec));
1928    }
1929    decode_avro_value_blob(&envelope.blob)
1930}
1931
1932pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1933    let blob = match codec {
1934        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1935        other => return Err(unsupported_payload_codec(other)),
1936    };
1937
1938    Ok(PayloadEnvelope {
1939        codec: codec.to_string(),
1940        blob,
1941    })
1942}
1943
1944pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1945    match envelope.codec.as_str() {
1946        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1947        other => Err(unsupported_payload_codec(other)),
1948    }
1949}
1950
1951fn handler_type_error<T>(
1952    handler_kind: HandlerKind,
1953    handler_name: &str,
1954    value_kind: HandlerValueKind,
1955    message: impl Into<String>,
1956) -> Error {
1957    Error::HandlerType {
1958        handler_kind,
1959        handler_name: handler_name.to_string(),
1960        value_kind,
1961        rust_type: type_name::<T>(),
1962        message: message.into(),
1963    }
1964}
1965
1966fn decode_handler_input<T: DeserializeOwned>(
1967    arguments: AvroValue,
1968    handler_kind: HandlerKind,
1969    handler_name: &str,
1970) -> Result<T> {
1971    let argument = match arguments {
1972        AvroValue::Array(mut arguments) if arguments.len() == 1 => {
1973            arguments.pop().expect("one typed handler argument")
1974        }
1975        AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
1976        AvroValue::Array(arguments) => {
1977            return Err(handler_type_error::<T>(
1978                handler_kind,
1979                handler_name,
1980                HandlerValueKind::Input,
1981                format!(
1982                    "typed handlers accept one request value, but the task carried {} arguments",
1983                    arguments.len()
1984                ),
1985            ));
1986        }
1987        argument => argument,
1988    };
1989
1990    argument.deserialize().map_err(|error| {
1991        handler_type_error::<T>(
1992            handler_kind,
1993            handler_name,
1994            HandlerValueKind::Input,
1995            error.to_string(),
1996        )
1997    })
1998}
1999
2000fn encode_handler_result<T: Serialize>(
2001    result: &T,
2002    handler_kind: HandlerKind,
2003    handler_name: &str,
2004) -> Result<AvroValue> {
2005    AvroValue::from_serialize(result).map_err(|error| {
2006        handler_type_error::<T>(
2007            handler_kind,
2008            handler_name,
2009            HandlerValueKind::Result,
2010            error.to_string(),
2011        )
2012    })
2013}
2014
2015fn decode_handler_result<T: DeserializeOwned>(
2016    result: AvroValue,
2017    handler_kind: HandlerKind,
2018    handler_name: &str,
2019) -> Result<T> {
2020    result.deserialize().map_err(|error| {
2021        handler_type_error::<T>(
2022            handler_kind,
2023            handler_name,
2024            HandlerValueKind::Result,
2025            error.to_string(),
2026        )
2027    })
2028}
2029
2030#[cfg(test)]
2031fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2032    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2033}
2034
2035fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2036    validate_payload_codec(fallback_codec)?;
2037
2038    if value.is_null() {
2039        return Ok(Value::Null);
2040    }
2041
2042    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2043        return decode_blob(blob, codec);
2044    }
2045
2046    if let Some(blob) = value.as_str() {
2047        return decode_blob(blob, fallback_codec);
2048    }
2049
2050    Err(untagged_payload_value())
2051}
2052
2053fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2054    let envelope = match codec {
2055        DEFAULT_CODEC => encode_avro_value(value)?,
2056        other => return Err(unsupported_payload_codec(other)),
2057    };
2058    Ok(serde_json::to_value(envelope)?)
2059}
2060
2061fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2062    validate_payload_codec(fallback_codec)?;
2063
2064    if value.is_null() {
2065        return Ok(AvroValue::Null);
2066    }
2067
2068    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2069        validate_payload_codec(codec)?;
2070        return decode_avro_value_blob(blob);
2071    }
2072
2073    if let Some(blob) = value.as_str() {
2074        return match fallback_codec {
2075            DEFAULT_CODEC => decode_avro_value_blob(blob),
2076            other => Err(unsupported_payload_codec(other)),
2077        };
2078    }
2079
2080    Err(untagged_payload_value())
2081}
2082
2083fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2084    match value {
2085        AvroValue::Null => AvroValue::Array(Vec::new()),
2086        AvroValue::Array(_) => value,
2087        other => AvroValue::Array(vec![other]),
2088    }
2089}
2090
2091fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2092    match codec {
2093        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2094        other => Err(unsupported_payload_codec(other)),
2095    }
2096}
2097
2098fn validate_payload_codec(codec: &str) -> Result<()> {
2099    match codec {
2100        DEFAULT_CODEC => Ok(()),
2101        MISSING_TASK_PAYLOAD_CODEC => {
2102            Err(invalid_task_payload_codec("task payload_codec is missing"))
2103        }
2104        NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2105        NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2106            "task payload_codec must be a string",
2107        )),
2108        other => Err(unsupported_payload_codec(other)),
2109    }
2110}
2111
2112fn invalid_task_payload_codec(reason: &str) -> Error {
2113    Error::Codec(format!(
2114        "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2115    ))
2116}
2117
2118fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2119    let Some(object) = value.as_object() else {
2120        return Ok(None);
2121    };
2122    if !object.contains_key("codec") && !object.contains_key("blob") {
2123        return Ok(None);
2124    }
2125
2126    let codec = object
2127        .get("codec")
2128        .and_then(Value::as_str)
2129        .ok_or_else(invalid_payload_envelope)?;
2130    validate_payload_codec(codec)?;
2131    let blob = object
2132        .get("blob")
2133        .and_then(Value::as_str)
2134        .ok_or_else(invalid_payload_envelope)?;
2135    Ok(Some((codec, blob)))
2136}
2137
2138fn invalid_payload_envelope() -> Error {
2139    Error::Codec(
2140        "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2141            .to_string(),
2142    )
2143}
2144
2145fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2146    for command in commands {
2147        let Some(command) = command.as_object() else {
2148            continue;
2149        };
2150        let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2151            continue;
2152        };
2153        let Some(payload_field) = workflow_command_payload_field(command_type) else {
2154            continue;
2155        };
2156
2157        if let Some(codec) = command.get("payload_codec") {
2158            let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2159            validate_payload_codec(codec)?;
2160        }
2161
2162        let payload = command
2163            .get(payload_field)
2164            .ok_or_else(invalid_payload_envelope)?;
2165        validate_outbound_payload_envelope(payload)?;
2166    }
2167    Ok(())
2168}
2169
2170fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2171    if commands.iter().any(|command| {
2172        command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2173            && command
2174                .get("condition_wait_occurrence_id")
2175                .and_then(Value::as_str)
2176                .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2177    }) {
2178        CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2179    } else if commands.iter().any(|command| {
2180        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2181            && command.get("attribute_types").is_some()
2182    }) {
2183        TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2184    } else if commands
2185        .iter()
2186        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2187    {
2188        MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2189    } else if commands
2190        .iter()
2191        .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2192    {
2193        CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2194    } else if commands.iter().any(|command| {
2195        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2196    }) {
2197        SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2198    } else {
2199        WORKER_PROTOCOL_VERSION
2200    }
2201}
2202
2203fn workflow_completion_protocol_version_with_message_streams(
2204    commands: &[Value],
2205    has_message_stream_metadata: bool,
2206) -> &'static str {
2207    let command_protocol = workflow_completion_protocol_version(commands);
2208    if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2209        MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2210    } else {
2211        command_protocol
2212    }
2213}
2214
2215fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2216    match command_type {
2217        "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2218        "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2219        "start_service_operation" => Some("request_payload"),
2220        "upsert_memo" => Some("entries"),
2221        _ => None,
2222    }
2223}
2224
2225fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2226    let Some((codec, blob)) = payload_envelope_parts(value)? else {
2227        return Err(untagged_payload_value());
2228    };
2229    validate_payload_codec(codec)?;
2230    decode_avro_value_blob(blob)?;
2231    Ok(())
2232}
2233
2234fn unsupported_payload_codec(codec: &str) -> Error {
2235    Error::Codec(format!(
2236        "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2237    ))
2238}
2239
2240fn untagged_payload_value() -> Error {
2241    Error::Codec(
2242        "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2243            .to_string(),
2244    )
2245}
2246
2247fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2248    let bytes = BASE64.decode(blob).map_err(|err| {
2249        Error::Codec(format!(
2250            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2251        ))
2252    })?;
2253
2254    if serde_json::from_slice::<Value>(&bytes).is_ok() {
2255        return Err(unsupported_payload_codec("json"));
2256    }
2257
2258    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2259        return Err(Error::Codec(
2260            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2261        ));
2262    }
2263
2264    let fingerprint: [u8; 8] = bytes[2..10]
2265        .try_into()
2266        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2267    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2268        return Err(Error::Codec(format!(
2269            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2270            fingerprint
2271                .iter()
2272                .map(|byte| format!("{byte:02x}"))
2273                .collect::<String>()
2274        )));
2275    }
2276
2277    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2278    // The current fingerprint selects the current immutable schema, so reader
2279    // resolution would only re-walk the same recursive union. Future retained
2280    // writer fingerprints supply a distinct reader schema in this branch.
2281    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2282    if datum_reader.truncated {
2283        return Err(Error::Codec(
2284            "invalid_payload_framing: truncated Avro Value datum".to_string(),
2285        ));
2286    }
2287    let datum = datum.map_err(|err| {
2288        Error::Codec(format!(
2289            "invalid_payload_framing: malformed Avro Value datum: {err}"
2290        ))
2291    })?;
2292    if datum_reader.remaining() != 0 {
2293        return Err(Error::Codec(format!(
2294            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2295            datum_reader.remaining()
2296        )));
2297    }
2298    avro_value_from_datum(datum)
2299}
2300
2301struct StrictAvroDatumReader<'a> {
2302    bytes: &'a [u8],
2303    offset: usize,
2304    truncated: bool,
2305}
2306
2307impl<'a> StrictAvroDatumReader<'a> {
2308    fn new(bytes: &'a [u8]) -> Self {
2309        Self {
2310            bytes,
2311            offset: 0,
2312            truncated: false,
2313        }
2314    }
2315
2316    fn remaining(&self) -> usize {
2317        self.bytes.len() - self.offset
2318    }
2319}
2320
2321impl Read for StrictAvroDatumReader<'_> {
2322    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2323        let count = buffer.len().min(self.remaining());
2324        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2325        self.offset += count;
2326        if count < buffer.len() {
2327            self.truncated = true;
2328        }
2329
2330        Ok(count)
2331    }
2332}
2333
2334fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2335    let branch = match value {
2336        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2337        AvroValue::Boolean(value) => AvroDatum::Union(
2338            1,
2339            Box::new(AvroDatum::Record(vec![(
2340                "boolean".to_string(),
2341                AvroDatum::Boolean(*value),
2342            )])),
2343        ),
2344        AvroValue::Long(value) => AvroDatum::Union(
2345            2,
2346            Box::new(AvroDatum::Record(vec![(
2347                "long".to_string(),
2348                AvroDatum::Long(*value),
2349            )])),
2350        ),
2351        AvroValue::Double(value) => {
2352            if !value.is_finite() {
2353                return Err(Error::Codec(
2354                    "non_finite_float: Avro Value doubles must be finite".to_string(),
2355                ));
2356            }
2357            AvroDatum::Union(
2358                3,
2359                Box::new(AvroDatum::Record(vec![(
2360                    "double".to_string(),
2361                    AvroDatum::Double(*value),
2362                )])),
2363            )
2364        }
2365        AvroValue::Bytes(value) => AvroDatum::Union(
2366            4,
2367            Box::new(AvroDatum::Record(vec![(
2368                "bytes".to_string(),
2369                AvroDatum::Bytes(value.clone()),
2370            )])),
2371        ),
2372        AvroValue::String(value) => AvroDatum::Union(
2373            5,
2374            Box::new(AvroDatum::Record(vec![(
2375                "string".to_string(),
2376                AvroDatum::String(value.clone()),
2377            )])),
2378        ),
2379        AvroValue::Array(values) => AvroDatum::Union(
2380            6,
2381            Box::new(AvroDatum::Record(vec![(
2382                "items".to_string(),
2383                AvroDatum::Array(
2384                    values
2385                        .iter()
2386                        .map(avro_value_to_datum)
2387                        .collect::<Result<Vec<_>>>()?,
2388                ),
2389            )])),
2390        ),
2391        AvroValue::Map(values) => AvroDatum::Union(
2392            7,
2393            Box::new(AvroDatum::Record(vec![(
2394                "entries".to_string(),
2395                AvroDatum::Array(
2396                    values
2397                        .iter()
2398                        .map(|(key, value)| {
2399                            Ok(AvroDatum::Record(vec![
2400                                ("key".to_string(), AvroDatum::String(key.clone())),
2401                                ("value".to_string(), avro_value_to_datum(value)?),
2402                            ]))
2403                        })
2404                        .collect::<Result<Vec<_>>>()?,
2405                ),
2406            )])),
2407        ),
2408    };
2409    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2410}
2411
2412fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2413    let AvroDatum::Record(mut outer) = datum else {
2414        return Err(Error::Codec(
2415            "invalid_payload_framing: datum is not a Value record".to_string(),
2416        ));
2417    };
2418    let (_, branch) = outer
2419        .pop()
2420        .filter(|(name, _)| name == "value")
2421        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2422    let AvroDatum::Union(_, branch) = branch else {
2423        return Err(Error::Codec(
2424            "invalid_payload_framing: invalid Value union".to_string(),
2425        ));
2426    };
2427    match *branch {
2428        AvroDatum::Null => Ok(AvroValue::Null),
2429        AvroDatum::Record(mut fields) => {
2430            let (name, value) = fields.pop().ok_or_else(|| {
2431                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2432            })?;
2433            match (name.as_str(), value) {
2434                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2435                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2436                ("double", AvroDatum::Double(value)) if value.is_finite() => {
2437                    Ok(AvroValue::Double(value))
2438                }
2439                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2440                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2441                ("items", AvroDatum::Array(values)) => values
2442                    .into_iter()
2443                    .map(avro_value_from_datum)
2444                    .collect::<Result<Vec<_>>>()
2445                    .map(AvroValue::Array),
2446                ("entries", AvroDatum::Map(values)) => values
2447                    .into_iter()
2448                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2449                    .collect::<Result<BTreeMap<_, _>>>()
2450                    .map(AvroValue::Map),
2451                _ => Err(Error::Codec(
2452                    "invalid_payload_framing: unknown Value branch".to_string(),
2453                )),
2454            }
2455        }
2456        _ => Err(Error::Codec(
2457            "invalid_payload_framing: invalid Value branch".to_string(),
2458        )),
2459    }
2460}
2461
2462fn avro_value_schema() -> Result<&'static Schema> {
2463    match AVRO_VALUE_SCHEMA.get_or_init(|| {
2464        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2465            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2466    }) {
2467        Ok(schema) => Ok(schema),
2468        Err(message) => Err(Error::Codec(message.clone())),
2469    }
2470}
2471
2472fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2473    match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2474        // Apache Avro's Value::Map uses a randomized HashMap. Arrays and maps
2475        // have the same block representation when each ordered array record
2476        // contains the map key followed by its value, so this encoding-only
2477        // adaptation lets the official encoder retain BTreeMap wire order.
2478        let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2479            .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2480        let entries_schema = schema
2481            .pointer_mut("/fields/0/type/7/fields/0/type")
2482            .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2483        if *entries_schema != json!({"type": "map", "values": "Value"}) {
2484            return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2485        }
2486        *entries_schema = json!({
2487            "type": "array",
2488            "items": {
2489                "type": "record",
2490                "name": "MapEntry",
2491                "fields": [
2492                    {"name": "key", "type": "string"},
2493                    {"name": "value", "type": "Value"}
2494                ]
2495            }
2496        });
2497        Schema::parse_str(&schema.to_string())
2498            .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2499    }) {
2500        Ok(schema) => Ok(schema),
2501        Err(message) => Err(Error::Codec(message.clone())),
2502    }
2503}
2504
2505#[derive(Clone, Debug)]
2506pub struct Client {
2507    http: reqwest::Client,
2508    base_url: String,
2509    token: Option<String>,
2510    control_token: Option<String>,
2511    worker_token: Option<String>,
2512    namespace: String,
2513}
2514
2515impl Client {
2516    pub fn new(base_url: impl Into<String>) -> Result<Self> {
2517        Self::builder(base_url).build()
2518    }
2519
2520    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2521        ClientBuilder {
2522            base_url: base_url.into(),
2523            token: None,
2524            control_token: None,
2525            worker_token: None,
2526            namespace: "default".to_string(),
2527            timeout: Duration::from_secs(60),
2528        }
2529    }
2530
2531    pub async fn health(&self) -> Result<Value> {
2532        self.request_json(
2533            reqwest::Method::GET,
2534            "/health",
2535            RequestProtocol::ControlPlane,
2536            Option::<&Value>::None,
2537        )
2538        .await
2539    }
2540
2541    pub async fn cluster_info(&self) -> Result<Value> {
2542        self.request_json(
2543            reqwest::Method::GET,
2544            "/cluster/info",
2545            RequestProtocol::ControlPlane,
2546            Option::<&Value>::None,
2547        )
2548        .await
2549    }
2550
2551    pub async fn start_workflow<T: Serialize>(
2552        &self,
2553        workflow_type: &str,
2554        task_queue: &str,
2555        workflow_id: &str,
2556        input: T,
2557    ) -> Result<WorkflowHandle> {
2558        self.start_workflow_with_options(
2559            workflow_type,
2560            task_queue,
2561            workflow_id,
2562            WorkflowStartOptions::default(),
2563            input,
2564        )
2565        .await
2566    }
2567
2568    /// Start a workflow with explicit server-enforced execution and run
2569    /// deadlines.
2570    pub async fn start_workflow_with_options<T: Serialize>(
2571        &self,
2572        workflow_type: &str,
2573        task_queue: &str,
2574        workflow_id: &str,
2575        options: WorkflowStartOptions,
2576        input: T,
2577    ) -> Result<WorkflowHandle> {
2578        options.validate()?;
2579        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2580        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2581        let body = json!({
2582            "workflow_id": workflow_id,
2583            "workflow_type": workflow_type,
2584            "task_queue": task_queue,
2585            "input": input_envelope,
2586            "execution_timeout_seconds": options.execution_timeout_seconds,
2587            "run_timeout_seconds": options.run_timeout_seconds
2588        });
2589
2590        let data: Value = self
2591            .request_json(
2592                reqwest::Method::POST,
2593                "/workflows",
2594                RequestProtocol::ControlPlane,
2595                Some(&body),
2596            )
2597            .await?;
2598
2599        Ok(WorkflowHandle {
2600            client: self.clone(),
2601            workflow_id: data
2602                .get("workflow_id")
2603                .and_then(Value::as_str)
2604                .unwrap_or(workflow_id)
2605                .to_string(),
2606            run_id: data
2607                .get("run_id")
2608                .and_then(Value::as_str)
2609                .map(str::to_string),
2610            workflow_type: data
2611                .get("workflow_type")
2612                .and_then(Value::as_str)
2613                .unwrap_or(workflow_type)
2614                .to_string(),
2615        })
2616    }
2617
2618    pub async fn signal_workflow<T: Serialize>(
2619        &self,
2620        workflow_id: &str,
2621        signal_name: &str,
2622        input: T,
2623    ) -> Result<Value> {
2624        self.signal_workflow_target(workflow_id, None, signal_name, input)
2625            .await
2626    }
2627
2628    /// Append one idempotently identified item to an instance-scoped input stream.
2629    pub async fn append_message_stream<T: Serialize>(
2630        &self,
2631        workflow_id: &str,
2632        stream_name: &str,
2633        message_id: &str,
2634        input: T,
2635    ) -> Result<Value> {
2636        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2637        let body = json!({
2638            "message_id": message_id,
2639            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2640        });
2641        self.request_json(
2642            reqwest::Method::POST,
2643            &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2644            RequestProtocol::ControlPlane,
2645            Some(&body),
2646        )
2647        .await
2648    }
2649
2650    /// Signal only if `run_id` is still the current run for this instance.
2651    pub async fn signal_workflow_run<T: Serialize>(
2652        &self,
2653        workflow_id: &str,
2654        run_id: &str,
2655        signal_name: &str,
2656        input: T,
2657    ) -> Result<Value> {
2658        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2659            .await
2660    }
2661
2662    async fn signal_workflow_target<T: Serialize>(
2663        &self,
2664        workflow_id: &str,
2665        run_id: Option<&str>,
2666        signal_name: &str,
2667        input: T,
2668    ) -> Result<Value> {
2669        validate_user_signal_name(signal_name)?;
2670        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2671        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2672        let body = json!({
2673            "input": input_envelope
2674        });
2675        let path = match run_id {
2676            Some(run_id) => {
2677                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2678            }
2679            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2680        };
2681        self.request_json(
2682            reqwest::Method::POST,
2683            &path,
2684            RequestProtocol::ControlPlane,
2685            Some(&body),
2686        )
2687        .await
2688    }
2689
2690    /// Request cooperative cancellation of the current run for an instance.
2691    pub async fn cancel_workflow(
2692        &self,
2693        workflow_id: &str,
2694        options: WorkflowCommandOptions,
2695    ) -> Result<WorkflowCommandResult> {
2696        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2697            .await
2698    }
2699
2700    /// Request cooperative cancellation only if `run_id` is still current.
2701    pub async fn cancel_workflow_run(
2702        &self,
2703        workflow_id: &str,
2704        run_id: &str,
2705        options: WorkflowCommandOptions,
2706    ) -> Result<WorkflowCommandResult> {
2707        self.workflow_command(
2708            workflow_id,
2709            Some(run_id),
2710            WorkflowCommandKind::Cancel,
2711            options,
2712        )
2713        .await
2714    }
2715
2716    /// Forcefully terminate the current run for an instance.
2717    pub async fn terminate_workflow(
2718        &self,
2719        workflow_id: &str,
2720        options: WorkflowCommandOptions,
2721    ) -> Result<WorkflowCommandResult> {
2722        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2723            .await
2724    }
2725
2726    /// Forcefully terminate only if `run_id` is still current.
2727    pub async fn terminate_workflow_run(
2728        &self,
2729        workflow_id: &str,
2730        run_id: &str,
2731        options: WorkflowCommandOptions,
2732    ) -> Result<WorkflowCommandResult> {
2733        self.workflow_command(
2734            workflow_id,
2735            Some(run_id),
2736            WorkflowCommandKind::Terminate,
2737            options,
2738        )
2739        .await
2740    }
2741
2742    async fn workflow_command(
2743        &self,
2744        workflow_id: &str,
2745        run_id: Option<&str>,
2746        command: WorkflowCommandKind,
2747        options: WorkflowCommandOptions,
2748    ) -> Result<WorkflowCommandResult> {
2749        let path = match run_id {
2750            Some(run_id) => format!(
2751                "/workflows/{workflow_id}/runs/{run_id}/{}",
2752                command.as_str()
2753            ),
2754            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2755        };
2756        let data = match self
2757            .request_json(
2758                reqwest::Method::POST,
2759                &path,
2760                RequestProtocol::ControlPlane,
2761                Some(&options),
2762            )
2763            .await
2764        {
2765            Ok(data) => data,
2766            Err(Error::Http { status, body }) => {
2767                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2768                    command,
2769                    status,
2770                    body,
2771                    workflow_id,
2772                    run_id,
2773                )));
2774            }
2775            Err(error) => return Err(error),
2776        };
2777
2778        Ok(workflow_command_result(command, data, workflow_id, run_id))
2779    }
2780
2781    /// Execute a named, read-only query against a running or completed workflow.
2782    ///
2783    /// Arguments and results use the platform payload envelope. Server and
2784    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
2785    /// reason, HTTP status, and original response body.
2786    pub async fn query_workflow<T: Serialize>(
2787        &self,
2788        workflow_id: &str,
2789        query_name: &str,
2790        input: T,
2791    ) -> Result<Value> {
2792        self.query_workflow_target(workflow_id, None, query_name, input)
2793            .await
2794    }
2795
2796    /// Query only if `run_id` is still current, preventing accidental retargeting.
2797    pub async fn query_workflow_run<T: Serialize>(
2798        &self,
2799        workflow_id: &str,
2800        run_id: &str,
2801        query_name: &str,
2802        input: T,
2803    ) -> Result<Value> {
2804        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2805            .await
2806    }
2807
2808    /// Query a workflow and return the lossless fixed Avro Value result.
2809    pub async fn query_workflow_avro_value<T: Serialize>(
2810        &self,
2811        workflow_id: &str,
2812        query_name: &str,
2813        input: T,
2814    ) -> Result<AvroValue> {
2815        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2816            .await
2817    }
2818
2819    /// Query a selected run and return the lossless fixed Avro Value result.
2820    pub async fn query_workflow_run_avro_value<T: Serialize>(
2821        &self,
2822        workflow_id: &str,
2823        run_id: &str,
2824        query_name: &str,
2825        input: T,
2826    ) -> Result<AvroValue> {
2827        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2828            .await
2829    }
2830
2831    async fn query_workflow_avro_value_target<T: Serialize>(
2832        &self,
2833        workflow_id: &str,
2834        run_id: Option<&str>,
2835        query_name: &str,
2836        input: T,
2837    ) -> Result<AvroValue> {
2838        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2839        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2840        let path = match run_id {
2841            Some(run_id) => {
2842                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2843            }
2844            None => format!("/workflows/{workflow_id}/query/{query_name}"),
2845        };
2846        let response: Value = match self
2847            .request_json(
2848                reqwest::Method::POST,
2849                &path,
2850                RequestProtocol::ControlPlane,
2851                Some(&body),
2852            )
2853            .await
2854        {
2855            Ok(response) => response,
2856            Err(Error::Http { status, body }) => {
2857                return Err(Error::QueryFailed(query_failure(status, body)));
2858            }
2859            Err(error) => return Err(error),
2860        };
2861
2862        let envelope = response
2863            .get("result_envelope")
2864            .filter(|envelope| !envelope.is_null())
2865            .ok_or_else(|| {
2866                Error::Codec(
2867                    "missing_payload_envelope: typed query result requires result_envelope"
2868                        .to_string(),
2869                )
2870            })?;
2871        decode_wire_avro_value(envelope, DEFAULT_CODEC)
2872    }
2873
2874    async fn query_workflow_target<T: Serialize>(
2875        &self,
2876        workflow_id: &str,
2877        run_id: Option<&str>,
2878        query_name: &str,
2879        input: T,
2880    ) -> Result<Value> {
2881        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2882        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2883        let body = json!({
2884            "input": input_envelope
2885        });
2886        let path = match run_id {
2887            Some(run_id) => {
2888                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2889            }
2890            None => format!("/workflows/{workflow_id}/query/{query_name}"),
2891        };
2892        let response: Value = match self
2893            .request_json(
2894                reqwest::Method::POST,
2895                &path,
2896                RequestProtocol::ControlPlane,
2897                Some(&body),
2898            )
2899            .await
2900        {
2901            Ok(response) => response,
2902            Err(Error::Http { status, body }) => {
2903                return Err(Error::QueryFailed(query_failure(status, body)));
2904            }
2905            Err(error) => return Err(error),
2906        };
2907
2908        if let Some(envelope) = response
2909            .get("result_envelope")
2910            .filter(|envelope| !envelope.is_null())
2911        {
2912            return decode_wire_value(envelope, DEFAULT_CODEC);
2913        }
2914
2915        Ok(response.get("result").cloned().unwrap_or(Value::Null))
2916    }
2917
2918    /// Send a synchronous update using fixed Avro Value arguments.
2919    pub async fn update_workflow<T: Serialize>(
2920        &self,
2921        workflow_id: &str,
2922        update_name: &str,
2923        input: T,
2924        request_id: Option<&str>,
2925    ) -> Result<Value> {
2926        let response = self
2927            .update_workflow_response(workflow_id, update_name, input, request_id)
2928            .await?;
2929        if let Some(envelope) = response
2930            .get("result_envelope")
2931            .filter(|envelope| !envelope.is_null())
2932        {
2933            return decode_wire_value(envelope, DEFAULT_CODEC);
2934        }
2935        Ok(response.get("result").cloned().unwrap_or(response))
2936    }
2937
2938    /// Send a synchronous update and retain a bytes-capable Avro result.
2939    pub async fn update_workflow_avro_value<T: Serialize>(
2940        &self,
2941        workflow_id: &str,
2942        update_name: &str,
2943        input: T,
2944        request_id: Option<&str>,
2945    ) -> Result<AvroValue> {
2946        let response = self
2947            .update_workflow_response(workflow_id, update_name, input, request_id)
2948            .await?;
2949        let envelope = response
2950            .get("result_envelope")
2951            .filter(|envelope| !envelope.is_null())
2952            .ok_or_else(|| {
2953                Error::Codec(
2954                    "missing_payload_envelope: typed update result requires result_envelope"
2955                        .to_string(),
2956                )
2957            })?;
2958        decode_wire_avro_value(envelope, DEFAULT_CODEC)
2959    }
2960
2961    async fn update_workflow_response<T: Serialize>(
2962        &self,
2963        workflow_id: &str,
2964        update_name: &str,
2965        input: T,
2966        request_id: Option<&str>,
2967    ) -> Result<Value> {
2968        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2969        let mut body = json!({
2970            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
2971            "wait_for": "completed",
2972        });
2973        if let Some(request_id) = request_id {
2974            body["request_id"] = json!(request_id);
2975        }
2976        self.request_json(
2977            reqwest::Method::POST,
2978            &format!("/workflows/{workflow_id}/update/{update_name}"),
2979            RequestProtocol::ControlPlane,
2980            Some(&body),
2981        )
2982        .await
2983    }
2984
2985    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2986        let path = format!("/workflows/{workflow_id}");
2987        let mut data: WorkflowDescription = self
2988            .request_json(
2989                reqwest::Method::GET,
2990                &path,
2991                RequestProtocol::ControlPlane,
2992                Option::<&Value>::None,
2993            )
2994            .await?;
2995        data.decode_payloads()?;
2996        Ok(data)
2997    }
2998
2999    /// Describe one selected run, including historical terminal runs.
3000    pub async fn describe_workflow_run(
3001        &self,
3002        workflow_id: &str,
3003        run_id: &str,
3004    ) -> Result<WorkflowDescription> {
3005        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3006        let mut data: WorkflowDescription = self
3007            .request_json(
3008                reqwest::Method::GET,
3009                &path,
3010                RequestProtocol::ControlPlane,
3011                Option::<&Value>::None,
3012            )
3013            .await?;
3014        data.decode_payloads()?;
3015        Ok(data)
3016    }
3017
3018    fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3019        let mut path = format!(
3020            "/workflows/{}/runs/{}/streams",
3021            percent_encode_path_segment(workflow_id),
3022            percent_encode_path_segment(run_id),
3023        );
3024        if let Some(stream_name) = stream_name {
3025            path.push('/');
3026            path.push_str(&percent_encode_path_segment(stream_name));
3027        }
3028        path
3029    }
3030
3031    /// List the run-scoped output streams already opened by a workflow.
3032    pub async fn list_workflow_streams(
3033        &self,
3034        workflow_id: &str,
3035        run_id: &str,
3036    ) -> Result<Vec<WorkflowStreamDescription>> {
3037        let response: WorkflowStreamListResponse = self
3038            .request_json(
3039                reqwest::Method::GET,
3040                &Self::workflow_stream_path(workflow_id, run_id, None),
3041                RequestProtocol::ControlPlane,
3042                Option::<&Value>::None,
3043            )
3044            .await?;
3045        Ok(response.streams)
3046    }
3047
3048    /// Describe stream lifecycle, offsets, pending count, and terminal error.
3049    pub async fn describe_workflow_stream(
3050        &self,
3051        workflow_id: &str,
3052        run_id: &str,
3053        stream_name: &str,
3054    ) -> Result<WorkflowStreamDescription> {
3055        let response: WorkflowStreamDescriptionResponse = self
3056            .request_json(
3057                reqwest::Method::GET,
3058                &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3059                RequestProtocol::ControlPlane,
3060                Option::<&Value>::None,
3061            )
3062            .await?;
3063        Ok(response.stream)
3064    }
3065
3066    /// Read one bounded page beginning at a zero-based offset.
3067    ///
3068    /// Delivery is at least once: persist `next_offset` only after processing
3069    /// the page. The future is cancellation-safe; dropping it cancels the
3070    /// in-flight request. Long polling is capped at 60 seconds by the SDK and
3071    /// service contract.
3072    pub async fn subscribe_workflow_stream(
3073        &self,
3074        workflow_id: &str,
3075        run_id: &str,
3076        stream_name: &str,
3077        from_offset: u64,
3078        max_items: usize,
3079        wait: Duration,
3080    ) -> Result<WorkflowStreamPage> {
3081        let max_items = max_items.clamp(1, 500);
3082        let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3083        let path = format!(
3084            "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3085            Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3086        );
3087        let response: WorkflowStreamPageResponse = self
3088            .request_json_with_timeout(
3089                reqwest::Method::GET,
3090                &path,
3091                RequestProtocol::ControlPlane,
3092                Option::<&Value>::None,
3093                Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3094            )
3095            .await?;
3096
3097        let items = response
3098            .items
3099            .into_iter()
3100            .map(|raw| {
3101                let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3102                let envelope = raw.get("payload").cloned();
3103                let payload = envelope
3104                    .as_ref()
3105                    .filter(|value| value.get("blob").is_some())
3106                    .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3107                    .transpose()?
3108                    .map(AvroValue::into_json)
3109                    .transpose()?;
3110                Ok(WorkflowStreamItem {
3111                    offset,
3112                    payload,
3113                    payload_envelope: envelope,
3114                    payload_reference: raw
3115                        .get("payload_reference")
3116                        .and_then(Value::as_str)
3117                        .map(str::to_string),
3118                    payload_codec: raw
3119                        .get("payload_codec")
3120                        .and_then(Value::as_str)
3121                        .map(str::to_string),
3122                    idempotency_key: raw
3123                        .get("idempotency_key")
3124                        .and_then(Value::as_str)
3125                        .map(str::to_string),
3126                    item_type: raw
3127                        .get("item_type")
3128                        .and_then(Value::as_str)
3129                        .map(str::to_string),
3130                    content_type: raw
3131                        .get("content_type")
3132                        .and_then(Value::as_str)
3133                        .map(str::to_string),
3134                    origin: raw
3135                        .get("origin")
3136                        .and_then(Value::as_str)
3137                        .map(str::to_string),
3138                    origin_reference: raw
3139                        .get("origin_reference")
3140                        .and_then(Value::as_str)
3141                        .map(str::to_string),
3142                    emitted_at: raw
3143                        .get("emitted_at")
3144                        .and_then(Value::as_str)
3145                        .map(str::to_string),
3146                    raw,
3147                })
3148            })
3149            .collect::<Result<Vec<_>>>()?;
3150        Ok(WorkflowStreamPage {
3151            stream: response.stream,
3152            items,
3153            next_offset: response.next_offset,
3154            terminal: response.terminal,
3155        })
3156    }
3157
3158    /// Append inline Avro envelopes or opaque external payload references.
3159    pub async fn append_workflow_stream(
3160        &self,
3161        workflow_id: &str,
3162        run_id: &str,
3163        stream_name: &str,
3164        items: &[WorkflowStreamAppendItem],
3165        max_pending_items: Option<u64>,
3166    ) -> Result<WorkflowStreamAppendResult> {
3167        if items.is_empty() {
3168            return Err(Error::Codec(
3169                "workflow_stream_items_empty: append requires at least one item".to_string(),
3170            ));
3171        }
3172        let mut body = json!({
3173            "items": items
3174                .iter()
3175                .map(|item| item.wire_value(None))
3176                .collect::<Vec<_>>(),
3177        });
3178        if let Some(max_pending_items) = max_pending_items {
3179            if max_pending_items == 0 {
3180                return Err(Error::Codec(
3181                    "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3182                        .to_string(),
3183                ));
3184            }
3185            body["max_pending_items"] = json!(max_pending_items);
3186        }
3187        let response: WorkflowStreamAppendResponse = self
3188            .request_json(
3189                reqwest::Method::POST,
3190                &format!(
3191                    "{}/items",
3192                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3193                ),
3194                RequestProtocol::ControlPlane,
3195                Some(&body),
3196            )
3197            .await?;
3198        Ok(WorkflowStreamAppendResult {
3199            stream: response.stream,
3200            accepted_offsets: response.accepted_offsets,
3201            accepted: response.accepted,
3202            deduped: response.deduped,
3203        })
3204    }
3205
3206    /// Close a stream, or mark it errored when `error_reason` is supplied.
3207    pub async fn close_workflow_stream(
3208        &self,
3209        workflow_id: &str,
3210        run_id: &str,
3211        stream_name: &str,
3212        error_reason: Option<&str>,
3213        retention_seconds: Option<u64>,
3214    ) -> Result<WorkflowStreamDescription> {
3215        let mut body = json!({});
3216        if let Some(error_reason) = error_reason {
3217            body["error_reason"] = json!(error_reason);
3218        }
3219        if let Some(retention_seconds) = retention_seconds {
3220            if retention_seconds == 0 {
3221                return Err(Error::Codec(
3222                    "workflow_stream_retention_invalid: retention_seconds must be positive"
3223                        .to_string(),
3224                ));
3225            }
3226            body["retention_seconds"] = json!(retention_seconds);
3227        }
3228        let response: WorkflowStreamDescriptionResponse = self
3229            .request_json(
3230                reqwest::Method::POST,
3231                &format!(
3232                    "{}/close",
3233                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3234                ),
3235                RequestProtocol::ControlPlane,
3236                Some(&body),
3237            )
3238            .await?;
3239        Ok(response.stream)
3240    }
3241
3242    pub async fn register_worker(
3243        &self,
3244        worker_id: &str,
3245        task_queue: &str,
3246        supported_workflow_types: Vec<String>,
3247        supported_activity_types: Vec<String>,
3248        max_concurrent_workflow_tasks: usize,
3249        max_concurrent_activity_tasks: usize,
3250    ) -> Result<RegisterWorkerResponse> {
3251        self.register_worker_with_capabilities(
3252            worker_id,
3253            task_queue,
3254            supported_workflow_types,
3255            supported_activity_types,
3256            max_concurrent_workflow_tasks,
3257            max_concurrent_activity_tasks,
3258            Vec::new(),
3259        )
3260        .await
3261    }
3262
3263    /// Register a worker and explicitly advertise additive worker capabilities.
3264    pub async fn register_worker_with_capabilities(
3265        &self,
3266        worker_id: &str,
3267        task_queue: &str,
3268        supported_workflow_types: Vec<String>,
3269        supported_activity_types: Vec<String>,
3270        max_concurrent_workflow_tasks: usize,
3271        max_concurrent_activity_tasks: usize,
3272        capabilities: Vec<String>,
3273    ) -> Result<RegisterWorkerResponse> {
3274        self.register_worker_with_command_contracts(
3275            worker_id,
3276            task_queue,
3277            supported_workflow_types,
3278            supported_activity_types,
3279            max_concurrent_workflow_tasks,
3280            max_concurrent_activity_tasks,
3281            capabilities,
3282            Value::Object(serde_json::Map::new()),
3283        )
3284        .await
3285    }
3286
3287    /// Register a worker and advertise its named query and update handlers.
3288    ///
3289    /// This Rust SDK cannot execute synchronous pre-accept update validation,
3290    /// so a workflow contract with a non-empty or malformed
3291    /// `update_validators` declaration returns
3292    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
3293    #[allow(clippy::too_many_arguments)]
3294    pub async fn register_worker_with_command_contracts(
3295        &self,
3296        worker_id: &str,
3297        task_queue: &str,
3298        supported_workflow_types: Vec<String>,
3299        supported_activity_types: Vec<String>,
3300        max_concurrent_workflow_tasks: usize,
3301        max_concurrent_activity_tasks: usize,
3302        capabilities: Vec<String>,
3303        workflow_command_contracts: Value,
3304    ) -> Result<RegisterWorkerResponse> {
3305        if let Some(contracts) = workflow_command_contracts.as_object() {
3306            for (workflow_type, contract) in contracts {
3307                let Some(update_validators) = contract.get("update_validators") else {
3308                    continue;
3309                };
3310                if !update_validators
3311                    .as_array()
3312                    .is_some_and(|validators| validators.is_empty())
3313                {
3314                    return Err(Error::UnsupportedUpdateValidators {
3315                        workflow_type: workflow_type.clone(),
3316                    });
3317                }
3318            }
3319        }
3320
3321        let mut body = json!({
3322            "worker_id": worker_id,
3323            "task_queue": task_queue,
3324            "runtime": "rust",
3325            "sdk_version": SDK_VERSION,
3326            "supported_workflow_types": supported_workflow_types,
3327            "supported_activity_types": supported_activity_types,
3328            "capabilities": capabilities,
3329            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3330            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3331        });
3332        if workflow_command_contracts
3333            .as_object()
3334            .is_some_and(|contracts| !contracts.is_empty())
3335        {
3336            body["workflow_command_contracts"] = workflow_command_contracts;
3337        }
3338
3339        self.request_json(
3340            reqwest::Method::POST,
3341            "/worker/register",
3342            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3343            Some(&body),
3344        )
3345        .await
3346    }
3347
3348    /// Gracefully remove one worker's registration through the worker plane.
3349    ///
3350    /// This operation is separate from operator-facing worker management. It
3351    /// uses worker-protocol authentication and returns the server's lease
3352    /// recovery result.
3353    pub async fn deregister_worker_registration(
3354        &self,
3355        worker_id: &str,
3356    ) -> Result<WorkerDeregistrationEnvelope> {
3357        let path = format!(
3358            "/worker/registrations/{}",
3359            percent_encode_path_segment(worker_id)
3360        );
3361        self.request_json(
3362            reqwest::Method::DELETE,
3363            &path,
3364            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3365            Option::<&Value>::None,
3366        )
3367        .await
3368    }
3369
3370    /// Long-poll for an ephemeral, read-only workflow query task.
3371    pub async fn poll_query_task(
3372        &self,
3373        worker_id: &str,
3374        task_queue: &str,
3375        timeout: Duration,
3376    ) -> Result<Option<QueryTask>> {
3377        Ok(self
3378            .poll_query_task_response(worker_id, task_queue, timeout)
3379            .await?
3380            .task)
3381    }
3382
3383    /// Poll a query task while preserving server stop and drain metadata.
3384    pub async fn poll_query_task_response(
3385        &self,
3386        worker_id: &str,
3387        task_queue: &str,
3388        timeout: Duration,
3389    ) -> Result<PollQueryTaskResponse> {
3390        let poll_request_id = unique_request_id("rust-query-poll");
3391        self.poll_query_task_response_with_request_id(
3392            worker_id,
3393            task_queue,
3394            timeout,
3395            &poll_request_id,
3396            1,
3397        )
3398        .await
3399    }
3400
3401    async fn poll_query_task_response_with_request_id(
3402        &self,
3403        worker_id: &str,
3404        task_queue: &str,
3405        timeout: Duration,
3406        poll_request_id: &str,
3407        transport_retries: usize,
3408    ) -> Result<PollQueryTaskResponse> {
3409        let timeout_seconds = long_poll_timeout_seconds(timeout);
3410        let body = json!({
3411            "worker_id": worker_id,
3412            "task_queue": task_queue,
3413            "poll_request_id": poll_request_id,
3414            "timeout_seconds": timeout_seconds,
3415        });
3416        self.poll_request_json(
3417            "/worker/query-tasks/poll",
3418            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3419            &body,
3420            timeout + Duration::from_secs(5),
3421            transport_retries,
3422        )
3423        .await
3424    }
3425
3426    /// Complete a query task without appending workflow history.
3427    pub async fn complete_query_task<T: Serialize>(
3428        &self,
3429        query_task_id: &str,
3430        lease_owner: &str,
3431        query_task_attempt: u64,
3432        result: T,
3433        codec: &str,
3434    ) -> Result<Value> {
3435        let typed_result = AvroValue::from_serialize(&result)?;
3436        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3437        self.complete_query_task_with_envelope(
3438            query_task_id,
3439            lease_owner,
3440            query_task_attempt,
3441            typed_result.into_json()?,
3442            result_envelope,
3443        )
3444        .await
3445    }
3446
3447    async fn complete_query_task_with_envelope(
3448        &self,
3449        query_task_id: &str,
3450        lease_owner: &str,
3451        query_task_attempt: u64,
3452        result: Value,
3453        result_envelope: Value,
3454    ) -> Result<Value> {
3455        let body = json!({
3456            "lease_owner": lease_owner,
3457            "query_task_attempt": query_task_attempt,
3458            "result": result,
3459            "result_envelope": result_envelope,
3460        });
3461        let path = format!("/worker/query-tasks/{query_task_id}/complete");
3462        let response = self
3463            .request_json(
3464                reqwest::Method::POST,
3465                &path,
3466                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3467                Some(&body),
3468            )
3469            .await;
3470        query_task_response(response)
3471    }
3472
3473    /// Report a stable machine-readable query-task failure.
3474    pub async fn fail_query_task(
3475        &self,
3476        query_task_id: &str,
3477        lease_owner: &str,
3478        query_task_attempt: u64,
3479        message: impl Into<String>,
3480        reason: impl Into<String>,
3481        failure_type: impl Into<String>,
3482    ) -> Result<Value> {
3483        let body = json!({
3484            "lease_owner": lease_owner,
3485            "query_task_attempt": query_task_attempt,
3486            "failure": {
3487                "message": message.into(),
3488                "reason": reason.into(),
3489                "type": failure_type.into(),
3490            }
3491        });
3492        let path = format!("/worker/query-tasks/{query_task_id}/fail");
3493        let response = self
3494            .request_json(
3495                reqwest::Method::POST,
3496                &path,
3497                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3498                Some(&body),
3499            )
3500            .await;
3501        query_task_response(response)
3502    }
3503
3504    pub async fn heartbeat_worker(
3505        &self,
3506        worker_id: &str,
3507        workflow_available: usize,
3508        activity_available: usize,
3509    ) -> Result<Value> {
3510        let body = json!({
3511            "worker_id": worker_id,
3512            "task_slots": {
3513                "workflow_available": workflow_available,
3514                "activity_available": activity_available
3515            },
3516            "process_metrics": {
3517                "process_id": std::process::id(),
3518                "process_uptime_seconds": 0
3519            }
3520        });
3521
3522        self.request_json(
3523            reqwest::Method::POST,
3524            "/worker/heartbeat",
3525            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3526            Some(&body),
3527        )
3528        .await
3529    }
3530
3531    pub async fn poll_workflow_task(
3532        &self,
3533        worker_id: &str,
3534        task_queue: &str,
3535        timeout: Duration,
3536    ) -> Result<Option<WorkflowTask>> {
3537        Ok(self
3538            .poll_workflow_task_response(worker_id, task_queue, timeout)
3539            .await?
3540            .task)
3541    }
3542
3543    pub async fn poll_workflow_task_response(
3544        &self,
3545        worker_id: &str,
3546        task_queue: &str,
3547        timeout: Duration,
3548    ) -> Result<PollWorkflowTaskResponse> {
3549        let poll_request_id = unique_request_id("rust-workflow-poll");
3550        self.poll_workflow_task_response_with_request_id(
3551            worker_id,
3552            task_queue,
3553            timeout,
3554            &poll_request_id,
3555            1,
3556        )
3557        .await
3558    }
3559
3560    async fn poll_workflow_task_response_with_request_id(
3561        &self,
3562        worker_id: &str,
3563        task_queue: &str,
3564        timeout: Duration,
3565        poll_request_id: &str,
3566        transport_retries: usize,
3567    ) -> Result<PollWorkflowTaskResponse> {
3568        let body = json!({
3569            "worker_id": worker_id,
3570            "task_queue": task_queue,
3571            "poll_request_id": poll_request_id,
3572            "timeout_seconds": long_poll_timeout_seconds(timeout),
3573        });
3574        let mut data: PollWorkflowTaskResponse = self
3575            .poll_request_json(
3576                "/worker/workflow-tasks/poll",
3577                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3578                &body,
3579                timeout + Duration::from_secs(5),
3580                transport_retries,
3581            )
3582            .await?;
3583
3584        if let Some(task) = data.task.as_mut() {
3585            self.fetch_remaining_workflow_history(worker_id, task)
3586                .await?;
3587        }
3588
3589        Ok(data)
3590    }
3591
3592    async fn fetch_remaining_workflow_history(
3593        &self,
3594        worker_id: &str,
3595        task: &mut WorkflowTask,
3596    ) -> Result<()> {
3597        let mut next_token = task.next_history_page_token.clone();
3598
3599        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3600            let lease_owner = task
3601                .lease_owner
3602                .clone()
3603                .unwrap_or_else(|| worker_id.to_string());
3604            let page = self
3605                .workflow_task_history_page(
3606                    &task.task_id,
3607                    &lease_owner,
3608                    task.workflow_task_attempt,
3609                    &token,
3610                )
3611                .await?;
3612
3613            task.append_history_page(page);
3614
3615            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3616                return Err(Error::Codec(
3617                    "workflow history pagination returned the same page token".to_string(),
3618                ));
3619            }
3620
3621            next_token = task.next_history_page_token.clone();
3622        }
3623
3624        Ok(())
3625    }
3626
3627    async fn workflow_task_history_page(
3628        &self,
3629        task_id: &str,
3630        lease_owner: &str,
3631        workflow_task_attempt: u64,
3632        next_history_page_token: &str,
3633    ) -> Result<WorkflowTaskHistoryPage> {
3634        let body = json!({
3635            "lease_owner": lease_owner,
3636            "workflow_task_attempt": workflow_task_attempt,
3637            "next_history_page_token": next_history_page_token
3638        });
3639        let path = format!("/worker/workflow-tasks/{task_id}/history");
3640
3641        self.request_json(
3642            reqwest::Method::POST,
3643            &path,
3644            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3645            Some(&body),
3646        )
3647        .await
3648    }
3649
3650    pub async fn complete_workflow_task(
3651        &self,
3652        task_id: &str,
3653        lease_owner: &str,
3654        workflow_task_attempt: u64,
3655        commands: Vec<Value>,
3656    ) -> Result<Value> {
3657        self.complete_workflow_task_with_message_streams(
3658            task_id,
3659            lease_owner,
3660            workflow_task_attempt,
3661            commands,
3662            Vec::new(),
3663            Vec::new(),
3664        )
3665        .await
3666    }
3667
3668    async fn complete_workflow_task_with_message_streams(
3669        &self,
3670        task_id: &str,
3671        lease_owner: &str,
3672        workflow_task_attempt: u64,
3673        commands: Vec<Value>,
3674        message_stream_cursors: Vec<Value>,
3675        message_stream_waits: Vec<Value>,
3676    ) -> Result<Value> {
3677        validate_workflow_task_commands(&commands)?;
3678        let has_message_stream_metadata =
3679            !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3680        if has_message_stream_metadata
3681            && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3682        {
3683            return Err(Error::Codec(
3684                "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3685                    .to_string(),
3686            ));
3687        }
3688        let protocol_version = workflow_completion_protocol_version_with_message_streams(
3689            &commands,
3690            has_message_stream_metadata,
3691        );
3692        let mut body = json!({
3693            "lease_owner": lease_owner,
3694            "workflow_task_attempt": workflow_task_attempt,
3695            "commands": commands
3696        });
3697        if !message_stream_cursors.is_empty() {
3698            body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3699        }
3700        if !message_stream_waits.is_empty() {
3701            body["message_stream_waits"] = Value::Array(message_stream_waits);
3702        }
3703        let path = format!("/worker/workflow-tasks/{task_id}/complete");
3704        self.request_json(
3705            reqwest::Method::POST,
3706            &path,
3707            RequestProtocol::Worker(protocol_version),
3708            Some(&body),
3709        )
3710        .await
3711    }
3712
3713    pub async fn fail_workflow_task(
3714        &self,
3715        task_id: &str,
3716        lease_owner: &str,
3717        workflow_task_attempt: u64,
3718        message: impl Into<String>,
3719    ) -> Result<Value> {
3720        self.fail_workflow_task_with_type(
3721            task_id,
3722            lease_owner,
3723            workflow_task_attempt,
3724            message,
3725            "RustWorkflowTaskFailure",
3726        )
3727        .await
3728    }
3729
3730    async fn fail_workflow_task_with_type(
3731        &self,
3732        task_id: &str,
3733        lease_owner: &str,
3734        workflow_task_attempt: u64,
3735        message: impl Into<String>,
3736        failure_type: &str,
3737    ) -> Result<Value> {
3738        let body = json!({
3739            "lease_owner": lease_owner,
3740            "workflow_task_attempt": workflow_task_attempt,
3741            "failure": {
3742                "message": message.into(),
3743                "type": failure_type
3744            }
3745        });
3746        let path = format!("/worker/workflow-tasks/{task_id}/fail");
3747        self.request_json(
3748            reqwest::Method::POST,
3749            &path,
3750            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3751            Some(&body),
3752        )
3753        .await
3754    }
3755
3756    pub async fn poll_activity_task(
3757        &self,
3758        worker_id: &str,
3759        task_queue: &str,
3760        timeout: Duration,
3761    ) -> Result<Option<ActivityTask>> {
3762        Ok(self
3763            .poll_activity_task_response(worker_id, task_queue, timeout)
3764            .await?
3765            .task)
3766    }
3767
3768    /// Poll an activity task while preserving server stop and drain metadata.
3769    pub async fn poll_activity_task_response(
3770        &self,
3771        worker_id: &str,
3772        task_queue: &str,
3773        timeout: Duration,
3774    ) -> Result<PollActivityTaskResponse> {
3775        let poll_request_id = unique_request_id("rust-activity-poll");
3776        self.poll_activity_task_response_with_request_id(
3777            worker_id,
3778            task_queue,
3779            timeout,
3780            &poll_request_id,
3781            1,
3782        )
3783        .await
3784    }
3785
3786    async fn poll_activity_task_response_with_request_id(
3787        &self,
3788        worker_id: &str,
3789        task_queue: &str,
3790        timeout: Duration,
3791        poll_request_id: &str,
3792        transport_retries: usize,
3793    ) -> Result<PollActivityTaskResponse> {
3794        let body = json!({
3795            "worker_id": worker_id,
3796            "task_queue": task_queue,
3797            "poll_request_id": poll_request_id,
3798            "timeout_seconds": long_poll_timeout_seconds(timeout),
3799        });
3800        let data: PollActivityTaskResponse = self
3801            .poll_request_json(
3802                "/worker/activity-tasks/poll",
3803                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3804                &body,
3805                timeout + Duration::from_secs(5),
3806                transport_retries,
3807            )
3808            .await?;
3809        Ok(data)
3810    }
3811
3812    pub async fn complete_activity_task<T: Serialize>(
3813        &self,
3814        task_id: &str,
3815        activity_attempt_id: &str,
3816        lease_owner: &str,
3817        result: T,
3818        codec: &str,
3819    ) -> Result<Value> {
3820        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3821        let body = json!({
3822            "activity_attempt_id": activity_attempt_id,
3823            "lease_owner": lease_owner,
3824            "result": result
3825        });
3826        let path = format!("/worker/activity-tasks/{task_id}/complete");
3827        activity_task_response(
3828            self.request_json(
3829                reqwest::Method::POST,
3830                &path,
3831                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3832                Some(&body),
3833            )
3834            .await,
3835            "complete",
3836            task_id,
3837            activity_attempt_id,
3838        )
3839    }
3840
3841    pub async fn fail_activity_task(
3842        &self,
3843        task_id: &str,
3844        activity_attempt_id: &str,
3845        lease_owner: &str,
3846        message: impl Into<String>,
3847        non_retryable: bool,
3848    ) -> Result<Value> {
3849        let body = json!({
3850            "activity_attempt_id": activity_attempt_id,
3851            "lease_owner": lease_owner,
3852            "failure": {
3853                "message": message.into(),
3854                "type": "RustActivityFailure",
3855                "non_retryable": non_retryable
3856            }
3857        });
3858        let path = format!("/worker/activity-tasks/{task_id}/fail");
3859        activity_task_response(
3860            self.request_json(
3861                reqwest::Method::POST,
3862                &path,
3863                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3864                Some(&body),
3865            )
3866            .await,
3867            "fail",
3868            task_id,
3869            activity_attempt_id,
3870        )
3871    }
3872
3873    pub async fn heartbeat_activity_task<T: Serialize>(
3874        &self,
3875        task_id: &str,
3876        activity_attempt_id: &str,
3877        lease_owner: &str,
3878        details: T,
3879    ) -> Result<ActivityHeartbeatResponse> {
3880        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3881        let body = json!({
3882            "activity_attempt_id": activity_attempt_id,
3883            "lease_owner": lease_owner,
3884            "details": details
3885        });
3886        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
3887        activity_task_response(
3888            self.request_json(
3889                reqwest::Method::POST,
3890                &path,
3891                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3892                Some(&body),
3893            )
3894            .await,
3895            "heartbeat",
3896            task_id,
3897            activity_attempt_id,
3898        )
3899    }
3900
3901    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
3902        &self,
3903        method: reqwest::Method,
3904        path: &str,
3905        protocol: RequestProtocol,
3906        body: Option<&B>,
3907    ) -> Result<T> {
3908        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
3909            .await
3910    }
3911
3912    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
3913        &self,
3914        method: reqwest::Method,
3915        path: &str,
3916        protocol: RequestProtocol,
3917        body: Option<&B>,
3918        timeout: Duration,
3919    ) -> Result<T> {
3920        let auth_token = self.auth_token(protocol)?;
3921        let mut request = self
3922            .http
3923            .request(method, format!("{}/api{}", self.base_url, path))
3924            .timeout(timeout)
3925            .header(reqwest::header::ACCEPT, "application/json")
3926            .header(reqwest::header::CONTENT_TYPE, "application/json")
3927            .header("X-Namespace", &self.namespace);
3928
3929        match protocol {
3930            RequestProtocol::Worker(version) => {
3931                request = request.header("X-Durable-Workflow-Protocol-Version", version);
3932            }
3933            RequestProtocol::ControlPlane => {
3934                request = request.header(
3935                    "X-Durable-Workflow-Control-Plane-Version",
3936                    CONTROL_PLANE_VERSION,
3937                );
3938            }
3939        }
3940
3941        if let Some(token) = auth_token {
3942            request = request.bearer_auth(token);
3943        }
3944
3945        if let Some(body) = body {
3946            request = request.json(body);
3947        }
3948
3949        let response = request.send().await?;
3950        let status = response.status();
3951        let bytes = response.bytes().await?;
3952
3953        if !status.is_success() {
3954            let body = String::from_utf8_lossy(&bytes).to_string();
3955            if let Some(protocol) = protocol_failure(status, &body) {
3956                return Err(Error::Protocol(protocol));
3957            }
3958            return Err(Error::Http { status, body });
3959        }
3960
3961        if bytes.is_empty() {
3962            return Ok(serde_json::from_value(Value::Null)?);
3963        }
3964
3965        Ok(serde_json::from_slice(&bytes)?)
3966    }
3967
3968    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
3969        &self,
3970        path: &str,
3971        protocol: RequestProtocol,
3972        body: &B,
3973        timeout: Duration,
3974        max_retries: usize,
3975    ) -> Result<T> {
3976        let mut retries = 0;
3977
3978        loop {
3979            let response = self
3980                .request_json_with_timeout(
3981                    reqwest::Method::POST,
3982                    path,
3983                    protocol,
3984                    Some(body),
3985                    timeout,
3986                )
3987                .await;
3988
3989            match response {
3990                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
3991                response => return worker_poll_response(response),
3992            }
3993        }
3994    }
3995
3996    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
3997        match protocol {
3998            RequestProtocol::Worker(_) => {
3999                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4000                    return Ok(Some(token));
4001                }
4002                if self.control_token.is_some() {
4003                    return Err(Error::MissingRoleCredentials {
4004                        role: "worker",
4005                        opposite_role: "control",
4006                    });
4007                }
4008                Ok(None)
4009            }
4010            RequestProtocol::ControlPlane => {
4011                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4012                    return Ok(Some(token));
4013                }
4014                if self.worker_token.is_some() {
4015                    return Err(Error::MissingRoleCredentials {
4016                        role: "control",
4017                        opposite_role: "worker",
4018                    });
4019                }
4020                Ok(None)
4021            }
4022        }
4023    }
4024}
4025
4026fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4027    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4028    let reason = body
4029        .get("reason")
4030        .and_then(Value::as_str)
4031        .unwrap_or("query_rejected")
4032        .to_string();
4033    let message = body
4034        .get("message")
4035        .or_else(|| body.get("error"))
4036        .and_then(Value::as_str)
4037        .unwrap_or("workflow query was rejected")
4038        .to_string();
4039
4040    QueryFailure {
4041        status: status.as_u16(),
4042        reason,
4043        message,
4044        body,
4045    }
4046}
4047
4048fn workflow_command_result(
4049    command: WorkflowCommandKind,
4050    data: Value,
4051    workflow_id: &str,
4052    run_id: Option<&str>,
4053) -> WorkflowCommandResult {
4054    WorkflowCommandResult {
4055        command,
4056        workflow_id: data
4057            .get("workflow_id")
4058            .and_then(Value::as_str)
4059            .unwrap_or(workflow_id)
4060            .to_string(),
4061        run_id: data
4062            .get("run_id")
4063            .and_then(Value::as_str)
4064            .or(run_id)
4065            .map(str::to_string),
4066        outcome: data
4067            .get("outcome")
4068            .and_then(Value::as_str)
4069            .map(str::to_string),
4070        reason: data
4071            .get("reason")
4072            .and_then(Value::as_str)
4073            .map(str::to_string),
4074        command_status: data
4075            .get("command_status")
4076            .and_then(Value::as_str)
4077            .map(str::to_string),
4078        raw: data,
4079    }
4080}
4081
4082fn workflow_command_rejection(
4083    command: WorkflowCommandKind,
4084    status: reqwest::StatusCode,
4085    raw_body: String,
4086    workflow_id: &str,
4087    run_id: Option<&str>,
4088) -> WorkflowCommandRejection {
4089    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4090    WorkflowCommandRejection {
4091        command,
4092        status: status.as_u16(),
4093        reason: body
4094            .get("reason")
4095            .and_then(Value::as_str)
4096            .unwrap_or("workflow_command_rejected")
4097            .to_string(),
4098        message: body
4099            .get("message")
4100            .or_else(|| body.get("error"))
4101            .and_then(Value::as_str)
4102            .unwrap_or("workflow lifecycle command was rejected")
4103            .to_string(),
4104        workflow_id: body
4105            .get("workflow_id")
4106            .and_then(Value::as_str)
4107            .unwrap_or(workflow_id)
4108            .to_string(),
4109        run_id: body
4110            .get("run_id")
4111            .and_then(Value::as_str)
4112            .or(run_id)
4113            .map(str::to_string),
4114        target_scope: body
4115            .get("target_scope")
4116            .and_then(Value::as_str)
4117            .map(str::to_string),
4118        body,
4119    }
4120}
4121
4122fn query_task_response(response: Result<Value>) -> Result<Value> {
4123    match response {
4124        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4125        response => response,
4126    }
4127}
4128
4129fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4130    match response {
4131        Err(Error::Http { status, body })
4132            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4133        {
4134            Ok(serde_json::from_str(&body)?)
4135        }
4136        response => response,
4137    }
4138}
4139
4140fn worker_poll_body_is_stop(body: &str) -> bool {
4141    serde_json::from_str::<Value>(body)
4142        .ok()
4143        .is_some_and(|body| {
4144            worker_poll_is_stop(
4145                body.get("poll_status").and_then(Value::as_str),
4146                body.get("reason").and_then(Value::as_str),
4147            )
4148        })
4149}
4150
4151fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4152    matches!(poll_status, Some("draining" | "stopped"))
4153        || matches!(reason, Some("worker_draining" | "worker_stopped"))
4154}
4155
4156fn query_task_rejection_is_final(error: &Error) -> bool {
4157    matches!(
4158        error,
4159        Error::QueryFailed(failure)
4160            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4161    )
4162}
4163
4164fn activity_task_response<T>(
4165    response: Result<T>,
4166    operation: &str,
4167    task_id: &str,
4168    activity_attempt_id: &str,
4169) -> Result<T> {
4170    match response {
4171        Err(Error::Http { status, body }) => {
4172            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4173            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4174                operation: operation.to_string(),
4175                status: status.as_u16(),
4176                reason: body
4177                    .get("reason")
4178                    .and_then(Value::as_str)
4179                    .unwrap_or("activity_task_rejected")
4180                    .to_string(),
4181                task_id: body
4182                    .get("task_id")
4183                    .and_then(Value::as_str)
4184                    .unwrap_or(task_id)
4185                    .to_string(),
4186                activity_attempt_id: body
4187                    .get("activity_attempt_id")
4188                    .and_then(Value::as_str)
4189                    .unwrap_or(activity_attempt_id)
4190                    .to_string(),
4191                cancel_requested: body
4192                    .get("cancel_requested")
4193                    .and_then(Value::as_bool)
4194                    .unwrap_or(false),
4195                can_continue: body.get("can_continue").and_then(Value::as_bool),
4196                run_closed_reason: body
4197                    .get("run_closed_reason")
4198                    .and_then(Value::as_str)
4199                    .map(str::to_string),
4200                body,
4201            }))
4202        }
4203        response => response,
4204    }
4205}
4206
4207fn activity_task_rejection_is_final(error: &Error) -> bool {
4208    matches!(
4209        error,
4210        Error::ActivityTaskRejected(rejection)
4211            if matches!(
4212                rejection.reason.as_str(),
4213                "run_cancelled"
4214                    | "run_terminated"
4215                    | "attempt_closed"
4216                    | "stale_attempt"
4217                    | "activity_cancelled"
4218                    | "task_cancelled"
4219                    | "run_closed"
4220                    | "activity_not_running"
4221                    | "attempt_not_found"
4222            )
4223    )
4224}
4225
4226fn workflow_task_completion_is_terminal_timeout(
4227    error: &Error,
4228    task_id: &str,
4229    workflow_task_attempt: u64,
4230    run_id: Option<&str>,
4231) -> bool {
4232    let Error::Http { status, body } = error else {
4233        return false;
4234    };
4235    if *status != reqwest::StatusCode::CONFLICT {
4236        return false;
4237    }
4238
4239    let Some(run_id) = run_id else {
4240        return false;
4241    };
4242    let Ok(body) = serde_json::from_str::<Value>(body) else {
4243        return false;
4244    };
4245
4246    body.get("recorded").and_then(Value::as_bool) == Some(false)
4247        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4248        && body.get("run_status").and_then(Value::as_str) == Some("failed")
4249        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4250        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4251        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4252}
4253
4254fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4255    let body: Value = serde_json::from_str(raw_body).ok()?;
4256    let reason = body.get("reason")?.as_str()?;
4257    if !matches!(
4258        reason,
4259        "missing_protocol_version"
4260            | "unsupported_protocol_version"
4261            | "missing_control_plane_version"
4262            | "unsupported_control_plane_version"
4263    ) {
4264        return None;
4265    }
4266
4267    Some(ProtocolFailure {
4268        status: status.as_u16(),
4269        reason: reason.to_string(),
4270        message: body
4271            .get("message")
4272            .or_else(|| body.get("error"))
4273            .and_then(Value::as_str)
4274            .unwrap_or("protocol version rejected")
4275            .to_string(),
4276        supported_version: body
4277            .get("supported_version")
4278            .and_then(Value::as_str)
4279            .map(str::to_string),
4280        requested_version: body
4281            .get("requested_version")
4282            .and_then(Value::as_str)
4283            .map(str::to_string),
4284        body,
4285    })
4286}
4287
4288fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4289    timeout
4290        .as_secs()
4291        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4292        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4293}
4294
4295fn worker_operation_is_retryable(error: &Error) -> bool {
4296    match error {
4297        Error::Transport(error) => {
4298            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4299        }
4300        Error::Http { status, .. } => {
4301            matches!(
4302                *status,
4303                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4304            ) || status.is_server_error()
4305        }
4306        _ => false,
4307    }
4308}
4309
4310fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4311    let exponent = retry.saturating_sub(1).min(31) as u32;
4312    policy
4313        .initial_backoff
4314        .saturating_mul(1_u32 << exponent)
4315        .min(policy.max_backoff)
4316}
4317
4318#[derive(Debug)]
4319pub struct ClientBuilder {
4320    base_url: String,
4321    token: Option<String>,
4322    control_token: Option<String>,
4323    worker_token: Option<String>,
4324    namespace: String,
4325    timeout: Duration,
4326}
4327
4328impl ClientBuilder {
4329    pub fn token(mut self, token: Option<String>) -> Self {
4330        self.token = token;
4331        self
4332    }
4333
4334    pub fn control_token(mut self, token: Option<String>) -> Self {
4335        self.control_token = token;
4336        self
4337    }
4338
4339    pub fn worker_token(mut self, token: Option<String>) -> Self {
4340        self.worker_token = token;
4341        self
4342    }
4343
4344    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4345        self.namespace = namespace.into();
4346        self
4347    }
4348
4349    pub fn timeout(mut self, timeout: Duration) -> Self {
4350        self.timeout = timeout;
4351        self
4352    }
4353
4354    pub fn build(self) -> Result<Client> {
4355        let base_url = self.base_url.trim_end_matches('/').to_string();
4356        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4357            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4358            .unwrap_or_else(|_| base_url.ends_with("/api"));
4359
4360        if has_sdk_api_suffix {
4361            return Err(Error::InvalidBaseUrl);
4362        }
4363
4364        Ok(Client {
4365            http: reqwest::Client::builder().timeout(self.timeout).build()?,
4366            base_url,
4367            token: self.token,
4368            control_token: self.control_token,
4369            worker_token: self.worker_token,
4370            namespace: self.namespace,
4371        })
4372    }
4373}
4374
4375#[derive(Clone, Debug)]
4376pub struct WorkflowHandle {
4377    client: Client,
4378    pub workflow_id: String,
4379    pub run_id: Option<String>,
4380    pub workflow_type: String,
4381}
4382
4383impl WorkflowHandle {
4384    /// Describe whichever run is current for this stable workflow instance.
4385    pub async fn describe(&self) -> Result<WorkflowDescription> {
4386        self.client.describe_workflow(&self.workflow_id).await
4387    }
4388
4389    /// Describe the run identity originally selected by this handle.
4390    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4391        let run_id = self.run_id.as_deref().ok_or_else(|| {
4392            Error::Codec("run_id is required for selected-run description".to_string())
4393        })?;
4394        self.client
4395            .describe_workflow_run(&self.workflow_id, run_id)
4396            .await
4397    }
4398
4399    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4400        self.client
4401            .signal_workflow(&self.workflow_id, signal_name, input)
4402            .await
4403    }
4404
4405    pub async fn append_message<T: Serialize>(
4406        &self,
4407        stream_name: &str,
4408        message_id: &str,
4409        input: T,
4410    ) -> Result<Value> {
4411        self.client
4412            .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4413            .await
4414    }
4415
4416    /// Signal only if this handle's selected run is still current.
4417    pub async fn signal_selected_run<T: Serialize>(
4418        &self,
4419        signal_name: &str,
4420        input: T,
4421    ) -> Result<Value> {
4422        let run_id = self.run_id.as_deref().ok_or_else(|| {
4423            Error::Codec("run_id is required for selected-run signaling".to_string())
4424        })?;
4425        self.client
4426            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4427            .await
4428    }
4429
4430    /// Request cooperative cancellation of whichever run is current.
4431    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4432        self.client
4433            .cancel_workflow(&self.workflow_id, options)
4434            .await
4435    }
4436
4437    /// Request cancellation only if this handle's selected run is still current.
4438    pub async fn cancel_selected_run(
4439        &self,
4440        options: WorkflowCommandOptions,
4441    ) -> Result<WorkflowCommandResult> {
4442        let run_id = self.run_id.as_deref().ok_or_else(|| {
4443            Error::Codec("run_id is required for selected-run cancellation".to_string())
4444        })?;
4445        self.client
4446            .cancel_workflow_run(&self.workflow_id, run_id, options)
4447            .await
4448    }
4449
4450    /// Forcefully terminate whichever run is current.
4451    pub async fn terminate(
4452        &self,
4453        options: WorkflowCommandOptions,
4454    ) -> Result<WorkflowCommandResult> {
4455        self.client
4456            .terminate_workflow(&self.workflow_id, options)
4457            .await
4458    }
4459
4460    /// Terminate only if this handle's selected run is still current.
4461    pub async fn terminate_selected_run(
4462        &self,
4463        options: WorkflowCommandOptions,
4464    ) -> Result<WorkflowCommandResult> {
4465        let run_id = self.run_id.as_deref().ok_or_else(|| {
4466            Error::Codec("run_id is required for selected-run termination".to_string())
4467        })?;
4468        self.client
4469            .terminate_workflow_run(&self.workflow_id, run_id, options)
4470            .await
4471    }
4472
4473    /// Execute a named, read-only query against this workflow.
4474    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4475        self.client
4476            .query_workflow(&self.workflow_id, query_name, input)
4477            .await
4478    }
4479
4480    pub async fn query_avro_value<T: Serialize>(
4481        &self,
4482        query_name: &str,
4483        input: T,
4484    ) -> Result<AvroValue> {
4485        self.client
4486            .query_workflow_avro_value(&self.workflow_id, query_name, input)
4487            .await
4488    }
4489
4490    pub async fn update<T: Serialize>(
4491        &self,
4492        update_name: &str,
4493        input: T,
4494        request_id: Option<&str>,
4495    ) -> Result<Value> {
4496        self.client
4497            .update_workflow(&self.workflow_id, update_name, input, request_id)
4498            .await
4499    }
4500
4501    pub async fn update_avro_value<T: Serialize>(
4502        &self,
4503        update_name: &str,
4504        input: T,
4505        request_id: Option<&str>,
4506    ) -> Result<AvroValue> {
4507        self.client
4508            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4509            .await
4510    }
4511
4512    /// Query only if this handle's selected run is still current.
4513    pub async fn query_selected_run<T: Serialize>(
4514        &self,
4515        query_name: &str,
4516        input: T,
4517    ) -> Result<Value> {
4518        let run_id = self
4519            .run_id
4520            .as_deref()
4521            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4522        self.client
4523            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4524            .await
4525    }
4526
4527    /// Await the final terminal outcome of the current continue-as-new chain.
4528    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4529        self.result_target(options, None).await
4530    }
4531
4532    /// Await the final result without projecting Avro bytes through JSON.
4533    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4534        self.result_avro_value_target(options, None).await
4535    }
4536
4537    /// Await the final result and decode it into a Serde application type.
4538    pub async fn result_typed<T: DeserializeOwned>(
4539        &self,
4540        options: WorkflowResultOptions,
4541    ) -> Result<T> {
4542        let result = self.result_avro_value(options).await?;
4543        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4544    }
4545
4546    /// Await only the run identity originally selected by this handle.
4547    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4548        let run_id = self.run_id.as_deref().ok_or_else(|| {
4549            Error::Codec("run_id is required for selected-run result".to_string())
4550        })?;
4551        self.result_target(options, Some(run_id)).await
4552    }
4553
4554    /// Await the selected run's result on the lossless Avro Value surface.
4555    pub async fn result_selected_run_avro_value(
4556        &self,
4557        options: WorkflowResultOptions,
4558    ) -> Result<AvroValue> {
4559        let run_id = self.run_id.as_deref().ok_or_else(|| {
4560            Error::Codec("run_id is required for selected-run result".to_string())
4561        })?;
4562        self.result_avro_value_target(options, Some(run_id)).await
4563    }
4564
4565    /// Await the selected run and decode its result into a Serde type.
4566    pub async fn result_selected_run_typed<T: DeserializeOwned>(
4567        &self,
4568        options: WorkflowResultOptions,
4569    ) -> Result<T> {
4570        let result = self.result_selected_run_avro_value(options).await?;
4571        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4572    }
4573
4574    async fn result_avro_value_target(
4575        &self,
4576        options: WorkflowResultOptions,
4577        selected_run_id: Option<&str>,
4578    ) -> Result<AvroValue> {
4579        let started = Instant::now();
4580
4581        loop {
4582            let description = match selected_run_id {
4583                Some(run_id) => {
4584                    self.client
4585                        .describe_workflow_run(&self.workflow_id, run_id)
4586                        .await?
4587                }
4588                None => self.describe().await?,
4589            };
4590            if description.is_completed() {
4591                return description.output_avro_value.ok_or_else(|| {
4592                    Error::Codec(
4593                        "missing_payload_envelope: typed workflow result requires output_envelope"
4594                            .to_string(),
4595                    )
4596                });
4597            }
4598            if description.is_terminal() {
4599                let outcome =
4600                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4601                return Err(match outcome.kind {
4602                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4603                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4604                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4605                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4606                });
4607            }
4608            if started.elapsed() >= options.timeout {
4609                return Err(Error::Timeout);
4610            }
4611            tokio::time::sleep(options.poll_interval).await;
4612        }
4613    }
4614
4615    async fn result_target(
4616        &self,
4617        options: WorkflowResultOptions,
4618        selected_run_id: Option<&str>,
4619    ) -> Result<Value> {
4620        let started = Instant::now();
4621
4622        loop {
4623            let description = match selected_run_id {
4624                Some(run_id) => {
4625                    self.client
4626                        .describe_workflow_run(&self.workflow_id, run_id)
4627                        .await?
4628                }
4629                None => self.describe().await?,
4630            };
4631            if description.is_completed() {
4632                return Ok(description.output.unwrap_or(Value::Null));
4633            }
4634
4635            if description.is_terminal() {
4636                let outcome =
4637                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4638                return Err(match outcome.kind {
4639                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4640                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4641                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4642                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4643                });
4644            }
4645
4646            if started.elapsed() >= options.timeout {
4647                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4648                    kind: WorkflowTerminalKind::TimedOut,
4649                    workflow_id: description
4650                        .workflow_id
4651                        .clone()
4652                        .unwrap_or_else(|| self.workflow_id.clone()),
4653                    run_id: description
4654                        .run_id
4655                        .clone()
4656                        .or_else(|| selected_run_id.map(str::to_string)),
4657                    reason: "result_wait_timeout".to_string(),
4658                    failure_category: Some("client_timeout".to_string()),
4659                    failure_id: None,
4660                    exception_type: None,
4661                    exception_class: None,
4662                    non_retryable: None,
4663                    message: Some(format!(
4664                        "workflow result was not terminal within {:?}",
4665                        options.timeout
4666                    )),
4667                    exception: None,
4668                    raw: description.raw_value(),
4669                }));
4670            }
4671
4672            tokio::time::sleep(options.poll_interval).await;
4673        }
4674    }
4675}
4676
4677#[derive(Clone, Copy, Debug)]
4678pub struct WorkflowResultOptions {
4679    pub poll_interval: Duration,
4680    pub timeout: Duration,
4681}
4682
4683impl Default for WorkflowResultOptions {
4684    fn default() -> Self {
4685        Self {
4686            poll_interval: Duration::from_millis(500),
4687            timeout: Duration::from_secs(30),
4688        }
4689    }
4690}
4691
4692#[derive(Clone, Debug, Deserialize)]
4693pub struct WorkflowDescription {
4694    pub workflow_id: Option<String>,
4695    pub run_id: Option<String>,
4696    pub workflow_type: Option<String>,
4697    pub status: Option<String>,
4698    #[serde(default)]
4699    pub closed_reason: Option<String>,
4700    #[serde(default)]
4701    pub error: Option<String>,
4702    #[serde(default)]
4703    pub failure: Option<Value>,
4704    #[serde(default)]
4705    pub exception: Option<Value>,
4706    #[serde(default)]
4707    pub failures: Vec<Value>,
4708    #[serde(default)]
4709    pub output: Option<Value>,
4710    #[serde(default)]
4711    pub output_envelope: Option<Value>,
4712    #[serde(skip)]
4713    pub output_avro_value: Option<AvroValue>,
4714    #[serde(flatten)]
4715    pub raw: HashMap<String, Value>,
4716}
4717
4718/// Lifecycle and backlog metadata for one run-scoped Workflow Stream.
4719#[derive(Clone, Debug, Deserialize)]
4720pub struct WorkflowStreamDescription {
4721    pub stream_name: String,
4722    pub status: String,
4723    pub last_offset: i64,
4724    pub total_items: u64,
4725    pub pending_items: u64,
4726    #[serde(default)]
4727    pub opened_at: Option<String>,
4728    #[serde(default)]
4729    pub last_appended_at: Option<String>,
4730    #[serde(default)]
4731    pub closed_at: Option<String>,
4732    #[serde(default)]
4733    pub error_reason: Option<String>,
4734    #[serde(default)]
4735    pub retention_seconds: Option<u64>,
4736    #[serde(flatten)]
4737    pub raw: HashMap<String, Value>,
4738}
4739
4740impl WorkflowStreamDescription {
4741    pub fn is_terminal(&self) -> bool {
4742        matches!(self.status.as_str(), "closed" | "errored")
4743    }
4744}
4745
4746/// One item for direct or replay-safe append.
4747#[derive(Clone, Debug, Default)]
4748pub struct WorkflowStreamAppendItem {
4749    pub payload_envelope: Option<Value>,
4750    pub payload_reference: Option<String>,
4751    pub item_type: Option<String>,
4752    pub content_type: Option<String>,
4753    pub idempotency_key: Option<String>,
4754}
4755
4756impl WorkflowStreamAppendItem {
4757    /// Encode an inline payload with the SDK's fixed Avro Value envelope.
4758    pub fn new<T: Serialize>(payload: T) -> Result<Self> {
4759        let value = AvroValue::from_serialize(&payload)?;
4760        Ok(Self {
4761            payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
4762            ..Self::default()
4763        })
4764    }
4765
4766    /// Preserve an external payload URI as an opaque service-contract reference.
4767    pub fn from_reference(reference: impl Into<String>) -> Self {
4768        Self {
4769            payload_reference: Some(reference.into()),
4770            ..Self::default()
4771        }
4772    }
4773
4774    pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
4775        self.item_type = Some(item_type.into());
4776        self
4777    }
4778
4779    pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
4780        self.content_type = Some(content_type.into());
4781        self
4782    }
4783
4784    pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
4785        self.idempotency_key = Some(idempotency_key.into());
4786        self
4787    }
4788
4789    fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
4790        let mut item = serde_json::Map::new();
4791        if let Some(payload) = &self.payload_envelope {
4792            item.insert("payload".to_string(), payload.clone());
4793            item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
4794        }
4795        if let Some(reference) = &self.payload_reference {
4796            item.insert("payload_reference".to_string(), json!(reference));
4797        }
4798        if let Some(item_type) = &self.item_type {
4799            item.insert("item_type".to_string(), json!(item_type));
4800        }
4801        if let Some(content_type) = &self.content_type {
4802            item.insert("content_type".to_string(), json!(content_type));
4803        }
4804        if let Some(key) = derived_idempotency_key
4805            .as_ref()
4806            .or(self.idempotency_key.as_ref())
4807        {
4808            item.insert("idempotency_key".to_string(), json!(key));
4809        }
4810        Value::Object(item)
4811    }
4812}
4813
4814/// One durable item at its stable zero-based offset.
4815#[derive(Clone, Debug)]
4816pub struct WorkflowStreamItem {
4817    pub offset: u64,
4818    pub payload: Option<Value>,
4819    pub payload_envelope: Option<Value>,
4820    pub payload_reference: Option<String>,
4821    pub payload_codec: Option<String>,
4822    pub idempotency_key: Option<String>,
4823    pub item_type: Option<String>,
4824    pub content_type: Option<String>,
4825    pub origin: Option<String>,
4826    pub origin_reference: Option<String>,
4827    pub emitted_at: Option<String>,
4828    pub raw: Value,
4829}
4830
4831/// One bounded at-least-once subscription page.
4832#[derive(Clone, Debug)]
4833pub struct WorkflowStreamPage {
4834    pub stream: WorkflowStreamDescription,
4835    pub items: Vec<WorkflowStreamItem>,
4836    pub next_offset: u64,
4837    pub terminal: bool,
4838}
4839
4840/// Durable acceptance and deduplication outcome for an append request.
4841#[derive(Clone, Debug)]
4842pub struct WorkflowStreamAppendResult {
4843    pub stream: WorkflowStreamDescription,
4844    pub accepted_offsets: Vec<u64>,
4845    pub accepted: u64,
4846    pub deduped: u64,
4847}
4848
4849#[derive(Deserialize)]
4850struct WorkflowStreamListResponse {
4851    #[serde(default)]
4852    streams: Vec<WorkflowStreamDescription>,
4853}
4854
4855#[derive(Deserialize)]
4856struct WorkflowStreamDescriptionResponse {
4857    stream: WorkflowStreamDescription,
4858}
4859
4860#[derive(Deserialize)]
4861struct WorkflowStreamPageResponse {
4862    stream: WorkflowStreamDescription,
4863    #[serde(default)]
4864    items: Vec<Value>,
4865    next_offset: u64,
4866    terminal: bool,
4867}
4868
4869#[derive(Deserialize)]
4870struct WorkflowStreamAppendResponse {
4871    stream: WorkflowStreamDescription,
4872    #[serde(default)]
4873    accepted_offsets: Vec<u64>,
4874    accepted: u64,
4875    deduped: u64,
4876}
4877
4878impl WorkflowDescription {
4879    pub fn is_completed(&self) -> bool {
4880        matches!(self.status.as_deref(), Some("completed" | "Completed"))
4881    }
4882
4883    pub fn is_terminal(&self) -> bool {
4884        matches!(
4885            self.status.as_deref(),
4886            Some(
4887                "completed"
4888                    | "Completed"
4889                    | "failed"
4890                    | "Failed"
4891                    | "cancelled"
4892                    | "Cancelled"
4893                    | "terminated"
4894                    | "Terminated"
4895                    | "timed_out"
4896                    | "TimedOut",
4897            )
4898        )
4899    }
4900
4901    fn decode_payloads(&mut self) -> Result<()> {
4902        if let Some(envelope) = &self.output_envelope {
4903            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
4904            self.output = Some(value.clone().into_json()?);
4905            self.output_avro_value = Some(value);
4906        }
4907
4908        Ok(())
4909    }
4910
4911    fn raw_value(&self) -> Value {
4912        let mut data = self.raw.clone();
4913        data.insert(
4914            "workflow_id".to_string(),
4915            self.workflow_id
4916                .clone()
4917                .map(Value::String)
4918                .unwrap_or(Value::Null),
4919        );
4920        data.insert(
4921            "run_id".to_string(),
4922            self.run_id
4923                .clone()
4924                .map(Value::String)
4925                .unwrap_or(Value::Null),
4926        );
4927        data.insert(
4928            "workflow_type".to_string(),
4929            self.workflow_type
4930                .clone()
4931                .map(Value::String)
4932                .unwrap_or(Value::Null),
4933        );
4934        data.insert(
4935            "status".to_string(),
4936            self.status
4937                .clone()
4938                .map(Value::String)
4939                .unwrap_or(Value::Null),
4940        );
4941        data.insert(
4942            "closed_reason".to_string(),
4943            self.closed_reason
4944                .clone()
4945                .map(Value::String)
4946                .unwrap_or(Value::Null),
4947        );
4948        if let Some(failure) = &self.failure {
4949            data.insert("failure".to_string(), failure.clone());
4950        }
4951        if let Some(exception) = &self.exception {
4952            data.insert("exception".to_string(), exception.clone());
4953        }
4954        Value::Object(data.into_iter().collect())
4955    }
4956}
4957
4958fn workflow_terminal_outcome(
4959    description: &WorkflowDescription,
4960    workflow_id: &str,
4961    run_id: Option<&str>,
4962) -> WorkflowTerminalOutcome {
4963    let terminal_kind = description
4964        .closed_reason
4965        .as_deref()
4966        .or(description.status.as_deref())
4967        .unwrap_or("failed")
4968        .to_ascii_lowercase();
4969    let kind = match terminal_kind.as_str() {
4970        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
4971        "terminated" => WorkflowTerminalKind::Terminated,
4972        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
4973        _ => WorkflowTerminalKind::Failed,
4974    };
4975    let default_reason = match kind {
4976        WorkflowTerminalKind::Failed => "workflow_failed",
4977        WorkflowTerminalKind::Cancelled => "cancelled",
4978        WorkflowTerminalKind::Terminated => "terminated",
4979        WorkflowTerminalKind::TimedOut => "timed_out",
4980    };
4981    let failure = description
4982        .failure
4983        .as_ref()
4984        .filter(|value| value.is_object());
4985    let nested_failure = failure
4986        .and_then(|value| value.get("failures"))
4987        .and_then(Value::as_array)
4988        .and_then(|failures| failures.last())
4989        .or_else(|| description.failures.last());
4990    let exception = description
4991        .exception
4992        .clone()
4993        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
4994        .or_else(|| {
4995            nested_failure
4996                .and_then(|value| value.get("exception_payload"))
4997                .cloned()
4998        });
4999    let string_field = |name: &str| {
5000        failure
5001            .and_then(|value| value.get(name))
5002            .and_then(Value::as_str)
5003            .or_else(|| {
5004                nested_failure
5005                    .and_then(|value| value.get(name))
5006                    .and_then(Value::as_str)
5007            })
5008            .map(str::to_string)
5009    };
5010    let exception_field = |name: &str| {
5011        exception
5012            .as_ref()
5013            .and_then(|value| value.get(name))
5014            .and_then(Value::as_str)
5015            .map(str::to_string)
5016    };
5017    let message = description
5018        .error
5019        .clone()
5020        .or_else(|| string_field("message"))
5021        .or_else(|| exception_field("message"));
5022    let reason = description
5023        .raw
5024        .get("reason")
5025        .and_then(Value::as_str)
5026        .map(str::to_string)
5027        .or_else(|| {
5028            failure
5029                .and_then(|value| value.get("reason"))
5030                .and_then(Value::as_str)
5031                .map(str::to_string)
5032        })
5033        .or_else(|| description.closed_reason.clone())
5034        .unwrap_or_else(|| default_reason.to_string());
5035    let failure_id = string_field("failure_id").or_else(|| {
5036        nested_failure
5037            .and_then(|value| value.get("id"))
5038            .and_then(Value::as_str)
5039            .map(str::to_string)
5040    });
5041
5042    WorkflowTerminalOutcome {
5043        kind,
5044        workflow_id: description
5045            .workflow_id
5046            .clone()
5047            .unwrap_or_else(|| workflow_id.to_string()),
5048        run_id: description
5049            .run_id
5050            .clone()
5051            .or_else(|| run_id.map(str::to_string)),
5052        reason,
5053        failure_category: string_field("failure_category")
5054            .or_else(|| Some(default_reason.to_string())),
5055        failure_id,
5056        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5057        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5058        non_retryable: failure
5059            .and_then(|value| value.get("non_retryable"))
5060            .and_then(Value::as_bool)
5061            .or_else(|| {
5062                nested_failure
5063                    .and_then(|value| value.get("non_retryable"))
5064                    .and_then(Value::as_bool)
5065            }),
5066        message,
5067        exception,
5068        raw: description.raw_value(),
5069    }
5070}
5071
5072#[derive(Clone, Debug, Deserialize)]
5073pub struct RegisterWorkerResponse {
5074    pub worker_id: String,
5075    pub registered: bool,
5076    #[serde(default)]
5077    pub heartbeat_interval_seconds: Option<u64>,
5078    #[serde(default)]
5079    pub protocol_version: Option<String>,
5080    #[serde(default)]
5081    pub server_capabilities: Option<Value>,
5082}
5083
5084/// Result of gracefully removing a worker-plane registration.
5085#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5086pub struct WorkerDeregistrationEnvelope {
5087    pub worker_id: String,
5088    pub outcome: String,
5089    pub recovered_workflow_task_count: u64,
5090}
5091
5092#[derive(Clone, Debug, Deserialize)]
5093pub struct PollWorkflowTaskResponse {
5094    #[serde(default)]
5095    pub task: Option<WorkflowTask>,
5096    #[serde(default)]
5097    pub poll_status: Option<String>,
5098    #[serde(default)]
5099    pub reason: Option<String>,
5100    #[serde(default)]
5101    pub protocol_version: Option<String>,
5102    #[serde(default)]
5103    pub server_capabilities: Option<Value>,
5104}
5105
5106impl PollWorkflowTaskResponse {
5107    /// Classify this response without parsing server display text.
5108    pub fn outcome(&self) -> WorkerPollOutcome {
5109        worker_poll_outcome(
5110            self.task.is_some(),
5111            self.poll_status.as_deref(),
5112            self.reason.as_deref(),
5113        )
5114    }
5115}
5116
5117fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5118    let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5119        return false;
5120    };
5121    let supported = capabilities
5122        .get("workflow_memo_updates")
5123        .and_then(Value::as_object)
5124        .and_then(|memo| memo.get("supported"))
5125        .and_then(Value::as_bool)
5126        == Some(true);
5127    let command_advertised = capabilities
5128        .get("supported_workflow_task_commands")
5129        .and_then(Value::as_array)
5130        .is_some_and(|commands| {
5131            commands
5132                .iter()
5133                .any(|command| command.as_str() == Some("upsert_memo"))
5134        });
5135    supported && command_advertised
5136}
5137
5138fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5139    commands
5140        .iter()
5141        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5142}
5143
5144#[derive(Clone, Debug, Deserialize)]
5145pub struct PollActivityTaskResponse {
5146    #[serde(default)]
5147    pub task: Option<ActivityTask>,
5148    #[serde(default)]
5149    pub poll_status: Option<String>,
5150    #[serde(default)]
5151    pub reason: Option<String>,
5152}
5153
5154impl PollActivityTaskResponse {
5155    /// Classify this response without parsing server display text.
5156    pub fn outcome(&self) -> WorkerPollOutcome {
5157        worker_poll_outcome(
5158            self.task.is_some(),
5159            self.poll_status.as_deref(),
5160            self.reason.as_deref(),
5161        )
5162    }
5163}
5164
5165#[derive(Clone, Debug, Deserialize)]
5166pub struct PollQueryTaskResponse {
5167    #[serde(default)]
5168    pub task: Option<QueryTask>,
5169    #[serde(default)]
5170    pub poll_status: Option<String>,
5171    #[serde(default)]
5172    pub reason: Option<String>,
5173}
5174
5175impl PollQueryTaskResponse {
5176    /// Classify this response without parsing server display text.
5177    pub fn outcome(&self) -> WorkerPollOutcome {
5178        worker_poll_outcome(
5179            self.task.is_some(),
5180            self.poll_status.as_deref(),
5181            self.reason.as_deref(),
5182        )
5183    }
5184}
5185
5186/// Stable classification for worker poll responses.
5187#[derive(Clone, Debug, PartialEq, Eq)]
5188pub enum WorkerPollOutcome {
5189    /// A task was leased and is available on the response.
5190    Task,
5191    /// No task was leased, but the worker should continue polling.
5192    Idle {
5193        poll_status: Option<String>,
5194        reason: Option<String>,
5195    },
5196    /// The server asked this worker to stop claiming new work.
5197    Stop {
5198        poll_status: Option<String>,
5199        reason: Option<String>,
5200    },
5201}
5202
5203impl WorkerPollOutcome {
5204    pub fn should_stop(&self) -> bool {
5205        matches!(self, Self::Stop { .. })
5206    }
5207}
5208
5209fn worker_poll_outcome(
5210    has_task: bool,
5211    poll_status: Option<&str>,
5212    reason: Option<&str>,
5213) -> WorkerPollOutcome {
5214    if worker_poll_is_stop(poll_status, reason) {
5215        return WorkerPollOutcome::Stop {
5216            poll_status: poll_status.map(str::to_string),
5217            reason: reason.map(str::to_string),
5218        };
5219    }
5220
5221    if has_task {
5222        WorkerPollOutcome::Task
5223    } else {
5224        WorkerPollOutcome::Idle {
5225            poll_status: poll_status.map(str::to_string),
5226            reason: reason.map(str::to_string),
5227        }
5228    }
5229}
5230
5231/// An ephemeral server-routed query task.
5232#[derive(Clone, Debug, Deserialize)]
5233pub struct QueryTask {
5234    pub query_task_id: String,
5235    #[serde(default = "default_workflow_task_attempt")]
5236    pub query_task_attempt: u64,
5237    #[serde(default)]
5238    pub lease_owner: Option<String>,
5239    #[serde(default)]
5240    pub workflow_id: Option<String>,
5241    #[serde(default)]
5242    pub run_id: Option<String>,
5243    pub workflow_type: String,
5244    pub query_name: String,
5245    #[serde(
5246        default = "missing_task_payload_codec",
5247        deserialize_with = "deserialize_task_payload_codec"
5248    )]
5249    pub payload_codec: String,
5250    #[serde(default)]
5251    pub workflow_arguments: Option<Value>,
5252    #[serde(default)]
5253    pub query_arguments: Option<Value>,
5254    #[serde(default)]
5255    pub history_events: Vec<HistoryEvent>,
5256    #[serde(default)]
5257    pub history_export: Option<Value>,
5258    #[serde(default)]
5259    pub run_status: Option<String>,
5260}
5261
5262#[derive(Clone, Debug, Deserialize)]
5263pub struct WorkflowTask {
5264    pub task_id: String,
5265    #[serde(default)]
5266    pub workflow_command_id: Option<String>,
5267    #[serde(default)]
5268    pub workflow_id: Option<String>,
5269    #[serde(default)]
5270    pub run_id: Option<String>,
5271    pub workflow_type: String,
5272    #[serde(default)]
5273    pub cancel_requested: bool,
5274    #[serde(
5275        default = "missing_task_payload_codec",
5276        deserialize_with = "deserialize_task_payload_codec"
5277    )]
5278    pub payload_codec: String,
5279    #[serde(default)]
5280    pub arguments: Option<Value>,
5281    #[serde(default)]
5282    pub history_events: Vec<HistoryEvent>,
5283    #[serde(default)]
5284    pub total_history_events: Option<u64>,
5285    #[serde(default)]
5286    pub history_size_bytes: Option<u64>,
5287    #[serde(default)]
5288    pub continue_as_new_recommended: Option<bool>,
5289    #[serde(default)]
5290    pub history_budget_pressure: Option<String>,
5291    #[serde(default)]
5292    pub next_history_page_token: Option<String>,
5293    #[serde(default = "default_workflow_task_attempt")]
5294    pub workflow_task_attempt: u64,
5295    #[serde(default)]
5296    pub workflow_signal_id: Option<String>,
5297    #[serde(default)]
5298    pub signal_name: Option<String>,
5299    #[serde(default)]
5300    pub signal_arguments: Option<Value>,
5301    #[serde(default)]
5302    pub workflow_update_id: Option<String>,
5303    #[serde(default)]
5304    pub update_name: Option<String>,
5305    #[serde(default)]
5306    pub lease_owner: Option<String>,
5307}
5308
5309impl WorkflowTask {
5310    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5311        self.history_events.extend(page.history_events);
5312
5313        if page.total_history_events.is_some() {
5314            self.total_history_events = page.total_history_events;
5315        }
5316
5317        self.next_history_page_token = page
5318            .next_history_page_token
5319            .filter(|token| !token.is_empty());
5320    }
5321}
5322
5323#[derive(Clone, Debug, Deserialize)]
5324struct WorkflowTaskHistoryPage {
5325    #[serde(default)]
5326    history_events: Vec<HistoryEvent>,
5327    #[serde(default)]
5328    total_history_events: Option<u64>,
5329    #[serde(default)]
5330    next_history_page_token: Option<String>,
5331}
5332
5333#[derive(Clone, Debug, Deserialize)]
5334pub struct ActivityTask {
5335    pub task_id: String,
5336    #[serde(default)]
5337    pub activity_attempt_id: Option<String>,
5338    #[serde(default)]
5339    pub attempt_id: Option<String>,
5340    pub activity_type: String,
5341    #[serde(
5342        default = "missing_task_payload_codec",
5343        deserialize_with = "deserialize_task_payload_codec"
5344    )]
5345    pub payload_codec: String,
5346    #[serde(default)]
5347    pub arguments: Option<Value>,
5348    #[serde(default = "default_attempt_number")]
5349    pub attempt_number: u64,
5350    #[serde(default)]
5351    pub lease_owner: Option<String>,
5352}
5353
5354#[derive(Clone, Debug, Deserialize)]
5355pub struct HistoryEvent {
5356    #[serde(alias = "type")]
5357    pub event_type: String,
5358    #[serde(default)]
5359    pub payload: Value,
5360    #[serde(flatten)]
5361    pub raw: HashMap<String, Value>,
5362}
5363
5364/// One decoded signal in the committed workflow-history snapshot.
5365#[derive(Clone, Debug, PartialEq)]
5366pub struct QuerySignal {
5367    pub id: Option<String>,
5368    pub name: String,
5369    pub arguments: Vec<Value>,
5370    avro_arguments: Vec<AvroValue>,
5371    pub workflow_sequence: Option<u64>,
5372}
5373
5374impl QuerySignal {
5375    /// Lossless fixed Avro Value arguments for this committed signal.
5376    pub fn arguments_avro_value(&self) -> &[AvroValue] {
5377        &self.avro_arguments
5378    }
5379}
5380
5381/// Immutable state supplied to a registered query handler.
5382///
5383/// This context intentionally exposes no activity, signal-wait, or command
5384/// APIs. Query handlers inspect committed history and return a value; query
5385/// completion does not append an event or advance deterministic execution.
5386#[derive(Clone, Debug)]
5387pub struct QueryContext {
5388    pub workflow_id: Option<String>,
5389    pub run_id: Option<String>,
5390    pub workflow_type: String,
5391    pub run_status: Option<String>,
5392    workflow_input: Value,
5393    workflow_input_avro_value: AvroValue,
5394    history_events: Arc<Vec<HistoryEvent>>,
5395    signal_events: Arc<Vec<QuerySignal>>,
5396}
5397
5398impl QueryContext {
5399    /// The normalized argument list used to start the workflow.
5400    pub fn workflow_input(&self) -> &Value {
5401        &self.workflow_input
5402    }
5403
5404    /// The lossless fixed Avro Value argument list used to start the workflow.
5405    pub fn workflow_input_avro_value(&self) -> &AvroValue {
5406        &self.workflow_input_avro_value
5407    }
5408
5409    /// The immutable committed history used for this query snapshot.
5410    pub fn history_events(&self) -> &[HistoryEvent] {
5411        self.history_events.as_slice()
5412    }
5413
5414    /// All decoded signals in committed workflow order.
5415    pub fn signal_events(&self) -> &[QuerySignal] {
5416        self.signal_events.as_slice()
5417    }
5418
5419    /// Decoded argument lists for each committed signal with `signal_name`.
5420    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5421        self.signal_events
5422            .iter()
5423            .filter(|signal| signal.name == signal_name)
5424            .map(|signal| signal.arguments.clone())
5425            .collect()
5426    }
5427
5428    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
5429    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5430        self.signal_events
5431            .iter()
5432            .filter(|signal| signal.name == signal_name)
5433            .map(|signal| signal.avro_arguments.clone())
5434            .collect()
5435    }
5436}
5437
5438#[derive(Clone, Debug, Deserialize)]
5439pub struct ActivityHeartbeatResponse {
5440    #[serde(default)]
5441    pub cancel_requested: bool,
5442    #[serde(default)]
5443    pub heartbeat_recorded: bool,
5444    #[serde(default)]
5445    pub can_continue: Option<bool>,
5446    #[serde(default)]
5447    pub reason: Option<String>,
5448    #[serde(default)]
5449    pub run_closed_reason: Option<String>,
5450    #[serde(default)]
5451    pub run_closed_at: Option<String>,
5452    #[serde(default)]
5453    pub lease_expires_at: Option<String>,
5454    #[serde(default)]
5455    pub last_heartbeat_at: Option<String>,
5456}
5457
5458impl ActivityHeartbeatResponse {
5459    /// Whether the activity should stop instead of attempting completion.
5460    pub fn should_stop(&self) -> bool {
5461        self.cancel_requested || self.can_continue == Some(false)
5462    }
5463}
5464
5465fn missing_task_payload_codec() -> String {
5466    MISSING_TASK_PAYLOAD_CODEC.to_string()
5467}
5468
5469fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5470where
5471    D: Deserializer<'de>,
5472{
5473    Ok(match Value::deserialize(deserializer)? {
5474        Value::String(codec) => codec,
5475        Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5476        _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5477    })
5478}
5479
5480fn default_workflow_task_attempt() -> u64 {
5481    1
5482}
5483
5484fn default_attempt_number() -> u64 {
5485    1
5486}
5487
5488type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5489type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5490type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5491type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5492type ReplayedWorkflowHandler =
5493    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5494type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5495type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5496type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5497type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5498type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5499type ReplayedQueryHandler = Arc<
5500    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5501        + Send
5502        + Sync,
5503>;
5504type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5505
5506struct ReplayedWorkflowInvocation {
5507    future: WorkflowFuture,
5508    snapshot: WorkflowStateSnapshot,
5509}
5510
5511#[derive(Clone)]
5512struct RegisteredWorkflow {
5513    execute: WorkflowHandler,
5514    replay: Option<ReplayedWorkflowHandler>,
5515    state_type: Option<TypeId>,
5516}
5517
5518#[derive(Debug)]
5519struct WorkflowTaskDecision {
5520    commands: Vec<Value>,
5521    message_stream_cursors: Vec<Value>,
5522    message_stream_waits: Vec<Value>,
5523}
5524
5525impl WorkflowTaskDecision {
5526    fn without_message_streams(commands: Vec<Value>) -> Self {
5527        Self {
5528            commands,
5529            message_stream_cursors: Vec::new(),
5530            message_stream_waits: Vec::new(),
5531        }
5532    }
5533}
5534
5535#[derive(Clone)]
5536enum RegisteredQuery {
5537    Snapshot(QueryHandler),
5538    Replayed {
5539        state_type: TypeId,
5540        handler: ReplayedQueryHandler,
5541    },
5542}
5543
5544#[derive(Clone, Debug)]
5545pub struct WorkerHeartbeatObservation {
5546    pub worker_id: String,
5547    pub task_queue: String,
5548    pub acknowledged_at_unix_millis: u64,
5549    pub acknowledgement: Value,
5550}
5551
5552/// Bounded retry policy for worker poll acquisition and worker heartbeats.
5553///
5554/// Expected empty long polls are normal successful responses. Transport
5555/// failures, HTTP 408/429 responses, and server errors are retried with capped
5556/// exponential backoff. Authentication, protocol, codec, and handler failures
5557/// are never retried by the worker.
5558#[derive(Clone, Copy, Debug)]
5559pub struct WorkerRetryPolicy {
5560    /// Number of retries after the initial request fails.
5561    pub max_retries: usize,
5562    /// Delay before the first retry.
5563    pub initial_backoff: Duration,
5564    /// Maximum delay between retries.
5565    pub max_backoff: Duration,
5566}
5567
5568impl Default for WorkerRetryPolicy {
5569    fn default() -> Self {
5570        Self {
5571            max_retries: 5,
5572            initial_backoff: Duration::from_millis(100),
5573            max_backoff: Duration::from_secs(5),
5574        }
5575    }
5576}
5577
5578#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5579enum ManagedPollOutcome {
5580    Idle,
5581    Handled,
5582    Stop,
5583}
5584
5585#[derive(Clone)]
5586pub struct Worker {
5587    client: Client,
5588    worker_id: String,
5589    task_queue: String,
5590    workflows: HashMap<String, RegisteredWorkflow>,
5591    activities: HashMap<String, ActivityHandler>,
5592    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5593    updates: HashMap<String, HashMap<String, UpdateHandler>>,
5594    max_concurrent_workflow_tasks: usize,
5595    max_concurrent_activity_tasks: usize,
5596    poll_timeout: Duration,
5597    heartbeat_interval: Duration,
5598    retry_policy: WorkerRetryPolicy,
5599    heartbeat_observer: Option<WorkerHeartbeatObserver>,
5600}
5601
5602impl Worker {
5603    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5604        Self {
5605            client,
5606            worker_id: default_worker_id(),
5607            task_queue: task_queue.into(),
5608            workflows: HashMap::new(),
5609            activities: HashMap::new(),
5610            queries: HashMap::new(),
5611            updates: HashMap::new(),
5612            max_concurrent_workflow_tasks: 10,
5613            max_concurrent_activity_tasks: 10,
5614            poll_timeout: Duration::from_secs(30),
5615            heartbeat_interval: Duration::from_secs(60),
5616            retry_policy: WorkerRetryPolicy::default(),
5617            heartbeat_observer: None,
5618        }
5619    }
5620
5621    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5622        self.worker_id = worker_id.into();
5623        self
5624    }
5625
5626    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5627        self.poll_timeout = timeout;
5628        self
5629    }
5630
5631    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5632        self.heartbeat_interval = interval;
5633        self
5634    }
5635
5636    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
5637    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5638        self.retry_policy = policy;
5639        self
5640    }
5641
5642    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5643    where
5644        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5645    {
5646        self.heartbeat_observer = Some(Arc::new(observer));
5647        self
5648    }
5649
5650    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5651        self.max_concurrent_workflow_tasks = count.max(1);
5652        self
5653    }
5654
5655    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5656        self.max_concurrent_activity_tasks = count.max(1);
5657        self
5658    }
5659
5660    /// Register a workflow handler.
5661    ///
5662    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
5663    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
5664    /// while acquiring or decoding a worker task remain worker-operation
5665    /// failures and do not get converted into workflow outcomes.
5666    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5667    where
5668        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5669        Fut: Future<Output = Result<Value>> + Send + 'static,
5670    {
5671        let handler = Arc::new(handler);
5672        self.workflows.insert(
5673            workflow_type.into(),
5674            RegisteredWorkflow {
5675                execute: Arc::new(move |ctx, input| {
5676                    let handler = Arc::clone(&handler);
5677                    Box::pin(async move {
5678                        let result = handler(ctx, input.into_json()?).await?;
5679                        AvroValue::from_serialize(&result)
5680                    })
5681                }),
5682                replay: None,
5683                state_type: None,
5684            },
5685        );
5686    }
5687
5688    /// Register a workflow with one Serde request value and a Serde result.
5689    ///
5690    /// This is an ergonomic adapter over the same fixed Avro Value protocol as
5691    /// [`Worker::register_workflow_avro_value`]. It does not create or publish a
5692    /// workflow-specific schema. A task must contain zero arguments for a unit
5693    /// request or exactly one argument for every other request type.
5694    ///
5695    /// See the runnable
5696    /// [`hello_world` example](https://github.com/durable-workflow/sdk-rust/blob/main/examples/hello_world.rs)
5697    /// for typed workflow and activity contracts with retry and timeout policy.
5698    pub fn register_typed_workflow<I, O, F, Fut>(
5699        &mut self,
5700        workflow_type: impl Into<String>,
5701        handler: F,
5702    ) where
5703        I: DeserializeOwned + Send + 'static,
5704        O: Serialize + Send + 'static,
5705        F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
5706        Fut: Future<Output = Result<O>> + Send + 'static,
5707    {
5708        let workflow_type = workflow_type.into();
5709        let handler_name = workflow_type.clone();
5710        let handler = Arc::new(handler);
5711        self.workflows.insert(
5712            workflow_type,
5713            RegisteredWorkflow {
5714                execute: Arc::new(move |ctx, input| {
5715                    let handler = Arc::clone(&handler);
5716                    let handler_name = handler_name.clone();
5717                    Box::pin(async move {
5718                        let input =
5719                            decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5720                        let result = handler(ctx, input).await?;
5721                        encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5722                    })
5723                }),
5724                replay: None,
5725                state_type: None,
5726            },
5727        );
5728    }
5729
5730    /// Register a workflow on the lossless fixed Avro Value surface.
5731    pub fn register_workflow_avro_value<F, Fut>(
5732        &mut self,
5733        workflow_type: impl Into<String>,
5734        handler: F,
5735    ) where
5736        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
5737        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5738    {
5739        self.workflows.insert(
5740            workflow_type.into(),
5741            RegisteredWorkflow {
5742                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
5743                replay: None,
5744                state_type: None,
5745            },
5746        );
5747    }
5748
5749    /// Register a workflow whose typed instance state can be reconstructed for queries.
5750    ///
5751    /// `state_factory` creates a fresh instance for every normal workflow task and
5752    /// query replay. The workflow handler is the single source of truth for state
5753    /// transitions: it updates [`WorkflowInstance`] after activities and signals
5754    /// resolve. Query replay runs this same handler over committed history and
5755    /// discards any commands it would emit.
5756    pub fn register_replayed_workflow<S, Factory, F, Fut>(
5757        &mut self,
5758        workflow_type: impl Into<String>,
5759        state_factory: Factory,
5760        handler: F,
5761    ) where
5762        S: Clone + Send + Sync + 'static,
5763        Factory: Fn() -> S + Send + Sync + 'static,
5764        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5765        Fut: Future<Output = Result<Value>> + Send + 'static,
5766    {
5767        let state_factory = Arc::new(state_factory);
5768        let handler = Arc::new(handler);
5769
5770        let execute_factory = Arc::clone(&state_factory);
5771        let execute_handler = Arc::clone(&handler);
5772        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5773            let state = WorkflowInstance::new(execute_factory());
5774            let handler = Arc::clone(&execute_handler);
5775            Box::pin(async move {
5776                let result = handler(ctx, input.into_json()?, state).await?;
5777                AvroValue::from_serialize(&result)
5778            }) as WorkflowFuture
5779        });
5780
5781        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5782            let state = WorkflowInstance::new(state_factory());
5783            let snapshot_state = state.clone();
5784            let snapshot: WorkflowStateSnapshot =
5785                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5786            let replay_handler = Arc::clone(&handler);
5787            let future = async move {
5788                let result = replay_handler(ctx, input.into_json()?, state).await?;
5789                AvroValue::from_serialize(&result)
5790            };
5791            ReplayedWorkflowInvocation {
5792                future: Box::pin(future),
5793                snapshot,
5794            }
5795        });
5796
5797        self.workflows.insert(
5798            workflow_type.into(),
5799            RegisteredWorkflow {
5800                execute,
5801                replay: Some(replay),
5802                state_type: Some(TypeId::of::<S>()),
5803            },
5804        );
5805    }
5806
5807    /// Register a replayable workflow with one Serde request value and result.
5808    ///
5809    /// Normal task execution and instance-state query replay both decode and
5810    /// encode through the fixed Avro Value codec. The state factory and handler
5811    /// otherwise follow [`Worker::register_replayed_workflow`].
5812    pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
5813        &mut self,
5814        workflow_type: impl Into<String>,
5815        state_factory: Factory,
5816        handler: F,
5817    ) where
5818        I: DeserializeOwned + Send + 'static,
5819        O: Serialize + Send + 'static,
5820        S: Clone + Send + Sync + 'static,
5821        Factory: Fn() -> S + Send + Sync + 'static,
5822        F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5823        Fut: Future<Output = Result<O>> + Send + 'static,
5824    {
5825        let workflow_type = workflow_type.into();
5826        let state_factory = Arc::new(state_factory);
5827        let handler = Arc::new(handler);
5828
5829        let execute_name = workflow_type.clone();
5830        let execute_factory = Arc::clone(&state_factory);
5831        let execute_handler = Arc::clone(&handler);
5832        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5833            let state = WorkflowInstance::new(execute_factory());
5834            let handler = Arc::clone(&execute_handler);
5835            let handler_name = execute_name.clone();
5836            Box::pin(async move {
5837                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5838                let result = handler(ctx, input, state).await?;
5839                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5840            }) as WorkflowFuture
5841        });
5842
5843        let replay_name = workflow_type.clone();
5844        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5845            let state = WorkflowInstance::new(state_factory());
5846            let snapshot_state = state.clone();
5847            let snapshot: WorkflowStateSnapshot =
5848                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5849            let handler = Arc::clone(&handler);
5850            let handler_name = replay_name.clone();
5851            let future = async move {
5852                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5853                let result = handler(ctx, input, state).await?;
5854                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5855            };
5856            ReplayedWorkflowInvocation {
5857                future: Box::pin(future),
5858                snapshot,
5859            }
5860        });
5861
5862        self.workflows.insert(
5863            workflow_type,
5864            RegisteredWorkflow {
5865                execute,
5866                replay: Some(replay),
5867                state_type: Some(TypeId::of::<S>()),
5868            },
5869        );
5870    }
5871
5872    /// Register a replayable workflow on the lossless fixed Avro Value surface.
5873    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
5874        &mut self,
5875        workflow_type: impl Into<String>,
5876        state_factory: Factory,
5877        handler: F,
5878    ) where
5879        S: Clone + Send + Sync + 'static,
5880        Factory: Fn() -> S + Send + Sync + 'static,
5881        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5882        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5883    {
5884        let state_factory = Arc::new(state_factory);
5885        let handler = Arc::new(handler);
5886
5887        let execute_factory = Arc::clone(&state_factory);
5888        let execute_handler = Arc::clone(&handler);
5889        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5890            let state = WorkflowInstance::new(execute_factory());
5891            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
5892        });
5893
5894        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5895            let state = WorkflowInstance::new(state_factory());
5896            let snapshot_state = state.clone();
5897            let snapshot: WorkflowStateSnapshot =
5898                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5899            ReplayedWorkflowInvocation {
5900                future: Box::pin(handler(ctx, input, state)),
5901                snapshot,
5902            }
5903        });
5904
5905        self.workflows.insert(
5906            workflow_type.into(),
5907            RegisteredWorkflow {
5908                execute,
5909                replay: Some(replay),
5910                state_type: Some(TypeId::of::<S>()),
5911            },
5912        );
5913    }
5914
5915    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
5916    where
5917        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
5918        Fut: Future<Output = Result<Value>> + Send + 'static,
5919    {
5920        let handler = Arc::new(handler);
5921        self.activities.insert(
5922            activity_type.into(),
5923            Arc::new(move |ctx, args| {
5924                let handler = Arc::clone(&handler);
5925                Box::pin(async move {
5926                    let result = handler(ctx, args.into_json()?).await?;
5927                    AvroValue::from_serialize(&result)
5928                })
5929            }),
5930        );
5931    }
5932
5933    /// Register an activity with one Serde request value and a Serde result.
5934    ///
5935    /// Inputs and results use the platform's fixed Avro Value schema. Shape
5936    /// mismatches and unsupported Serde values return [`Error::HandlerType`]
5937    /// with the activity name and Rust type.
5938    pub fn register_typed_activity<I, O, F, Fut>(
5939        &mut self,
5940        activity_type: impl Into<String>,
5941        handler: F,
5942    ) where
5943        I: DeserializeOwned + Send + 'static,
5944        O: Serialize + Send + 'static,
5945        F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
5946        Fut: Future<Output = Result<O>> + Send + 'static,
5947    {
5948        let activity_type = activity_type.into();
5949        let handler_name = activity_type.clone();
5950        let handler = Arc::new(handler);
5951        self.activities.insert(
5952            activity_type,
5953            Arc::new(move |ctx, input| {
5954                let handler = Arc::clone(&handler);
5955                let handler_name = handler_name.clone();
5956                Box::pin(async move {
5957                    let input =
5958                        decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
5959                    let result = handler(ctx, input).await?;
5960                    encode_handler_result(&result, HandlerKind::Activity, &handler_name)
5961                })
5962            }),
5963        );
5964    }
5965
5966    /// Register an activity on the lossless fixed Avro Value surface.
5967    pub fn register_activity_avro_value<F, Fut>(
5968        &mut self,
5969        activity_type: impl Into<String>,
5970        handler: F,
5971    ) where
5972        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
5973        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5974    {
5975        self.activities.insert(
5976            activity_type.into(),
5977            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
5978        );
5979    }
5980
5981    /// Register a named, read-only query handler for a workflow type.
5982    ///
5983    /// The workflow type must also be registered with [`Worker::register_workflow`]
5984    /// before the worker runs. The handler receives only an immutable committed
5985    /// state snapshot and normalized query arguments.
5986    pub fn register_query<F, Fut>(
5987        &mut self,
5988        workflow_type: impl Into<String>,
5989        query_name: impl Into<String>,
5990        handler: F,
5991    ) where
5992        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
5993        Fut: Future<Output = Result<Value>> + Send + 'static,
5994    {
5995        let handler = Arc::new(handler);
5996        self.queries
5997            .entry(workflow_type.into())
5998            .or_default()
5999            .insert(
6000                query_name.into(),
6001                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6002                    let handler = Arc::clone(&handler);
6003                    Box::pin(async move {
6004                        let result = handler(ctx, args.into_json()?).await?;
6005                        AvroValue::from_serialize(&result)
6006                    })
6007                })),
6008            );
6009    }
6010
6011    /// Register a query handler on the lossless fixed Avro Value surface.
6012    pub fn register_query_avro_value<F, Fut>(
6013        &mut self,
6014        workflow_type: impl Into<String>,
6015        query_name: impl Into<String>,
6016        handler: F,
6017    ) where
6018        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6019        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6020    {
6021        self.queries
6022            .entry(workflow_type.into())
6023            .or_default()
6024            .insert(
6025                query_name.into(),
6026                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6027            );
6028    }
6029
6030    /// Register a named query against deterministically replayed instance state.
6031    ///
6032    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
6033    /// same state type `S`. The handler receives an immutable, detached state
6034    /// clone, so successful and failed queries cannot affect workflow execution
6035    /// or the state reconstructed by a later query.
6036    pub fn register_replayed_query<S, F, Fut>(
6037        &mut self,
6038        workflow_type: impl Into<String>,
6039        query_name: impl Into<String>,
6040        handler: F,
6041    ) where
6042        S: Clone + Send + Sync + 'static,
6043        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6044        Fut: Future<Output = Result<Value>> + Send + 'static,
6045    {
6046        let handler = Arc::new(handler);
6047        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6048            let state = state.downcast::<S>().map_err(|_| {
6049                "registered query state type does not match the replayed workflow state".to_string()
6050            })?;
6051            let handler = Arc::clone(&handler);
6052            Ok(Box::pin(async move {
6053                let result = handler(ctx, state, args.into_json()?).await?;
6054                AvroValue::from_serialize(&result)
6055            }))
6056        });
6057
6058        self.queries
6059            .entry(workflow_type.into())
6060            .or_default()
6061            .insert(
6062                query_name.into(),
6063                RegisteredQuery::Replayed {
6064                    state_type: TypeId::of::<S>(),
6065                    handler: erased_handler,
6066                },
6067            );
6068    }
6069
6070    /// Register a replayed-state query on the lossless fixed Avro Value surface.
6071    pub fn register_replayed_query_avro_value<S, F, Fut>(
6072        &mut self,
6073        workflow_type: impl Into<String>,
6074        query_name: impl Into<String>,
6075        handler: F,
6076    ) where
6077        S: Clone + Send + Sync + 'static,
6078        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6079        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6080    {
6081        let handler = Arc::new(handler);
6082        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6083            let state = state.downcast::<S>().map_err(|_| {
6084                "registered query state type does not match the replayed workflow state".to_string()
6085            })?;
6086            Ok(Box::pin(handler(ctx, state, args)))
6087        });
6088
6089        self.queries
6090            .entry(workflow_type.into())
6091            .or_default()
6092            .insert(
6093                query_name.into(),
6094                RegisteredQuery::Replayed {
6095                    state_type: TypeId::of::<S>(),
6096                    handler: erased_handler,
6097                },
6098            );
6099    }
6100
6101    /// Register a synchronous workflow update handler.
6102    pub fn register_update<F, Fut>(
6103        &mut self,
6104        workflow_type: impl Into<String>,
6105        update_name: impl Into<String>,
6106        handler: F,
6107    ) where
6108        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6109        Fut: Future<Output = Result<Value>> + Send + 'static,
6110    {
6111        let handler = Arc::new(handler);
6112        self.updates
6113            .entry(workflow_type.into())
6114            .or_default()
6115            .insert(
6116                update_name.into(),
6117                Arc::new(move |ctx, args| {
6118                    let handler = Arc::clone(&handler);
6119                    Box::pin(async move {
6120                        let result = handler(ctx, args.into_json()?).await?;
6121                        AvroValue::from_serialize(&result)
6122                    })
6123                }),
6124            );
6125    }
6126
6127    /// Register an update handler on the lossless fixed Avro Value surface.
6128    pub fn register_update_avro_value<F, Fut>(
6129        &mut self,
6130        workflow_type: impl Into<String>,
6131        update_name: impl Into<String>,
6132        handler: F,
6133    ) where
6134        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6135        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6136    {
6137        self.updates
6138            .entry(workflow_type.into())
6139            .or_default()
6140            .insert(
6141                update_name.into(),
6142                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6143            );
6144    }
6145
6146    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6147        let mut command_contracts = serde_json::Map::new();
6148        for workflow_type in self.workflows.keys() {
6149            let mut queries = self
6150                .queries
6151                .get(workflow_type)
6152                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6153                .unwrap_or_default();
6154            queries.sort();
6155            let mut updates = self
6156                .updates
6157                .get(workflow_type)
6158                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6159                .unwrap_or_default();
6160            updates.sort();
6161            command_contracts.insert(
6162                workflow_type.clone(),
6163                json!({
6164                    "queries": queries,
6165                    "query_contracts": [],
6166                    "signals": [],
6167                    "signal_contracts": [],
6168                    "updates": updates,
6169                    "update_contracts": [],
6170                    "update_validators": [],
6171                }),
6172            );
6173        }
6174
6175        self.client
6176            .register_worker_with_command_contracts(
6177                &self.worker_id,
6178                &self.task_queue,
6179                self.workflows.keys().cloned().collect(),
6180                self.activities.keys().cloned().collect(),
6181                self.max_concurrent_workflow_tasks,
6182                self.max_concurrent_activity_tasks,
6183                [
6184                    Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6185                    Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6186                    Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6187                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6188                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6189                    worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6190                        .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6191                ]
6192                .into_iter()
6193                .flatten()
6194                .collect(),
6195                Value::Object(command_contracts),
6196            )
6197            .await
6198    }
6199
6200    /// Run until shutdown or a terminal worker error occurs.
6201    ///
6202    /// Empty long-poll expirations do not stop the worker. Retryable poll and
6203    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
6204    /// authentication, protocol, and other non-retryable failures are returned.
6205    pub async fn run(&self) -> Result<()> {
6206        self.run_until(std::future::pending::<()>()).await
6207    }
6208
6209    /// Run until `shutdown` resolves or a terminal worker error occurs.
6210    ///
6211    /// This has the same liveness and terminal-error contract as [`Worker::run`].
6212    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6213    where
6214        F: Future<Output = ()>,
6215    {
6216        let registration = self.register().await?;
6217        if !registration.registered {
6218            return Err(Error::WorkerLoop(format!(
6219                "worker registration for {:?} was not accepted",
6220                self.worker_id
6221            )));
6222        }
6223        let registered_worker_id = registration.worker_id.clone();
6224        let primary = self.run_registered_until(shutdown, registration).await;
6225        let deregistration = self
6226            .client
6227            .deregister_worker_registration(&registered_worker_id)
6228            .await;
6229
6230        match (primary, deregistration) {
6231            (Ok(()), Ok(_)) => Ok(()),
6232            (Ok(()), Err(deregistration)) => Err(deregistration),
6233            (Err(primary), Ok(_)) => Err(primary),
6234            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6235                primary: Box::new(primary),
6236                deregistration: Box::new(deregistration),
6237            }),
6238        }
6239    }
6240
6241    async fn run_registered_until<F>(
6242        &self,
6243        shutdown: F,
6244        registration: RegisterWorkerResponse,
6245    ) -> Result<()>
6246    where
6247        F: Future<Output = ()>,
6248    {
6249        let heartbeat_interval = Duration::from_secs(
6250            registration
6251                .heartbeat_interval_seconds
6252                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6253        );
6254        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
6255        // from the completion of the preceding attempt, including its bounded
6256        // retries. A fixed-epoch interval can leave an already-due tick queued
6257        // while an acknowledgement is slow, producing a catch-up heartbeat as
6258        // soon as that request completes.
6259        let heartbeat = tokio::time::sleep(Duration::ZERO);
6260        tokio::pin!(heartbeat);
6261        tokio::pin!(shutdown);
6262        let stop = Arc::new(AtomicBool::new(false));
6263        // Poll responses may already have leased server-side work by the time
6264        // they become ready, so each poller owns its responses through
6265        // completion or failure instead of racing raw polls in this select.
6266        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6267            let worker = self.clone();
6268            let stop = Arc::clone(&stop);
6269            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6270        });
6271        let mut activity_poller = (!self.activities.is_empty()).then(|| {
6272            let worker = self.clone();
6273            let stop = Arc::clone(&stop);
6274            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6275        });
6276        let mut query_poller = (!self.queries.is_empty()).then(|| {
6277            let worker = self.clone();
6278            let stop = Arc::clone(&stop);
6279            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6280        });
6281
6282        loop {
6283            tokio::select! {
6284                _ = &mut shutdown => {
6285                    stop.store(true, Ordering::SeqCst);
6286                    break;
6287                }
6288                _ = &mut heartbeat => {
6289                    let result = self.retry_worker_operation(|| {
6290                        self.client.heartbeat_worker(
6291                            &self.worker_id,
6292                            self.max_concurrent_workflow_tasks,
6293                            self.max_concurrent_activity_tasks,
6294                        )
6295                    }).await;
6296                    heartbeat
6297                        .as_mut()
6298                        .reset(tokio::time::Instant::now() + heartbeat_interval);
6299                    match result {
6300                        Ok(acknowledgement) => {
6301                            if let Some(observer) = &self.heartbeat_observer {
6302                                observer(&WorkerHeartbeatObservation {
6303                                    worker_id: self.worker_id.clone(),
6304                                    task_queue: self.task_queue.clone(),
6305                                    acknowledged_at_unix_millis: SystemTime::now()
6306                                        .duration_since(UNIX_EPOCH)
6307                                        .unwrap_or_default()
6308                                        .as_millis()
6309                                        .min(u64::MAX as u128)
6310                                        as u64,
6311                                    acknowledgement,
6312                                });
6313                            }
6314                        }
6315                        Err(error) => {
6316                            stop.store(true, Ordering::SeqCst);
6317                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6318                            return Err(error);
6319                        }
6320                    }
6321                }
6322                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6323                    workflow_poller = None;
6324                    let stopped_by_server = stop.load(Ordering::SeqCst);
6325                    stop.store(true, Ordering::SeqCst);
6326                    let poller_result = optional_poller_result("workflow", result);
6327                    let join_result =
6328                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6329                    poller_result?;
6330                    join_result?;
6331                    if stopped_by_server {
6332                        return Ok(());
6333                    }
6334                    return Err(Error::WorkerLoop(
6335                        "workflow poller stopped unexpectedly".to_string(),
6336                    ));
6337                }
6338                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6339                    activity_poller = None;
6340                    let stopped_by_server = stop.load(Ordering::SeqCst);
6341                    stop.store(true, Ordering::SeqCst);
6342                    let poller_result = optional_poller_result("activity", result);
6343                    let join_result =
6344                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6345                    poller_result?;
6346                    join_result?;
6347                    if stopped_by_server {
6348                        return Ok(());
6349                    }
6350                    return Err(Error::WorkerLoop(
6351                        "activity poller stopped unexpectedly".to_string(),
6352                    ));
6353                }
6354                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6355                    query_poller = None;
6356                    let stopped_by_server = stop.load(Ordering::SeqCst);
6357                    stop.store(true, Ordering::SeqCst);
6358                    let poller_result = optional_poller_result("query", result);
6359                    let join_result =
6360                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6361                    poller_result?;
6362                    join_result?;
6363                    if stopped_by_server {
6364                        return Ok(());
6365                    }
6366                    return Err(Error::WorkerLoop(
6367                        "query poller stopped unexpectedly".to_string(),
6368                    ));
6369                }
6370            }
6371        }
6372
6373        join_pollers(
6374            workflow_poller.take(),
6375            activity_poller.take(),
6376            query_poller.take(),
6377        )
6378        .await
6379    }
6380
6381    /// Poll and settle at most one task from each enabled task family.
6382    ///
6383    /// A workflow may reach its server-enforced run deadline while this worker
6384    /// holds a task. When the completion endpoint authoritatively rejects that
6385    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
6386    /// terminal `run_status=failed`, the workflow tick is considered settled:
6387    /// the late command was not recorded and cannot replace the terminal run.
6388    /// Every other completion rejection remains an error. This worker-level
6389    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
6390    /// only bounds how long a client waits for a result.
6391    ///
6392    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
6393    /// the original [`Error::Http`] status and response body.
6394    pub async fn run_once(&self) -> Result<usize> {
6395        let mut handled = 0;
6396        match self.poll_workflow_once().await? {
6397            ManagedPollOutcome::Handled => handled += 1,
6398            ManagedPollOutcome::Stop => return Ok(handled),
6399            ManagedPollOutcome::Idle => {}
6400        }
6401        match self.poll_activity_once().await? {
6402            ManagedPollOutcome::Handled => handled += 1,
6403            ManagedPollOutcome::Stop => return Ok(handled),
6404            ManagedPollOutcome::Idle => {}
6405        }
6406        if !self.queries.is_empty() {
6407            match self.poll_query_once().await? {
6408                ManagedPollOutcome::Handled => handled += 1,
6409                ManagedPollOutcome::Stop => return Ok(handled),
6410                ManagedPollOutcome::Idle => {}
6411            }
6412        }
6413        Ok(handled)
6414    }
6415
6416    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6417        let poll_request_id = unique_request_id("rust-workflow-poll");
6418        let response = self
6419            .retry_worker_operation(|| {
6420                self.client.poll_workflow_task_response_with_request_id(
6421                    &self.worker_id,
6422                    &self.task_queue,
6423                    self.poll_timeout,
6424                    &poll_request_id,
6425                    0,
6426                )
6427            })
6428            .await?;
6429        if response.outcome().should_stop() {
6430            return Ok(ManagedPollOutcome::Stop);
6431        }
6432        let memo_updates_supported =
6433            runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6434        let Some(task) = response.task else {
6435            return Ok(ManagedPollOutcome::Idle);
6436        };
6437
6438        let task_id = task.task_id.clone();
6439        let attempt = task.workflow_task_attempt;
6440        let run_id = task.run_id.clone();
6441        let lease_owner = task
6442            .lease_owner
6443            .clone()
6444            .unwrap_or_else(|| self.worker_id.clone());
6445
6446        match self.execute_workflow_task_decision(task) {
6447            Ok(decision)
6448                if commands_use_workflow_memo_updates(&decision.commands)
6449                    && !memo_updates_supported =>
6450            {
6451                self.client
6452                    .fail_workflow_task(
6453                        &task_id,
6454                        &lease_owner,
6455                        attempt,
6456                        Error::WorkflowMemoUpdatesUnavailable.to_string(),
6457                    )
6458                    .await?;
6459            }
6460            Ok(decision) if decision.commands.is_empty() => {
6461                // A replay can consume a recorded pending durable command
6462                // without producing a new command. The standalone protocol
6463                // acknowledges that state through the typed waiting outcome;
6464                // an empty completion is rejected by servers that require at
6465                // least one executable command.
6466                self.client
6467                    .fail_workflow_task_with_type(
6468                        &task_id,
6469                        &lease_owner,
6470                        attempt,
6471                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6472                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6473                    )
6474                    .await?;
6475            }
6476            Ok(decision) => {
6477                let completion = self
6478                    .client
6479                    .complete_workflow_task_with_message_streams(
6480                        &task_id,
6481                        &lease_owner,
6482                        attempt,
6483                        decision.commands,
6484                        decision.message_stream_cursors,
6485                        decision.message_stream_waits,
6486                    )
6487                    .await;
6488                if let Err(error) = completion {
6489                    if !workflow_task_completion_is_terminal_timeout(
6490                        &error,
6491                        &task_id,
6492                        attempt,
6493                        run_id.as_deref(),
6494                    ) {
6495                        return Err(error);
6496                    }
6497                }
6498            }
6499            Err(error) => {
6500                self.client
6501                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6502                    .await?;
6503            }
6504        }
6505
6506        Ok(ManagedPollOutcome::Handled)
6507    }
6508
6509    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6510        while !stop.load(Ordering::SeqCst) {
6511            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6512                stop.store(true, Ordering::SeqCst);
6513                break;
6514            }
6515        }
6516
6517        Ok(())
6518    }
6519
6520    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6521        let poll_request_id = unique_request_id("rust-activity-poll");
6522        let response = self
6523            .retry_worker_operation(|| {
6524                self.client.poll_activity_task_response_with_request_id(
6525                    &self.worker_id,
6526                    &self.task_queue,
6527                    self.poll_timeout,
6528                    &poll_request_id,
6529                    0,
6530                )
6531            })
6532            .await?;
6533        if response.outcome().should_stop() {
6534            return Ok(ManagedPollOutcome::Stop);
6535        }
6536        let Some(task) = response.task else {
6537            return Ok(ManagedPollOutcome::Idle);
6538        };
6539
6540        let task_id = task.task_id.clone();
6541        let attempt_id = task
6542            .activity_attempt_id
6543            .clone()
6544            .or(task.attempt_id.clone())
6545            .unwrap_or_default();
6546        let lease_owner = task
6547            .lease_owner
6548            .clone()
6549            .unwrap_or_else(|| self.worker_id.clone());
6550        let codec = task.payload_codec.clone();
6551        let result = self.execute_activity_task(task).await;
6552        match result {
6553            Ok(value) => {
6554                let completion = self
6555                    .client
6556                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6557                    .await;
6558                if let Err(error) = completion {
6559                    if !activity_task_rejection_is_final(&error) {
6560                        return Err(error);
6561                    }
6562                }
6563            }
6564            Err(error) => {
6565                let failure = self
6566                    .client
6567                    .fail_activity_task(
6568                        &task_id,
6569                        &attempt_id,
6570                        &lease_owner,
6571                        error.to_string(),
6572                        false,
6573                    )
6574                    .await;
6575                if let Err(error) = failure {
6576                    if !activity_task_rejection_is_final(&error) {
6577                        return Err(error);
6578                    }
6579                }
6580            }
6581        }
6582
6583        Ok(ManagedPollOutcome::Handled)
6584    }
6585
6586    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6587        while !stop.load(Ordering::SeqCst) {
6588            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6589                stop.store(true, Ordering::SeqCst);
6590                break;
6591            }
6592        }
6593
6594        Ok(())
6595    }
6596
6597    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6598        let poll_request_id = unique_request_id("rust-query-poll");
6599        let response = self
6600            .retry_worker_operation(|| {
6601                self.client.poll_query_task_response_with_request_id(
6602                    &self.worker_id,
6603                    &self.task_queue,
6604                    self.poll_timeout,
6605                    &poll_request_id,
6606                    0,
6607                )
6608            })
6609            .await?;
6610        if response.outcome().should_stop() {
6611            return Ok(ManagedPollOutcome::Stop);
6612        }
6613        let Some(task) = response.task else {
6614            return Ok(ManagedPollOutcome::Idle);
6615        };
6616
6617        let query_task_id = task.query_task_id.clone();
6618        let attempt = task.query_task_attempt;
6619        let lease_owner = task
6620            .lease_owner
6621            .clone()
6622            .unwrap_or_else(|| self.worker_id.clone());
6623        let codec = task.payload_codec.clone();
6624
6625        match self.execute_query_task(task).await {
6626            Ok(value) => {
6627                let result_envelope = match encode_typed_envelope(&value, &codec) {
6628                    Ok(result_envelope) => result_envelope,
6629                    Err(error) => {
6630                        let failure = self
6631                            .client
6632                            .fail_query_task(
6633                                &query_task_id,
6634                                &lease_owner,
6635                                attempt,
6636                                error.to_string(),
6637                                "query_result_encode_failed",
6638                                "QueryResultEncodeFailed",
6639                            )
6640                            .await;
6641                        if let Err(error) = failure {
6642                            if !query_task_rejection_is_final(&error) {
6643                                return Err(error);
6644                            }
6645                        }
6646                        return Ok(ManagedPollOutcome::Handled);
6647                    }
6648                };
6649
6650                if let Err(error) = self
6651                    .client
6652                    .complete_query_task_with_envelope(
6653                        &query_task_id,
6654                        &lease_owner,
6655                        attempt,
6656                        value.clone().into_json()?,
6657                        result_envelope,
6658                    )
6659                    .await
6660                {
6661                    if !query_task_rejection_is_final(&error) {
6662                        return Err(error);
6663                    }
6664                }
6665            }
6666            Err(failure) => {
6667                let result = self
6668                    .client
6669                    .fail_query_task(
6670                        &query_task_id,
6671                        &lease_owner,
6672                        attempt,
6673                        failure.message,
6674                        failure.reason,
6675                        failure.failure_type,
6676                    )
6677                    .await;
6678                if let Err(error) = result {
6679                    if !query_task_rejection_is_final(&error) {
6680                        return Err(error);
6681                    }
6682                }
6683            }
6684        }
6685
6686        Ok(ManagedPollOutcome::Handled)
6687    }
6688
6689    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6690        while !stop.load(Ordering::SeqCst) {
6691            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
6692                stop.store(true, Ordering::SeqCst);
6693                break;
6694            }
6695        }
6696
6697        Ok(())
6698    }
6699
6700    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
6701    where
6702        F: FnMut() -> Fut,
6703        Fut: Future<Output = Result<T>>,
6704    {
6705        let mut retries = 0;
6706
6707        loop {
6708            match operation().await {
6709                Err(error)
6710                    if worker_operation_is_retryable(&error)
6711                        && retries < self.retry_policy.max_retries =>
6712                {
6713                    retries += 1;
6714                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
6715                }
6716                result => return result,
6717            }
6718        }
6719    }
6720
6721    async fn execute_query_task(
6722        &self,
6723        mut task: QueryTask,
6724    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
6725        validate_query_task_payloads(&task).map_err(|error| {
6726            QueryTaskExecutionFailure::new(
6727                "query_payload_decode_failed",
6728                error.to_string(),
6729                "QueryPayloadDecodeFailed",
6730            )
6731        })?;
6732
6733        if !self.workflows.contains_key(&task.workflow_type) {
6734            return Err(QueryTaskExecutionFailure::new(
6735                "query_workflow_type_not_registered",
6736                format!("no workflow registered for type {:?}", task.workflow_type),
6737                "WorkflowTypeNotRegistered",
6738            ));
6739        }
6740
6741        let Some(handlers) = self.queries.get(&task.workflow_type) else {
6742            return Err(QueryTaskExecutionFailure::new(
6743                "query_handler_unavailable",
6744                format!(
6745                    "query handlers are unavailable for workflow type {:?}",
6746                    task.workflow_type
6747                ),
6748                "QueryHandlerUnavailable",
6749            ));
6750        };
6751        let Some(query) = handlers.get(&task.query_name) else {
6752            return Err(QueryTaskExecutionFailure::new(
6753                "rejected_unknown_query",
6754                format!("unknown query {:?}", task.query_name),
6755                "QueryFailed",
6756            ));
6757        };
6758
6759        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
6760            .map_err(|error| {
6761            QueryTaskExecutionFailure::new(
6762                "query_payload_decode_failed",
6763                format!("cannot decode query arguments: {error}"),
6764                "QueryPayloadDecodeFailed",
6765            )
6766        })?;
6767        let workflow_input_typed =
6768            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
6769                .map_err(|error| {
6770                    QueryTaskExecutionFailure::new(
6771                        "query_workflow_state_unavailable",
6772                        format!("cannot decode workflow start input: {error}"),
6773                        "QueryWorkflowStateUnavailable",
6774                    )
6775                })?;
6776        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
6777            QueryTaskExecutionFailure::new(
6778                "query_workflow_state_unavailable",
6779                format!("cannot project workflow start input: {error}"),
6780                "QueryWorkflowStateUnavailable",
6781            )
6782        })?;
6783        hydrate_query_history_from_export(&mut task).map_err(|error| {
6784            QueryTaskExecutionFailure::new(
6785                "query_workflow_state_unavailable",
6786                format!("cannot restore query history snapshot: {error}"),
6787                "QueryWorkflowStateUnavailable",
6788            )
6789        })?;
6790        enrich_query_history_from_export(&mut task).map_err(|error| {
6791            QueryTaskExecutionFailure::new(
6792                "query_workflow_state_unavailable",
6793                format!("cannot restore compact query history payloads: {error}"),
6794                "QueryWorkflowStateUnavailable",
6795            )
6796        })?;
6797        let signal_events = query_signal_events(&task).map_err(|error| {
6798            QueryTaskExecutionFailure::new(
6799                "query_workflow_state_unavailable",
6800                format!("cannot decode committed workflow signals: {error}"),
6801                "QueryWorkflowStateUnavailable",
6802            )
6803        })?;
6804        let history_events = Arc::new(std::mem::take(&mut task.history_events));
6805        let context = QueryContext {
6806            workflow_id: task.workflow_id,
6807            run_id: task.run_id,
6808            workflow_type: task.workflow_type.clone(),
6809            run_status: task.run_status,
6810            workflow_input,
6811            workflow_input_avro_value: workflow_input_typed.clone(),
6812            history_events: Arc::clone(&history_events),
6813            signal_events: Arc::new(signal_events),
6814        };
6815
6816        let future = match query {
6817            RegisteredQuery::Snapshot(handler) => handler(context, args),
6818            RegisteredQuery::Replayed {
6819                state_type,
6820                handler,
6821            } => {
6822                let workflow = self
6823                    .workflows
6824                    .get(&task.workflow_type)
6825                    .expect("workflow registration was checked above");
6826                if workflow.state_type != Some(*state_type) {
6827                    return Err(QueryTaskExecutionFailure::new(
6828                        "query_workflow_state_unavailable",
6829                        "replayed query state type does not match its workflow registration",
6830                        "QueryWorkflowStateUnavailable",
6831                    ));
6832                }
6833                let replay = workflow.replay.as_ref().ok_or_else(|| {
6834                    QueryTaskExecutionFailure::new(
6835                        "query_workflow_state_unavailable",
6836                        format!(
6837                            "workflow type {:?} is not registered for instance-state replay",
6838                            task.workflow_type
6839                        ),
6840                        "QueryWorkflowStateUnavailable",
6841                    )
6842                })?;
6843                let workflow_state = Arc::new(Mutex::new(
6844                    WorkflowState::new_with_identity(
6845                        history_events.as_ref().clone(),
6846                        context.workflow_id.clone(),
6847                        context.run_id.clone(),
6848                        self.task_queue.clone(),
6849                        task.payload_codec,
6850                        None,
6851                    )
6852                    .map_err(|error| {
6853                        QueryTaskExecutionFailure::new(
6854                            "query_workflow_state_unavailable",
6855                            format!("workflow replay failed before query: {error}"),
6856                            "QueryWorkflowStateUnavailable",
6857                        )
6858                    })?,
6859                ));
6860                let workflow_context = WorkflowContext {
6861                    state: workflow_state,
6862                };
6863                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
6864                let mut cx = TaskContext::from_waker(noop_waker_ref());
6865                match invocation.future.as_mut().poll(&mut cx) {
6866                    Poll::Ready(Ok(_)) => {
6867                        workflow_context
6868                            .ensure_history_consumed()
6869                            .map_err(|error| {
6870                                QueryTaskExecutionFailure::new(
6871                                    "query_workflow_state_unavailable",
6872                                    format!("workflow replay failed before query: {error}"),
6873                                    "QueryWorkflowStateUnavailable",
6874                                )
6875                            })?;
6876                    }
6877                    Poll::Ready(Err(error)) => {
6878                        return Err(QueryTaskExecutionFailure::new(
6879                            "query_workflow_state_unavailable",
6880                            format!("workflow replay failed before query: {error}"),
6881                            "QueryWorkflowStateUnavailable",
6882                        ));
6883                    }
6884                    Poll::Pending => {
6885                        let commands = workflow_context.take_commands().map_err(|error| {
6886                            QueryTaskExecutionFailure::new(
6887                                "query_workflow_state_unavailable",
6888                                format!("workflow replay failed before query: {error}"),
6889                                "QueryWorkflowStateUnavailable",
6890                            )
6891                        })?;
6892                        if commands.is_empty()
6893                            && !workflow_context
6894                                .matched_recorded_pending()
6895                                .map_err(|error| {
6896                                    QueryTaskExecutionFailure::new(
6897                                        "query_workflow_state_unavailable",
6898                                        format!("workflow replay failed before query: {error}"),
6899                                        "QueryWorkflowStateUnavailable",
6900                                    )
6901                                })?
6902                        {
6903                            return Err(QueryTaskExecutionFailure::new(
6904                                "query_workflow_state_unavailable",
6905                                "workflow replay yielded without a durable command",
6906                                "QueryWorkflowStateUnavailable",
6907                            ));
6908                        }
6909                    }
6910                }
6911                let state = (invocation.snapshot)().map_err(|error| {
6912                    QueryTaskExecutionFailure::new(
6913                        "query_workflow_state_unavailable",
6914                        format!("cannot snapshot replayed workflow state: {error}"),
6915                        "QueryWorkflowStateUnavailable",
6916                    )
6917                })?;
6918                handler(context, state, args).map_err(|message| {
6919                    QueryTaskExecutionFailure::new(
6920                        "query_workflow_state_unavailable",
6921                        message,
6922                        "QueryWorkflowStateUnavailable",
6923                    )
6924                })?
6925            }
6926        };
6927
6928        future.await.map_err(|error| {
6929            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
6930        })
6931    }
6932
6933    #[cfg(test)]
6934    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
6935        Ok(self.execute_workflow_task_decision(task)?.commands)
6936    }
6937
6938    fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
6939        validate_workflow_task_payloads(&task)?;
6940
6941        if let Some(update_id) = task
6942            .workflow_update_id
6943            .as_deref()
6944            .filter(|update_id| !update_id.is_empty())
6945        {
6946            return self
6947                .execute_update_task(&task, update_id)
6948                .map(WorkflowTaskDecision::without_message_streams);
6949        }
6950
6951        let workflow = self
6952            .workflows
6953            .get(&task.workflow_type)
6954            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
6955        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
6956        let resume_signal = decode_resume_signal(&task)?;
6957        let history_budget = WorkflowHistoryBudget {
6958            event_count: task
6959                .total_history_events
6960                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
6961            size_bytes: task.history_size_bytes,
6962            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
6963            pressure: task.history_budget_pressure.clone(),
6964        };
6965        let workflow_command_identity = task
6966            .workflow_command_id
6967            .clone()
6968            .filter(|identity| !identity.is_empty())
6969            .unwrap_or_default();
6970        let mut workflow_state = WorkflowState::new_with_identity(
6971            task.history_events,
6972            task.workflow_id,
6973            task.run_id,
6974            self.task_queue.clone(),
6975            task.payload_codec.clone(),
6976            resume_signal,
6977        )?;
6978        workflow_state.history_budget = history_budget;
6979        workflow_state.workflow_command_identity = workflow_command_identity;
6980        workflow_state.cancel_requested = task.cancel_requested;
6981        let state = Arc::new(Mutex::new(workflow_state));
6982        let ctx = WorkflowContext { state };
6983        let mut future = (workflow.execute)(ctx.clone(), input);
6984        let mut cx = TaskContext::from_waker(noop_waker_ref());
6985
6986        match future.as_mut().poll(&mut cx) {
6987            Poll::Ready(Ok(result)) => {
6988                ctx.ensure_history_consumed()?;
6989                let result = encode_typed_envelope(&result, &task.payload_codec)?;
6990                let mut commands = ctx.take_commands()?;
6991                commands.push(json!({
6992                    "type": "complete_workflow",
6993                    "result": result
6994                }));
6995                self.message_stream_decision(&ctx, commands)
6996            }
6997            Poll::Ready(Err(error)) => {
6998                if let Error::ContinueAsNew(request) = error {
6999                    let mut commands = ctx.take_commands()?;
7000                    if let Some(command) = ctx.continue_as_new_command(request)? {
7001                        commands.push(command);
7002                    }
7003                    ctx.ensure_history_consumed()?;
7004                    return self.message_stream_decision(&ctx, commands);
7005                }
7006                if workflow_task_integrity_error(&error) {
7007                    // Replay and protocol failures describe the workflow-task
7008                    // decision itself. Preserve their specific failure reason
7009                    // instead of replacing it with the derivative fact that
7010                    // recorded commands remain unconsumed.
7011                    return Err(error);
7012                }
7013                // A handler error must not hide a committed durable command that
7014                // upgraded workflow code no longer consumes.
7015                ctx.ensure_history_consumed()?;
7016                let mut commands = ctx.take_commands()?;
7017                commands.push(workflow_failure_command(&error));
7018                self.message_stream_decision(&ctx, commands)
7019            }
7020            Poll::Pending => {
7021                let commands = ctx.take_commands()?;
7022                if commands.is_empty() && !ctx.matched_recorded_pending()? {
7023                    Err(Error::WorkflowYieldedWithoutCommand)
7024                } else {
7025                    self.message_stream_decision(&ctx, commands)
7026                }
7027            }
7028        }
7029    }
7030
7031    fn message_stream_decision(
7032        &self,
7033        ctx: &WorkflowContext,
7034        commands: Vec<Value>,
7035    ) -> Result<WorkflowTaskDecision> {
7036        let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7037        Ok(WorkflowTaskDecision {
7038            commands,
7039            message_stream_cursors,
7040            message_stream_waits,
7041        })
7042    }
7043
7044    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7045        if !self.workflows.contains_key(&task.workflow_type) {
7046            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7047        }
7048
7049        let accepted = task.history_events.iter().rev().find_map(|event| {
7050            (event.event_type == "UpdateAccepted"
7051                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7052            .then_some(&event.payload)
7053        });
7054        let update_name = accepted
7055            .and_then(|payload| payload.get("update_name"))
7056            .and_then(Value::as_str)
7057            .or(task.update_name.as_deref())
7058            .unwrap_or_default();
7059        let Some(handler) = self
7060            .updates
7061            .get(&task.workflow_type)
7062            .and_then(|handlers| handlers.get(update_name))
7063        else {
7064            return Ok(vec![json!({
7065                "type": "fail_update",
7066                "update_id": update_id,
7067                "message": format!(
7068                    "no update handler is registered for {}.{update_name}",
7069                    task.workflow_type
7070                ),
7071                "exception_type": "UnknownUpdate",
7072                "non_retryable": true,
7073            })]);
7074        };
7075        let arguments = accepted
7076            .and_then(|payload| payload.get("arguments"))
7077            .or(task.arguments.as_ref());
7078        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7079        let context = QueryContext {
7080            workflow_id: task.workflow_id.clone(),
7081            run_id: task.run_id.clone(),
7082            workflow_type: task.workflow_type.clone(),
7083            run_status: Some("running".to_string()),
7084            workflow_input: Value::Null,
7085            workflow_input_avro_value: AvroValue::Null,
7086            history_events: Arc::new(task.history_events.clone()),
7087            signal_events: Arc::new(Vec::new()),
7088        };
7089        let mut future = handler(context, arguments);
7090        let mut cx = TaskContext::from_waker(noop_waker_ref());
7091
7092        match future.as_mut().poll(&mut cx) {
7093            Poll::Ready(Ok(result)) => Ok(vec![json!({
7094                "type": "complete_update",
7095                "update_id": update_id,
7096                "result": encode_typed_envelope(&result, &task.payload_codec)?,
7097            })]),
7098            Poll::Ready(Err(error)) => Ok(vec![json!({
7099                "type": "fail_update",
7100                "update_id": update_id,
7101                "message": error.to_string(),
7102                "exception_type": "UpdateFailed",
7103                "non_retryable": true,
7104            })]),
7105            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7106        }
7107    }
7108
7109    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7110        validate_activity_task_payloads(&task)?;
7111
7112        let handler = self
7113            .activities
7114            .get(&task.activity_type)
7115            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7116        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7117        let attempt_id = task
7118            .activity_attempt_id
7119            .clone()
7120            .or(task.attempt_id.clone())
7121            .unwrap_or_default();
7122        let lease_owner = task
7123            .lease_owner
7124            .clone()
7125            .unwrap_or_else(|| self.worker_id.clone());
7126        let ctx = ActivityContext {
7127            client: self.client.clone(),
7128            task_id: task.task_id,
7129            activity_attempt_id: attempt_id,
7130            lease_owner,
7131            activity_type: task.activity_type,
7132            attempt_number: task.attempt_number,
7133            task_queue: self.task_queue.clone(),
7134            worker_id: self.worker_id.clone(),
7135        };
7136
7137        handler(ctx, args).await
7138    }
7139}
7140
7141fn poller_result(
7142    kind: &str,
7143    result: std::result::Result<Result<()>, tokio::task::JoinError>,
7144) -> Result<()> {
7145    match result {
7146        Ok(result) => result,
7147        Err(error) => Err(Error::WorkerLoop(format!(
7148            "{kind} poller join error: {error}"
7149        ))),
7150    }
7151}
7152
7153fn optional_poller_result(
7154    kind: &str,
7155    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7156) -> Result<()> {
7157    match result {
7158        Some(result) => poller_result(kind, result),
7159        None => Ok(()),
7160    }
7161}
7162
7163async fn join_pollers(
7164    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7165    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7166    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7167) -> Result<()> {
7168    let mut first_error = None;
7169
7170    if let Some(handle) = workflow_poller {
7171        if let Err(error) = poller_result("workflow", handle.await) {
7172            first_error.get_or_insert(error);
7173        }
7174    }
7175
7176    if let Some(handle) = activity_poller {
7177        if let Err(error) = poller_result("activity", handle.await) {
7178            first_error.get_or_insert(error);
7179        }
7180    }
7181
7182    if let Some(handle) = query_poller {
7183        if let Err(error) = poller_result("query", handle.await) {
7184            first_error.get_or_insert(error);
7185        }
7186    }
7187
7188    if let Some(error) = first_error {
7189        Err(error)
7190    } else {
7191        Ok(())
7192    }
7193}
7194
7195fn default_worker_id() -> String {
7196    let millis = SystemTime::now()
7197        .duration_since(UNIX_EPOCH)
7198        .unwrap_or_default()
7199        .as_millis();
7200    format!("rust-worker-{}-{millis}", std::process::id())
7201}
7202
7203fn percent_encode_path_segment(segment: &str) -> String {
7204    const HEX: &[u8; 16] = b"0123456789ABCDEF";
7205    let mut encoded = String::with_capacity(segment.len());
7206
7207    for byte in segment.bytes() {
7208        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7209            encoded.push(char::from(byte));
7210        } else {
7211            encoded.push('%');
7212            encoded.push(char::from(HEX[(byte >> 4) as usize]));
7213            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7214        }
7215    }
7216
7217    encoded
7218}
7219
7220fn unique_request_id(prefix: &str) -> String {
7221    let nanos = SystemTime::now()
7222        .duration_since(UNIX_EPOCH)
7223        .unwrap_or_default()
7224        .as_nanos();
7225    format!("{prefix}-{}-{nanos}", std::process::id())
7226}
7227
7228#[derive(Debug)]
7229struct QueryTaskExecutionFailure {
7230    reason: String,
7231    message: String,
7232    failure_type: String,
7233}
7234
7235impl QueryTaskExecutionFailure {
7236    fn new(
7237        reason: impl Into<String>,
7238        message: impl Into<String>,
7239        failure_type: impl Into<String>,
7240    ) -> Self {
7241        Self {
7242            reason: reason.into(),
7243            message: message.into(),
7244            failure_type: failure_type.into(),
7245        }
7246    }
7247}
7248
7249/// Typed local state owned by one deterministic workflow invocation.
7250///
7251/// Use [`WorkflowInstance::update`] for the same state transitions during
7252/// ordinary execution and replay. A replayed query receives a detached
7253/// immutable `Arc<S>` rather than this mutation-capable handle.
7254#[derive(Clone, Debug)]
7255pub struct WorkflowInstance<S> {
7256    state: Arc<Mutex<S>>,
7257}
7258
7259impl<S> WorkflowInstance<S> {
7260    fn new(state: S) -> Self {
7261        Self {
7262            state: Arc::new(Mutex::new(state)),
7263        }
7264    }
7265
7266    /// Read the current workflow-instance state without changing it.
7267    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7268        let state = self
7269            .state
7270            .lock()
7271            .map_err(|_| Error::WorkflowStatePoisoned)?;
7272        Ok(reader(&state))
7273    }
7274
7275    /// Apply one deterministic workflow-instance state transition.
7276    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7277        let mut state = self
7278            .state
7279            .lock()
7280            .map_err(|_| Error::WorkflowStatePoisoned)?;
7281        Ok(transition(&mut state))
7282    }
7283}
7284
7285impl<S: Clone> WorkflowInstance<S> {
7286    fn snapshot(&self) -> Result<S> {
7287        self.read(Clone::clone)
7288    }
7289}
7290
7291#[derive(Clone, Debug, PartialEq)]
7292pub struct MessageStreamMessage {
7293    pub stream_name: String,
7294    pub message_id: String,
7295    pub position: u64,
7296    pub arguments: Vec<AvroValue>,
7297}
7298
7299#[derive(Clone, Debug)]
7300pub struct MessageStream {
7301    ctx: WorkflowContext,
7302    name: String,
7303}
7304
7305impl MessageStream {
7306    /// Wait for one message, then return a bounded currently-available batch.
7307    pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7308        if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7309            return Err(Error::Codec(format!(
7310                "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7311            )));
7312        }
7313        loop {
7314            if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7315                return Ok(batch);
7316            }
7317
7318            self.ctx.record_message_stream_wait(&self.name)?;
7319            let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7320            let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7321            self.ctx.buffer_message_stream_delivery(arguments)?;
7322            if let Some(sequence) = replay_wait_sequence {
7323                self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7324            }
7325        }
7326    }
7327
7328    pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7329        self.receive(1)
7330            .await?
7331            .into_iter()
7332            .next()
7333            .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7334    }
7335}
7336
7337#[derive(Clone, Debug)]
7338pub struct WorkflowContext {
7339    state: Arc<Mutex<WorkflowState>>,
7340}
7341
7342fn valid_memo_key(key: &str) -> bool {
7343    let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7344
7345    !key.is_empty()
7346        && key.len() <= 64
7347        && (numeric_candidate.is_empty()
7348            || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7349        && key
7350            .bytes()
7351            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7352}
7353
7354fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7355    BASE64
7356        .decode(encode_avro_value(value)?.blob)
7357        .map(|bytes| bytes.len())
7358        .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7359}
7360
7361fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7362    let AvroValue::Map(entries) = value else {
7363        return Err(Error::InvalidMemoUpdate(
7364            "entries must serialize to an Avro string-keyed map".to_string(),
7365        ));
7366    };
7367    if require_entries && entries.is_empty() {
7368        return Err(Error::InvalidMemoUpdate(
7369            "at least one entry is required".to_string(),
7370        ));
7371    }
7372    if entries.len() > MAX_MEMO_ENTRIES {
7373        return Err(Error::InvalidMemoUpdate(format!(
7374            "at most {MAX_MEMO_ENTRIES} entries are allowed"
7375        )));
7376    }
7377
7378    for (key, value) in &entries {
7379        if !valid_memo_key(&key) {
7380            return Err(Error::InvalidMemoUpdate(
7381                "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7382            ));
7383        }
7384        if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7385            return Err(Error::InvalidMemoUpdate(format!(
7386                "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7387            )));
7388        }
7389    }
7390
7391    let value = AvroValue::Map(entries);
7392    if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7393        return Err(Error::InvalidMemoUpdate(format!(
7394            "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7395        )));
7396    }
7397    Ok(value)
7398}
7399
7400fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7401    let object = envelope.as_object().ok_or_else(|| {
7402        Error::InvalidMemoUpdate(
7403            "history field must use the public {codec, blob} payload envelope".to_string(),
7404        )
7405    })?;
7406    if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7407        return Err(Error::InvalidMemoUpdate(
7408            "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7409        ));
7410    }
7411
7412    canonical_memo_entries(
7413        decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7414        require_entries,
7415    )
7416}
7417
7418impl WorkflowContext {
7419    pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7420        let name = name.into();
7421        if name.is_empty()
7422            || name.len() > 128
7423            || !name.bytes().all(|byte| {
7424                byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7425            })
7426        {
7427            return Err(Error::Codec(
7428                "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7429                    .to_string(),
7430            ));
7431        }
7432        Ok(MessageStream {
7433            ctx: self.clone(),
7434            name,
7435        })
7436    }
7437
7438    fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7439        let mut state = self
7440            .state
7441            .lock()
7442            .map_err(|_| Error::WorkflowStatePoisoned)?;
7443        let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7444        state
7445            .message_stream_waits
7446            .insert(name.to_string(), position);
7447        Ok(())
7448    }
7449
7450    fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7451        let mut state = self
7452            .state
7453            .lock()
7454            .map_err(|_| Error::WorkflowStatePoisoned)?;
7455        let cursor = state
7456            .message_stream_cursors
7457            .get(&message.stream_name)
7458            .copied()
7459            .unwrap_or(0);
7460        if message.position <= cursor {
7461            return Ok(());
7462        }
7463        let pending = state
7464            .message_stream_messages
7465            .entry(message.stream_name.clone())
7466            .or_default();
7467        if pending.iter().any(|candidate| {
7468            candidate.position == message.position || candidate.message_id == message.message_id
7469        }) {
7470            return Ok(());
7471        }
7472        pending.push(message);
7473        pending.sort_by_key(|candidate| candidate.position);
7474        Ok(())
7475    }
7476
7477    fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7478        if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7479            match delivery {
7480                MessageStreamDelivery::Message(message) => {
7481                    let stream_name = message.stream_name.clone();
7482                    self.buffer_message_stream(message)?;
7483                    return Ok(Some(stream_name));
7484                }
7485                MessageStreamDelivery::Cursor {
7486                    stream_name,
7487                    through_position,
7488                } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7489            }
7490        }
7491        Ok(None)
7492    }
7493
7494    fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7495        let state = self
7496            .state
7497            .lock()
7498            .map_err(|_| Error::WorkflowStatePoisoned)?;
7499        Ok(match state.recorded_commands.get(state.command_cursor) {
7500            Some(RecordedCommand::SignalWait {
7501                sequence,
7502                signal_name,
7503                ..
7504            }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7505            _ => None,
7506        })
7507    }
7508
7509    fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7510        let (history, payload_codec) = {
7511            let state = self
7512                .state
7513                .lock()
7514                .map_err(|_| Error::WorkflowStatePoisoned)?;
7515            (
7516                Arc::clone(&state.history_events),
7517                state.payload_codec.clone(),
7518            )
7519        };
7520
7521        let Some(opened_index) = history.iter().position(|event| {
7522            event.event_type == "SignalWaitOpened"
7523                && durable_event_sequence(event) == Some(wait_sequence)
7524                && event.payload.get("signal_name").and_then(Value::as_str)
7525                    == Some(MESSAGE_STREAM_SIGNAL)
7526        }) else {
7527            return Ok(());
7528        };
7529        let boundary_index = history
7530            .iter()
7531            .enumerate()
7532            .skip(opened_index + 1)
7533            .find_map(|(index, event)| {
7534                (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7535                    && is_authored_command_open_event(event))
7536                .then_some(index)
7537            })
7538            .unwrap_or(history.len());
7539
7540        for event in history[opened_index + 1..boundary_index]
7541            .iter()
7542            .filter(|event| {
7543                event.event_type == "SignalReceived"
7544                    && event.payload.get("signal_name").and_then(Value::as_str)
7545                        == Some(MESSAGE_STREAM_SIGNAL)
7546            })
7547        {
7548            let arguments = decode_signal_event_arguments(event, &payload_codec)?
7549                .into_iter()
7550                .map(AvroValue::into_json)
7551                .collect::<Result<Vec<_>>>()?;
7552            self.buffer_message_stream_delivery(arguments)?;
7553        }
7554        Ok(())
7555    }
7556
7557    fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7558        let mut state = self
7559            .state
7560            .lock()
7561            .map_err(|_| Error::WorkflowStatePoisoned)?;
7562        let cursor = state
7563            .message_stream_cursors
7564            .entry(name.to_string())
7565            .or_default();
7566        *cursor = (*cursor).max(through_position);
7567        if let Some(pending) = state.message_stream_messages.get_mut(name) {
7568            pending.retain(|message| message.position > through_position);
7569        }
7570        Ok(())
7571    }
7572
7573    fn take_message_stream_batch(
7574        &self,
7575        name: &str,
7576        max_items: usize,
7577    ) -> Result<Option<Vec<MessageStreamMessage>>> {
7578        let mut state = self
7579            .state
7580            .lock()
7581            .map_err(|_| Error::WorkflowStatePoisoned)?;
7582        let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7583        let pending = state
7584            .message_stream_messages
7585            .entry(name.to_string())
7586            .or_default();
7587        let count = contiguous_message_stream_count(pending, cursor, max_items);
7588        if count == 0 {
7589            return Ok(None);
7590        }
7591        let batch = pending.drain(..count).collect::<Vec<_>>();
7592        let position = batch.last().map(|message| message.position).unwrap_or(0);
7593        state
7594            .message_stream_cursors
7595            .insert(name.to_string(), position);
7596        state.message_stream_waits.remove(name);
7597        Ok(Some(batch))
7598    }
7599
7600    fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
7601        let state = self
7602            .state
7603            .lock()
7604            .map_err(|_| Error::WorkflowStatePoisoned)?;
7605        let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
7606        cursors.sort_by_key(|(name, _)| *name);
7607        let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
7608        waits.sort_by_key(|(name, _)| *name);
7609        Ok((
7610            cursors
7611                .into_iter()
7612                .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
7613                .collect(),
7614            waits
7615                .into_iter()
7616                .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
7617                .collect(),
7618        ))
7619    }
7620    /// Identity of the parent workflow currently being replayed.
7621    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7622        let state = self
7623            .state
7624            .lock()
7625            .map_err(|_| Error::WorkflowStatePoisoned)?;
7626        Ok(WorkflowIdentity {
7627            workflow_id: state.workflow_id.clone(),
7628            run_id: state.run_id.clone(),
7629        })
7630    }
7631
7632    /// Return the server-published history budget for this workflow task.
7633    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
7634        let state = self
7635            .state
7636            .lock()
7637            .map_err(|_| Error::WorkflowStatePoisoned)?;
7638        Ok(state.history_budget.clone())
7639    }
7640
7641    /// Continue this workflow instance as a fresh run with replacement arguments.
7642    ///
7643    /// Return this value directly from the workflow handler. The worker converts
7644    /// it to the terminal protocol command only after replay has consumed every
7645    /// recorded durable command.
7646    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
7647        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
7648    }
7649
7650    /// Continue as new with optional workflow-type and task-queue overrides.
7651    pub fn continue_as_new_with_options<T: Serialize>(
7652        &self,
7653        options: ContinueAsNewOptions,
7654        args: T,
7655    ) -> Result<Value> {
7656        options.validate()?;
7657        Err(Error::ContinueAsNew(ContinueAsNewRequest {
7658            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
7659            options,
7660        }))
7661    }
7662
7663    pub fn activity<T: Serialize>(
7664        &self,
7665        activity_type: impl Into<String>,
7666        args: T,
7667    ) -> ActivityCall {
7668        self.activity_with_options(activity_type, ActivityOptions::new(), args)
7669    }
7670
7671    pub fn activity_on_queue<T, Q>(
7672        &self,
7673        activity_type: impl Into<String>,
7674        task_queue: Option<Q>,
7675        args: T,
7676    ) -> ActivityCall
7677    where
7678        T: Serialize,
7679        Q: Into<String>,
7680    {
7681        let mut options = ActivityOptions::new();
7682        options.task_queue = task_queue.map(Into::into);
7683        self.activity_with_options(activity_type, options, args)
7684    }
7685
7686    /// Schedule one durable activity with retry, routing, and timeout options.
7687    ///
7688    /// Options are validated before the command is emitted. Once the command is
7689    /// recorded, replay consumes the same activity lifecycle at this command
7690    /// position and never emits a duplicate schedule.
7691    ///
7692    /// ```no_run
7693    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
7694    /// # use std::time::Duration;
7695    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
7696    /// let result = ctx
7697    ///     .activity_with_options(
7698    ///         "charge-card",
7699    ///         ActivityOptions::new()
7700    ///             .task_queue("payments")
7701    ///             .retry_policy(
7702    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
7703    ///                     Duration::from_secs(1),
7704    ///                     2,
7705    ///                     Some(Duration::from_secs(30)),
7706    ///                 ),
7707    ///             )
7708    ///             .start_to_close_timeout(Duration::from_secs(60))
7709    ///             .schedule_to_close_timeout(Duration::from_secs(180))
7710    ///             .heartbeat_timeout(Duration::from_secs(15)),
7711    ///         json!([{"order_id": "order-42"}]),
7712    ///     )
7713    ///     .await;
7714    /// match result {
7715    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
7716    ///         "reason": failure.reason,
7717    ///         "timeout_kind": failure.timeout_kind,
7718    ///     })),
7719    ///     other => other,
7720    /// }
7721    /// # }
7722    /// ```
7723    pub fn activity_with_options<T: Serialize>(
7724        &self,
7725        activity_type: impl Into<String>,
7726        options: ActivityOptions,
7727        args: T,
7728    ) -> ActivityCall {
7729        ActivityCall {
7730            ctx: self.clone(),
7731            activity_type: activity_type.into(),
7732            options,
7733            args: Some(AvroValue::from_serialize(&args)),
7734            scheduled: false,
7735            parallel_group_path: Vec::new(),
7736        }
7737    }
7738
7739    pub async fn activity_avro_value<T: Serialize>(
7740        &self,
7741        activity_type: impl Into<String>,
7742        args: T,
7743    ) -> Result<AvroValue> {
7744        let mut call = self.activity(activity_type, args);
7745        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7746    }
7747
7748    pub async fn activity_avro_value_with_options<T: Serialize>(
7749        &self,
7750        activity_type: impl Into<String>,
7751        options: ActivityOptions,
7752        args: T,
7753    ) -> Result<AvroValue> {
7754        let mut call = self.activity_with_options(activity_type, options, args);
7755        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7756    }
7757
7758    /// Schedule an activity with a Serde request and decode its Serde result.
7759    pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
7760    where
7761        I: Serialize,
7762        O: DeserializeOwned,
7763    {
7764        self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
7765            .await
7766    }
7767
7768    /// Schedule an activity with options and decode its result into `O`.
7769    ///
7770    /// Both directions use the fixed Avro Value codec. In particular, this
7771    /// method does not deserialize the JSON-safe inspection projection returned
7772    /// by the dynamic [`ActivityCall`] future.
7773    pub async fn activity_typed_with_options<I, O>(
7774        &self,
7775        activity_type: impl Into<String>,
7776        options: ActivityOptions,
7777        args: I,
7778    ) -> Result<O>
7779    where
7780        I: Serialize,
7781        O: DeserializeOwned,
7782    {
7783        let activity_type = activity_type.into();
7784        let encoded = AvroValue::from_serialize(&args).map_err(|error| {
7785            handler_type_error::<I>(
7786                HandlerKind::Activity,
7787                &activity_type,
7788                HandlerValueKind::Input,
7789                error.to_string(),
7790            )
7791        });
7792        let mut call = ActivityCall {
7793            ctx: self.clone(),
7794            activity_type: activity_type.clone(),
7795            options,
7796            args: Some(encoded),
7797            scheduled: false,
7798            parallel_group_path: Vec::new(),
7799        };
7800        let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
7801        decode_handler_result(result, HandlerKind::Activity, &activity_type)
7802    }
7803
7804    /// Schedule and join a deterministic activity/child/timer group.
7805    ///
7806    /// Nested groups retain their input shape. Every durable leaf is scheduled
7807    /// before this future yields, results are assembled by declaration order,
7808    /// and a failure returns [`Error::ParallelFailed`] with typed cause,
7809    /// declaration path, stable group metadata, and completed siblings.
7810    pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7811        ParallelCall::new(self.clone(), operations)
7812    }
7813
7814    /// Alias for [`WorkflowContext::parallel`].
7815    pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7816        self.parallel(operations)
7817    }
7818
7819    /// Lossless fixed-Avro variant of [`WorkflowContext::parallel`].
7820    pub async fn parallel_avro_value(
7821        &self,
7822        operations: Vec<ParallelOperation>,
7823    ) -> Result<Vec<ParallelAvroResult>> {
7824        let mut call = self.parallel(operations);
7825        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7826    }
7827
7828    /// Create a workflow-local deterministic compensation registry.
7829    pub fn saga(&self) -> Saga {
7830        Saga::new(self.clone())
7831    }
7832
7833    /// Whether the current workflow task carries a cooperative cancel request.
7834    pub fn is_cancellation_requested(&self) -> Result<bool> {
7835        let state = self
7836            .state
7837            .lock()
7838            .map_err(|_| Error::WorkflowStatePoisoned)?;
7839        Ok(state.cancel_requested)
7840    }
7841
7842    /// Raise a typed cooperative cancellation at an author-controlled point.
7843    ///
7844    /// Passing this result to [`Saga::finish`] compensates already registered
7845    /// forward steps before the cancellation remains the initiating outcome.
7846    pub fn throw_if_cancellation_requested(&self) -> Result<()> {
7847        if self.is_cancellation_requested()? {
7848            return Err(Error::WorkflowCancellationRequested(
7849                WorkflowCancellationRequested,
7850            ));
7851        }
7852        Ok(())
7853    }
7854
7855    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
7856        SignalCall {
7857            ctx: self.clone(),
7858            signal_name: signal_name.into(),
7859            runtime_reserved_allowed: false,
7860            opened_wait: false,
7861            matched_pending: false,
7862        }
7863    }
7864
7865    fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
7866        SignalCall {
7867            ctx: self.clone(),
7868            signal_name: signal_name.into(),
7869            runtime_reserved_allowed: true,
7870            opened_wait: false,
7871            matched_pending: false,
7872        }
7873    }
7874
7875    pub async fn wait_signal_avro_value(
7876        &self,
7877        signal_name: impl Into<String>,
7878    ) -> Result<Vec<AvroValue>> {
7879        let mut call = self.wait_signal(signal_name);
7880        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7881    }
7882
7883    /// Return every committed signal argument list with the given name.
7884    ///
7885    /// This history-backed view is deterministic and is intended for
7886    /// condition predicates that must be re-evaluated after a signal while the
7887    /// workflow is blocked on [`WorkflowContext::wait_condition`].
7888    pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
7889        self.signals_avro_value(signal_name)?
7890            .into_iter()
7891            .map(|arguments| {
7892                arguments
7893                    .into_iter()
7894                    .map(AvroValue::into_json)
7895                    .collect::<Result<Vec<_>>>()
7896            })
7897            .collect()
7898    }
7899
7900    /// Lossless fixed Avro Value view of committed signals with the given name.
7901    pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
7902        let state = self
7903            .state
7904            .lock()
7905            .map_err(|_| Error::WorkflowStatePoisoned)?;
7906        state
7907            .history_events
7908            .iter()
7909            .filter(|event| {
7910                event.event_type == "SignalReceived"
7911                    && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
7912            })
7913            .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
7914            .collect()
7915    }
7916
7917    /// Return every committed update argument list with the given name.
7918    ///
7919    /// Accepted and applied records for the same update ID are de-duplicated.
7920    /// A Server task created after an update therefore replays the workflow and
7921    /// re-evaluates an open condition without application polling.
7922    pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
7923        self.updates_avro_value(update_name)?
7924            .into_iter()
7925            .map(|arguments| {
7926                arguments
7927                    .into_iter()
7928                    .map(AvroValue::into_json)
7929                    .collect::<Result<Vec<_>>>()
7930            })
7931            .collect()
7932    }
7933
7934    /// Lossless fixed Avro Value view of committed updates with the given name.
7935    pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
7936        let state = self
7937            .state
7938            .lock()
7939            .map_err(|_| Error::WorkflowStatePoisoned)?;
7940        let mut seen = Vec::new();
7941        let mut updates = Vec::new();
7942        for event in state.history_events.iter() {
7943            if !matches!(
7944                event.event_type.as_str(),
7945                "UpdateAccepted" | "UpdateApplied"
7946            ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
7947                || event.payload.get("arguments").is_none()
7948            {
7949                continue;
7950            }
7951            if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
7952                if seen.iter().any(|recorded| recorded == update_id) {
7953                    continue;
7954                }
7955                seen.push(update_id.to_string());
7956            }
7957            updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
7958        }
7959        Ok(updates)
7960    }
7961
7962    /// Wait for a deterministic predicate to become true or for its durable
7963    /// timeout to elapse.
7964    ///
7965    /// Prefer [`wait_condition!`] for inline predicates so changes to the Rust
7966    /// predicate tokens automatically change the recorded definition
7967    /// fingerprint. Direct callers must provide an equally stable identity in
7968    /// [`ConditionWaitOptions`].
7969    pub fn wait_condition<F>(
7970        &self,
7971        options: ConditionWaitOptions,
7972        predicate: F,
7973    ) -> ConditionWaitCall
7974    where
7975        F: Fn() -> Result<bool> + Send + 'static,
7976    {
7977        ConditionWaitCall {
7978            ctx: self.clone(),
7979            options,
7980            predicate: Box::new(predicate),
7981            occurrence_id: None,
7982            opened_wait: false,
7983        }
7984    }
7985
7986    /// Wait for server-backed durable time without blocking the worker executor.
7987    ///
7988    /// Polling this future emits one `start_timer` command and yields. The
7989    /// server records the deadline, so neither worker nor server restarts reset
7990    /// the wait. Replay resolves the future only from a `TimerScheduled` and
7991    /// `TimerFired` pair at the same position in the shared durable-command
7992    /// stream, with matching sequence, timer identity, and delay. Sub-second
7993    /// durations round up because protocol deadlines use whole seconds.
7994    ///
7995    /// ```no_run
7996    /// # use durable_workflow::{json, Client, Worker};
7997    /// # use std::time::Duration;
7998    /// # fn configure(client: Client) {
7999    /// let mut worker = Worker::new(client, "rust-workers");
8000    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
8001    ///     ctx.sleep(Duration::from_secs(5)).await?;
8002    ///     Ok(json!({"status": "timer fired"}))
8003    /// });
8004    /// # }
8005    /// ```
8006    pub fn sleep(&self, duration: Duration) -> TimerCall {
8007        let delay_seconds = duration
8008            .as_secs()
8009            .checked_add(u64::from(duration.subsec_nanos() > 0));
8010        TimerCall {
8011            ctx: self.clone(),
8012            delay_seconds,
8013            scheduled: false,
8014            matched_pending: false,
8015            parallel_group_path: Vec::new(),
8016        }
8017    }
8018
8019    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
8020    pub fn start_timer(&self, duration: Duration) -> TimerCall {
8021        self.sleep(duration)
8022    }
8023
8024    /// Evaluate a non-deterministic callback once and durably record its typed value.
8025    ///
8026    /// On replay the callback is not invoked: the value is decoded from the
8027    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
8028    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
8029    /// small values that must remain fixed for the lifetime of a workflow run.
8030    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8031    where
8032        T: Serialize + DeserializeOwned,
8033        F: FnOnce() -> T,
8034    {
8035        {
8036            let mut state = self
8037                .state
8038                .lock()
8039                .map_err(|_| Error::WorkflowStatePoisoned)?;
8040            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8041                return match recorded {
8042                    RecordedCommand::SideEffect { sequence, value } => {
8043                        state.command_cursor += 1;
8044                        value.deserialize().map_err(|error| {
8045                            Error::NonDeterministicReplay(ReplayFailure::new(
8046                                "side_effect_type_mismatch",
8047                                Some(sequence),
8048                                Some(std::any::type_name::<T>().to_string()),
8049                                Some(error.to_string()),
8050                                "recorded side-effect value is incompatible with the requested Rust type",
8051                            ))
8052                        })
8053                    }
8054                    other => Err(command_mismatch(&other, "side effect")),
8055                };
8056            }
8057        }
8058
8059        let value = callback();
8060        let avro_value = AvroValue::from_serialize(&value)?;
8061        let mut state = self
8062            .state
8063            .lock()
8064            .map_err(|_| Error::WorkflowStatePoisoned)?;
8065        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8066        state.commands.push(json!({
8067            "type": "record_side_effect",
8068            "result": result,
8069        }));
8070        Ok(value)
8071    }
8072
8073    /// Record or replay a lossless fixed Avro Value side effect.
8074    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8075    where
8076        F: FnOnce() -> AvroValue,
8077    {
8078        {
8079            let mut state = self
8080                .state
8081                .lock()
8082                .map_err(|_| Error::WorkflowStatePoisoned)?;
8083            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8084                return match recorded {
8085                    RecordedCommand::SideEffect { value, .. } => {
8086                        state.command_cursor += 1;
8087                        Ok(value)
8088                    }
8089                    other => Err(command_mismatch(&other, "side effect")),
8090                };
8091            }
8092        }
8093
8094        let value = callback();
8095        let mut state = self
8096            .state
8097            .lock()
8098            .map_err(|_| Error::WorkflowStatePoisoned)?;
8099        let result = encode_typed_envelope(&value, &state.payload_codec)?;
8100        state.commands.push(json!({
8101            "type": "record_side_effect",
8102            "result": result,
8103        }));
8104        Ok(value)
8105    }
8106
8107    /// Append output items at a deterministic workflow command boundary.
8108    ///
8109    /// Stable idempotency keys are derived from the server-provided durable
8110    /// workflow command identity, command ordinal, and item index. Replay
8111    /// consumes the recorded side effect and never emits another append.
8112    pub fn append_workflow_stream(
8113        &self,
8114        stream_name: impl Into<String>,
8115        items: &[WorkflowStreamAppendItem],
8116        max_pending_items: Option<u64>,
8117    ) -> Result<()> {
8118        if items.is_empty() {
8119            return Err(Error::Codec(
8120                "workflow_stream_items_empty: append requires at least one item".to_string(),
8121            ));
8122        }
8123        if max_pending_items == Some(0) {
8124            return Err(Error::Codec(
8125                "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8126                    .to_string(),
8127            ));
8128        }
8129        let stream_name = stream_name.into();
8130        if stream_name.is_empty() {
8131            return Err(Error::Codec(
8132                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8133            ));
8134        }
8135
8136        let mut state = self
8137            .state
8138            .lock()
8139            .map_err(|_| Error::WorkflowStatePoisoned)?;
8140        let command_ordinal = state.workflow_stream_command_counter;
8141        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8142            state.workflow_stream_command_counter += 1;
8143            return match recorded {
8144                RecordedCommand::SideEffect { .. } => {
8145                    state.command_cursor += 1;
8146                    Ok(())
8147                }
8148                other => Err(command_mismatch(&other, "workflow stream append")),
8149            };
8150        }
8151
8152        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8153        state.workflow_stream_command_counter += 1;
8154        let wire_items = items
8155            .iter()
8156            .enumerate()
8157            .map(|(item_index, item)| {
8158                item.wire_value(Some(format!(
8159                    "dw-stream:{identity}:{command_ordinal}:{item_index}"
8160                )))
8161            })
8162            .collect::<Vec<_>>();
8163        let mut directive = json!({
8164            "operation": "append",
8165            "stream_name": stream_name,
8166            "command_identity": identity,
8167            "command_ordinal": command_ordinal,
8168            "items": wire_items,
8169        });
8170        if let Some(max_pending_items) = max_pending_items {
8171            directive["max_pending_items"] = json!(max_pending_items);
8172        }
8173        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8174        state.commands.push(json!({
8175            "type": "record_side_effect",
8176            "result": result,
8177            "workflow_stream": directive,
8178        }));
8179        Ok(())
8180    }
8181
8182    /// Close a run-scoped output stream at a deterministic command boundary.
8183    pub fn close_workflow_stream(
8184        &self,
8185        stream_name: impl Into<String>,
8186        retention_seconds: Option<u64>,
8187    ) -> Result<()> {
8188        self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8189    }
8190
8191    /// Mark a run-scoped output stream errored at a deterministic command boundary.
8192    pub fn error_workflow_stream(
8193        &self,
8194        stream_name: impl Into<String>,
8195        error_reason: impl Into<String>,
8196        retention_seconds: Option<u64>,
8197    ) -> Result<()> {
8198        let error_reason = error_reason.into();
8199        if error_reason.is_empty() {
8200            return Err(Error::Codec(
8201                "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8202            ));
8203        }
8204        self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8205    }
8206
8207    fn finish_workflow_stream(
8208        &self,
8209        stream_name: String,
8210        error_reason: Option<String>,
8211        retention_seconds: Option<u64>,
8212    ) -> Result<()> {
8213        if stream_name.is_empty() {
8214            return Err(Error::Codec(
8215                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8216            ));
8217        }
8218        if retention_seconds == Some(0) {
8219            return Err(Error::Codec(
8220                "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8221            ));
8222        }
8223        let mut state = self
8224            .state
8225            .lock()
8226            .map_err(|_| Error::WorkflowStatePoisoned)?;
8227        let command_ordinal = state.workflow_stream_command_counter;
8228        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8229            state.workflow_stream_command_counter += 1;
8230            return match recorded {
8231                RecordedCommand::SideEffect { .. } => {
8232                    state.command_cursor += 1;
8233                    Ok(())
8234                }
8235                other => Err(command_mismatch(&other, "workflow stream close")),
8236            };
8237        }
8238        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8239        state.workflow_stream_command_counter += 1;
8240        let mut directive = json!({
8241            "operation": if error_reason.is_some() { "error" } else { "close" },
8242            "stream_name": stream_name,
8243            "command_identity": identity,
8244            "command_ordinal": command_ordinal,
8245        });
8246        if let Some(error_reason) = error_reason {
8247            directive["error_reason"] = json!(error_reason);
8248        }
8249        if let Some(retention_seconds) = retention_seconds {
8250            directive["retention_seconds"] = json!(retention_seconds);
8251        }
8252        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8253        state.commands.push(json!({
8254            "type": "record_side_effect",
8255            "result": result,
8256            "workflow_stream": directive,
8257        }));
8258        Ok(())
8259    }
8260
8261    fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8262        let identity = state.workflow_command_identity.as_str();
8263        if identity.is_empty() {
8264            return Err(Error::MissingWorkflowCommandIdentity);
8265        }
8266        Ok(identity)
8267    }
8268
8269    /// Validate, emit, or replay a typed workflow search-attribute update.
8270    ///
8271    /// The command is non-blocking within a workflow decision, but its
8272    /// `SearchAttributesUpserted` event occupies the same deterministic command
8273    /// stream as activities, timers, conditions, and other durable operations.
8274    pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8275        update.validate()?;
8276        let (attributes, attribute_types) = update.into_wire_parts();
8277        let mut state = self
8278            .state
8279            .lock()
8280            .map_err(|_| Error::WorkflowStatePoisoned)?;
8281
8282        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8283            return match recorded {
8284                RecordedCommand::SearchAttributes {
8285                    sequence,
8286                    attributes: recorded_attributes,
8287                    attribute_types: recorded_attribute_types,
8288                } => {
8289                    if recorded_attributes != attributes {
8290                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8291                            "search_attribute_value_mismatch",
8292                            Some(sequence),
8293                            Some(recorded_attributes.to_string()),
8294                            Some(attributes.to_string()),
8295                            "search-attribute values differ from the recorded durable command",
8296                        )));
8297                    }
8298                    if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8299                        if recorded_types != attribute_types {
8300                            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8301                                "search_attribute_type_mismatch",
8302                                Some(sequence),
8303                                Some(json!(recorded_types).to_string()),
8304                                Some(json!(attribute_types).to_string()),
8305                                "search-attribute declared types differ from the recorded durable command",
8306                            )));
8307                        }
8308                    }
8309                    state.command_cursor += 1;
8310                    Ok(())
8311                }
8312                other => Err(command_mismatch(&other, "search-attribute update")),
8313            };
8314        }
8315
8316        let mut command = serde_json::Map::from_iter([
8317            ("type".to_string(), json!("upsert_search_attributes")),
8318            ("attributes".to_string(), attributes),
8319        ]);
8320        if !attribute_types.is_empty() {
8321            command.insert("attribute_types".to_string(), json!(attribute_types));
8322        }
8323        state.commands.push(Value::Object(command));
8324        Ok(())
8325    }
8326
8327    /// Record a UUIDv4 once and return the same UUID on every replay.
8328    pub fn uuid_v4(&self) -> Result<Uuid> {
8329        self.side_effect(Uuid::new_v4)
8330    }
8331
8332    /// Select the newest supported version for a change, or replay the version
8333    /// already committed for that stable change ID.
8334    pub fn get_version(
8335        &self,
8336        change_id: impl Into<String>,
8337        min_supported: i32,
8338        max_supported: i32,
8339    ) -> Result<i32> {
8340        let change_id = change_id.into();
8341        if change_id.trim().is_empty() {
8342            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8343                "version_change_id_invalid",
8344                None,
8345                Some("non-empty change ID".to_string()),
8346                Some(change_id),
8347                "version markers require a stable non-empty change ID",
8348            )));
8349        }
8350        if min_supported > max_supported {
8351            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8352                "version_range_invalid",
8353                None,
8354                Some("min_supported <= max_supported".to_string()),
8355                Some(format!("{min_supported}..={max_supported}")),
8356                "version marker supported range is invalid",
8357            )));
8358        }
8359
8360        let mut state = self
8361            .state
8362            .lock()
8363            .map_err(|_| Error::WorkflowStatePoisoned)?;
8364        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8365            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8366            return Ok(version);
8367        }
8368
8369        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8370            return match recorded {
8371                RecordedCommand::VersionMarker {
8372                    sequence,
8373                    change_id: recorded_change_id,
8374                    version,
8375                    ..
8376                } => {
8377                    if recorded_change_id != change_id {
8378                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8379                            "version_change_id_mismatch",
8380                            Some(sequence),
8381                            Some(recorded_change_id),
8382                            Some(change_id),
8383                            "recorded version marker change ID differs from current workflow code",
8384                        )));
8385                    }
8386                    ensure_version_supported(
8387                        &change_id,
8388                        version,
8389                        min_supported,
8390                        max_supported,
8391                        sequence,
8392                    )?;
8393                    state.command_cursor += 1;
8394                    state.version_markers.insert(change_id, (version, sequence));
8395                    Ok(version)
8396                }
8397                other => Err(command_mismatch(
8398                    &other,
8399                    format!("version marker:{change_id}"),
8400                )),
8401            };
8402        }
8403
8404        let version = max_supported;
8405        state.commands.push(json!({
8406            "type": "record_version_marker",
8407            "change_id": change_id,
8408            "version": version,
8409            "min_supported": min_supported,
8410            "max_supported": max_supported,
8411        }));
8412        // Sequence numbers are assigned by the server. Zero identifies a marker
8413        // selected in this uncommitted decision batch for duplicate-call checks.
8414        state.version_markers.insert(change_id, (version, 0));
8415        Ok(version)
8416    }
8417
8418    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
8419    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8420        Ok(self.get_version(change_id, -1, 1)? == 1)
8421    }
8422
8423    /// Keep a patch marker in history after the legacy branch has been removed.
8424    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8425        self.get_version(change_id, -1, 1).map(|_| ())
8426    }
8427
8428    /// Merge non-indexed workflow memo metadata through durable history.
8429    ///
8430    /// Avro `null` deletes a key. The SDK encodes the complete patch in the
8431    /// public Avro payload envelope consumed by Server and Cloud runtimes.
8432    pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8433        let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8434        let mut state = self
8435            .state
8436            .lock()
8437            .map_err(|_| Error::WorkflowStatePoisoned)?;
8438
8439        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8440            return match recorded {
8441                RecordedCommand::Memo {
8442                    sequence,
8443                    entries: recorded_entries,
8444                } => {
8445                    if recorded_entries != entries {
8446                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8447                            "memo_update_mismatch",
8448                            Some(sequence),
8449                            Some(format!("{recorded_entries:?}")),
8450                            Some(format!("{entries:?}")),
8451                            "recorded memo entries differ from the current workflow update",
8452                        )));
8453                    }
8454                    state.command_cursor += 1;
8455                    Ok(())
8456                }
8457                other => Err(command_mismatch(&other, "memo upsert")),
8458            };
8459        }
8460
8461        let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8462        state.commands.push(json!({
8463            "type": "upsert_memo",
8464            "entries": entries_envelope,
8465        }));
8466        Ok(())
8467    }
8468
8469    /// Start a named durable child on an explicit queue and await its result.
8470    ///
8471    /// The command is recorded in the parent's sequence-ordered durable command
8472    /// stream. Replay keeps a scheduled child pending without emitting another
8473    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
8474    /// values preserve the history payload codec and include both sides of the
8475    /// durable relationship; failures are returned as
8476    /// [`Error::ChildWorkflowFailed`].
8477    ///
8478    /// ```no_run
8479    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
8480    /// # fn configure(client: Client) {
8481    /// let mut worker = Worker::new(client, "parent-workers");
8482    /// worker.register_workflow("order-parent", |ctx, _input| async move {
8483    ///     let child = ctx
8484    ///         .start_child_workflow(
8485    ///             "fulfil-order",
8486    ///             ChildWorkflowOptions::new("fulfilment-workers")
8487    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
8488    ///             json!([{"order_id": "order-42"}]),
8489    ///         )
8490    ///         .await?;
8491    ///     Ok(child.result)
8492    /// });
8493    /// # }
8494    /// ```
8495    pub fn start_child_workflow<T: Serialize>(
8496        &self,
8497        workflow_type: impl Into<String>,
8498        options: ChildWorkflowOptions,
8499        args: T,
8500    ) -> ChildWorkflowCall {
8501        ChildWorkflowCall {
8502            ctx: self.clone(),
8503            workflow_type: workflow_type.into(),
8504            options,
8505            args: Some(AvroValue::from_serialize(&args)),
8506            scheduled: false,
8507            matched_pending: false,
8508            parallel_group_path: Vec::new(),
8509        }
8510    }
8511
8512    pub async fn start_child_workflow_avro_value<T: Serialize>(
8513        &self,
8514        workflow_type: impl Into<String>,
8515        options: ChildWorkflowOptions,
8516        args: T,
8517    ) -> Result<ChildWorkflowAvroResult> {
8518        let mut call = self.start_child_workflow(workflow_type, options, args);
8519        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8520    }
8521
8522    fn take_commands(&self) -> Result<Vec<Value>> {
8523        let mut state = self
8524            .state
8525            .lock()
8526            .map_err(|_| Error::WorkflowStatePoisoned)?;
8527        Ok(std::mem::take(&mut state.commands))
8528    }
8529
8530    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8531        let mut state = self
8532            .state
8533            .lock()
8534            .map_err(|_| Error::WorkflowStatePoisoned)?;
8535
8536        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8537            return Err(command_mismatch(&recorded, "continue as new"));
8538        }
8539        if state.recorded_continue_as_new_sequence.is_some() {
8540            state.continue_as_new_consumed = true;
8541            return Ok(None);
8542        }
8543
8544        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8545        let mut command = serde_json::Map::from_iter([
8546            ("type".to_string(), json!("continue_as_new")),
8547            ("arguments".to_string(), arguments),
8548            ("queue".to_string(), json!(state.task_queue.clone())),
8549        ]);
8550        if let Some(workflow_type) = request.options.workflow_type {
8551            command.insert("workflow_type".to_string(), json!(workflow_type));
8552        }
8553        if let Some(task_queue) = request.options.task_queue {
8554            command.insert("queue".to_string(), json!(task_queue));
8555        }
8556        Ok(Some(Value::Object(command)))
8557    }
8558
8559    fn matched_recorded_pending(&self) -> Result<bool> {
8560        let state = self
8561            .state
8562            .lock()
8563            .map_err(|_| Error::WorkflowStatePoisoned)?;
8564        Ok(state.matched_recorded_pending)
8565    }
8566
8567    fn ensure_history_consumed(&self) -> Result<()> {
8568        let state = self
8569            .state
8570            .lock()
8571            .map_err(|_| Error::WorkflowStatePoisoned)?;
8572        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8573            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8574                "recorded_commands_unconsumed",
8575                Some(command.sequence()),
8576                Some(command.shape().to_string()),
8577                Some("workflow completion".to_string()),
8578                "workflow completed before consuming all recorded durable commands",
8579            )));
8580        }
8581        if let Some(sequence) = state
8582            .recorded_continue_as_new_sequence
8583            .filter(|_| !state.continue_as_new_consumed)
8584        {
8585            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8586                "recorded_continue_as_new_unconsumed",
8587                Some(sequence),
8588                Some("continue as new".to_string()),
8589                Some("workflow completion".to_string()),
8590                "workflow completed without consuming its recorded continue-as-new transition",
8591            )));
8592        }
8593        Ok(())
8594    }
8595}
8596
8597fn contiguous_message_stream_count(
8598    pending: &[MessageStreamMessage],
8599    cursor: u64,
8600    max_items: usize,
8601) -> usize {
8602    pending
8603        .iter()
8604        .take(max_items)
8605        .enumerate()
8606        .take_while(|(offset, message)| {
8607            u64::try_from(*offset)
8608                .ok()
8609                .and_then(|offset| cursor.checked_add(offset + 1))
8610                == Some(message.position)
8611        })
8612        .count()
8613}
8614
8615fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
8616    matches!(
8617        event.event_type.as_str(),
8618        "ActivityScheduled"
8619            | "TimerScheduled"
8620            | "ChildWorkflowScheduled"
8621            | "SignalWaitOpened"
8622            | "ConditionWaitOpened"
8623            | "SearchAttributesUpserted"
8624            | "SideEffectRecorded"
8625            | "VersionMarkerRecorded"
8626            | "MemoUpserted"
8627            | "WorkflowContinuedAsNew"
8628    )
8629}
8630
8631#[derive(Debug)]
8632struct WorkflowState {
8633    workflow_id: Option<String>,
8634    run_id: Option<String>,
8635    task_queue: String,
8636    payload_codec: String,
8637    history_events: Arc<Vec<HistoryEvent>>,
8638    history_budget: WorkflowHistoryBudget,
8639    cancel_requested: bool,
8640    resume_signal: Option<ResumeSignal>,
8641    recorded_commands: Vec<RecordedCommand>,
8642    recorded_continue_as_new_sequence: Option<u64>,
8643    continue_as_new_consumed: bool,
8644    command_cursor: usize,
8645    condition_wait_occurrence_counter: u64,
8646    matched_recorded_pending: bool,
8647    version_markers: HashMap<String, (i32, u64)>,
8648    workflow_command_identity: String,
8649    workflow_stream_command_counter: u64,
8650    commands: Vec<Value>,
8651    message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
8652    message_stream_cursors: HashMap<String, u64>,
8653    message_stream_waits: HashMap<String, u64>,
8654}
8655
8656impl WorkflowState {
8657    #[cfg(test)]
8658    fn new(
8659        history: Vec<HistoryEvent>,
8660        task_queue: String,
8661        payload_codec: String,
8662        resume_signal: Option<ResumeSignal>,
8663    ) -> Result<Self> {
8664        Self::new_with_identity(
8665            history,
8666            None,
8667            None,
8668            task_queue,
8669            payload_codec,
8670            resume_signal,
8671        )
8672    }
8673
8674    fn new_with_identity(
8675        history: Vec<HistoryEvent>,
8676        workflow_id: Option<String>,
8677        run_id: Option<String>,
8678        task_queue: String,
8679        payload_codec: String,
8680        resume_signal: Option<ResumeSignal>,
8681    ) -> Result<Self> {
8682        let recorded_commands = recorded_commands(
8683            &history,
8684            &payload_codec,
8685            WorkflowIdentity {
8686                workflow_id: workflow_id.clone(),
8687                run_id: run_id.clone(),
8688            },
8689        )?;
8690        let recorded_continue_as_new = history
8691            .iter()
8692            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
8693            .collect::<Vec<_>>();
8694        if recorded_continue_as_new.len() > 1 {
8695            return Err(invalid_recorded_history(
8696                "duplicate_continue_as_new_transition",
8697                recorded_continue_as_new
8698                    .last()
8699                    .and_then(|event| durable_event_sequence(event))
8700                    .unwrap_or(0),
8701                "one WorkflowContinuedAsNew event",
8702                &format!(
8703                    "{} WorkflowContinuedAsNew events",
8704                    recorded_continue_as_new.len()
8705                ),
8706                "workflow history records one continue-as-new transition more than once",
8707            ));
8708        }
8709        let recorded_continue_as_new_sequence = recorded_continue_as_new
8710            .first()
8711            .map(|event| {
8712                durable_event_sequence(event).ok_or_else(|| {
8713                    Error::NonDeterministicReplay(ReplayFailure::new(
8714                        "continue_as_new_sequence_missing",
8715                        None,
8716                        Some("recorded transition sequence".to_string()),
8717                        Some("missing sequence".to_string()),
8718                        "WorkflowContinuedAsNew history is missing its recorded sequence",
8719                    ))
8720                })
8721            })
8722            .transpose()?;
8723        let mut message_stream_cursors = HashMap::new();
8724        for event in &history {
8725            if !matches!(
8726                event.event_type.as_str(),
8727                "SignalReceived" | "SignalApplied"
8728            ) || event.payload.get("signal_name").and_then(Value::as_str)
8729                != Some(MESSAGE_STREAM_SIGNAL)
8730            {
8731                continue;
8732            }
8733            let arguments = decode_signal_event_arguments(event, &payload_codec)?;
8734            if arguments.len() != 1 {
8735                continue;
8736            }
8737            let envelope = arguments[0].clone().into_json()?;
8738            let Some(envelope) = envelope.as_object() else {
8739                continue;
8740            };
8741            if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
8742            {
8743                continue;
8744            }
8745            let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
8746                continue;
8747            };
8748            let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
8749            else {
8750                continue;
8751            };
8752            let cursor = message_stream_cursors
8753                .entry(stream_name.to_string())
8754                .or_insert(0);
8755            *cursor = (*cursor).max(through_position);
8756        }
8757        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
8758        let cancel_requested = history.iter().any(|event| {
8759            matches!(
8760                event.event_type.as_str(),
8761                "WorkflowCancellationRequested" | "WorkflowCancelRequested"
8762            )
8763        });
8764        Ok(Self {
8765            workflow_command_identity: String::new(),
8766            workflow_stream_command_counter: 0,
8767            workflow_id,
8768            run_id,
8769            task_queue,
8770            payload_codec,
8771            history_events: Arc::new(history),
8772            history_budget: WorkflowHistoryBudget {
8773                event_count,
8774                ..WorkflowHistoryBudget::default()
8775            },
8776            cancel_requested,
8777            resume_signal,
8778            recorded_commands,
8779            recorded_continue_as_new_sequence,
8780            continue_as_new_consumed: false,
8781            command_cursor: 0,
8782            condition_wait_occurrence_counter: 0,
8783            matched_recorded_pending: false,
8784            version_markers: HashMap::new(),
8785            commands: Vec::new(),
8786            message_stream_messages: HashMap::new(),
8787            message_stream_cursors,
8788            message_stream_waits: HashMap::new(),
8789        })
8790    }
8791}
8792
8793enum MessageStreamDelivery {
8794    Message(MessageStreamMessage),
8795    Cursor {
8796        stream_name: String,
8797        through_position: u64,
8798    },
8799}
8800
8801fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
8802    if arguments.len() != 1 {
8803        return Ok(None);
8804    }
8805    let envelope = arguments
8806        .into_iter()
8807        .next()
8808        .expect("one argument was checked");
8809    let Some(envelope) = envelope.as_object() else {
8810        return Ok(None);
8811    };
8812    let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
8813        return Ok(None);
8814    };
8815    if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
8816        let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
8817        else {
8818            return Ok(None);
8819        };
8820        return Ok(Some(MessageStreamDelivery::Cursor {
8821            stream_name: stream_name.to_string(),
8822            through_position,
8823        }));
8824    }
8825    if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
8826        return Ok(None);
8827    }
8828    let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
8829        return Ok(None);
8830    };
8831    let Some(position) = envelope
8832        .get("position")
8833        .and_then(Value::as_u64)
8834        .filter(|value| *value > 0)
8835    else {
8836        return Ok(None);
8837    };
8838    let Some(payload_envelope) = envelope.get("payload_envelope") else {
8839        return Ok(None);
8840    };
8841    let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
8842    else {
8843        return Ok(None);
8844    };
8845    let decoded = decode_avro_value(&payload_envelope)?;
8846    let AvroValue::Array(values) = decoded else {
8847        return Ok(None);
8848    };
8849    Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
8850        stream_name: stream_name.to_string(),
8851        message_id: message_id.to_string(),
8852        position,
8853        arguments: values,
8854    })))
8855}
8856
8857#[derive(Clone, Debug)]
8858enum RecordedCommand {
8859    Activity {
8860        sequence: u64,
8861        activity_type: Option<String>,
8862        options: Option<RecordedActivityOptions>,
8863        outcome: Option<ActivityOutcome>,
8864        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
8865    },
8866    Timer {
8867        sequence: u64,
8868        delay_seconds: u64,
8869        fired: bool,
8870        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
8871    },
8872    ChildWorkflow {
8873        sequence: u64,
8874        workflow_type: Option<String>,
8875        outcome: Option<ChildWorkflowOutcome>,
8876        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
8877    },
8878    SignalWait {
8879        sequence: u64,
8880        signal_name: String,
8881        value: Option<Vec<AvroValue>>,
8882    },
8883    ConditionWait {
8884        sequence: u64,
8885        occurrence_id: String,
8886        condition_key: Option<String>,
8887        predicate_identity: String,
8888        timeout_seconds: Option<u64>,
8889        result: Option<ConditionWaitResult>,
8890    },
8891    SearchAttributes {
8892        sequence: u64,
8893        attributes: Value,
8894        attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
8895    },
8896    SideEffect {
8897        sequence: u64,
8898        value: AvroValue,
8899    },
8900    VersionMarker {
8901        sequence: u64,
8902        change_id: String,
8903        version: i32,
8904    },
8905    Memo {
8906        sequence: u64,
8907        entries: AvroValue,
8908    },
8909}
8910
8911#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
8912struct RecordedActivityOptions {
8913    task_queue: RecordedSnapshotValue<Option<String>>,
8914    execution_mode: RecordedSnapshotValue<Option<String>>,
8915    retry_policy: ActivityRetrySnapshot,
8916}
8917
8918#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
8919enum RecordedSnapshotValue<T> {
8920    /// Older history did not persist this field, so it cannot constrain replay.
8921    Unknown,
8922    Known(T),
8923}
8924
8925impl<T: PartialEq> RecordedSnapshotValue<T> {
8926    fn matches_current(&self, current: &Self) -> bool {
8927        match self {
8928            Self::Unknown => true,
8929            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
8930        }
8931    }
8932}
8933
8934#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
8935struct ActivityRetrySnapshot {
8936    snapshot_version: RecordedSnapshotValue<Option<u64>>,
8937    max_attempts: RecordedSnapshotValue<Option<u64>>,
8938    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
8939    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
8940    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
8941    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
8942    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
8943    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
8944}
8945
8946impl ActivityRetrySnapshot {
8947    fn matches_current(&self, current: &Self) -> bool {
8948        self.snapshot_version
8949            .matches_current(&current.snapshot_version)
8950            && self.max_attempts.matches_current(&current.max_attempts)
8951            && self
8952                .backoff_seconds
8953                .matches_current(&current.backoff_seconds)
8954            && self
8955                .start_to_close_timeout
8956                .matches_current(&current.start_to_close_timeout)
8957            && self
8958                .schedule_to_start_timeout
8959                .matches_current(&current.schedule_to_start_timeout)
8960            && self
8961                .schedule_to_close_timeout
8962                .matches_current(&current.schedule_to_close_timeout)
8963            && self
8964                .heartbeat_timeout
8965                .matches_current(&current.heartbeat_timeout)
8966            && self
8967                .non_retryable_error_types
8968                .matches_current(&current.non_retryable_error_types)
8969    }
8970}
8971
8972fn recorded_optional_u64(
8973    object: Option<&serde_json::Map<String, Value>>,
8974    field: &str,
8975) -> RecordedSnapshotValue<Option<u64>> {
8976    match object.and_then(|object| object.get(field)) {
8977        None => RecordedSnapshotValue::Unknown,
8978        Some(Value::Null) => RecordedSnapshotValue::Known(None),
8979        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
8980    }
8981}
8982
8983fn recorded_optional_string(
8984    object: &serde_json::Map<String, Value>,
8985    field: &str,
8986) -> RecordedSnapshotValue<Option<String>> {
8987    match object.get(field) {
8988        None => RecordedSnapshotValue::Unknown,
8989        Some(Value::Null) => RecordedSnapshotValue::Known(None),
8990        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
8991    }
8992}
8993
8994fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
8995    let policy = policy.and_then(Value::as_object);
8996    let backoff_seconds = policy
8997        .and_then(|policy| policy.get("backoff_seconds"))
8998        .and_then(Value::as_array)
8999        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9000        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9001    let mut non_retryable_error_types = Vec::new();
9002    for error_type in policy
9003        .and_then(|policy| policy.get("non_retryable_error_types"))
9004        .and_then(Value::as_array)
9005        .into_iter()
9006        .flatten()
9007        .filter_map(Value::as_str)
9008        .map(str::trim)
9009        .filter(|error_type| !error_type.is_empty())
9010    {
9011        if !non_retryable_error_types
9012            .iter()
9013            .any(|recorded| recorded == error_type)
9014        {
9015            non_retryable_error_types.push(error_type.to_string());
9016        }
9017    }
9018
9019    ActivityRetrySnapshot {
9020        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9021        max_attempts: recorded_optional_u64(policy, "max_attempts"),
9022        backoff_seconds,
9023        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9024        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9025        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9026        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9027        non_retryable_error_types: if policy
9028            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9029        {
9030            RecordedSnapshotValue::Known(non_retryable_error_types)
9031        } else {
9032            RecordedSnapshotValue::Unknown
9033        },
9034    }
9035}
9036
9037fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9038    let policy = options.retry_policy.as_ref();
9039    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9040        Some(Value::Null) => None,
9041        Some(value) => value_as_u64(value),
9042        None => Some(1),
9043    };
9044    let backoff_seconds = policy
9045        .and_then(|policy| policy.get("backoff_seconds"))
9046        .and_then(Value::as_array)
9047        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9048        .unwrap_or_default();
9049    let non_retryable_error_types = policy
9050        .and_then(|policy| policy.get("non_retryable_error_types"))
9051        .and_then(Value::as_array)
9052        .into_iter()
9053        .flatten()
9054        .filter_map(Value::as_str)
9055        .map(str::to_string)
9056        .collect();
9057
9058    ActivityRetrySnapshot {
9059        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9060        max_attempts: RecordedSnapshotValue::Known(max_attempts),
9061        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9062        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9063        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9064        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9065        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9066        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9067    }
9068}
9069
9070fn activity_options_description(options: &RecordedActivityOptions) -> String {
9071    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9072}
9073
9074impl RecordedCommand {
9075    fn sequence(&self) -> u64 {
9076        match self {
9077            Self::Activity { sequence, .. }
9078            | Self::Timer { sequence, .. }
9079            | Self::ChildWorkflow { sequence, .. }
9080            | Self::SignalWait { sequence, .. }
9081            | Self::ConditionWait { sequence, .. }
9082            | Self::SearchAttributes { sequence, .. }
9083            | Self::SideEffect { sequence, .. }
9084            | Self::VersionMarker { sequence, .. }
9085            | Self::Memo { sequence, .. } => *sequence,
9086        }
9087    }
9088
9089    fn shape(&self) -> &'static str {
9090        match self {
9091            Self::Activity { .. } => "activity",
9092            Self::Timer { .. } => "timer",
9093            Self::ChildWorkflow { .. } => "child workflow",
9094            Self::SignalWait { .. } => "signal wait",
9095            Self::ConditionWait { .. } => "condition wait",
9096            Self::SearchAttributes { .. } => "search-attribute update",
9097            Self::SideEffect { .. } => "side effect",
9098            Self::VersionMarker { .. } => "version marker",
9099            Self::Memo { .. } => "memo upsert",
9100        }
9101    }
9102}
9103
9104fn ensure_version_supported(
9105    change_id: &str,
9106    version: i32,
9107    min_supported: i32,
9108    max_supported: i32,
9109    sequence: u64,
9110) -> Result<()> {
9111    if (min_supported..=max_supported).contains(&version) {
9112        return Ok(());
9113    }
9114    Err(Error::NonDeterministicReplay(ReplayFailure::new(
9115        "version_marker_incompatible_range",
9116        (sequence != 0).then_some(sequence),
9117        Some(format!("{min_supported}..={max_supported}")),
9118        Some(format!("{change_id}:{version}")),
9119        "recorded workflow version is outside the range supported by current code",
9120    )))
9121}
9122
9123#[derive(Clone, Debug)]
9124struct ResumeSignal {
9125    signal_name: String,
9126    arguments: Vec<AvroValue>,
9127}
9128
9129const MAX_PARALLEL_OPERATIONS: usize = 1000;
9130
9131fn parallel_group_prefix(kind: &str) -> &'static str {
9132    match kind {
9133        "activity" => "parallel-activities",
9134        "child" => "parallel-children",
9135        "timer" => "parallel-timers",
9136        _ => "parallel-calls",
9137    }
9138}
9139
9140fn parallel_group_entry(
9141    base_sequence: u64,
9142    size: usize,
9143    index: usize,
9144    kind: &str,
9145) -> ParallelGroupMetadata {
9146    ParallelGroupMetadata {
9147        parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9148        parallel_group_kind: kind.to_string(),
9149        parallel_group_base_sequence: base_sequence,
9150        parallel_group_size: size,
9151        parallel_group_index: index,
9152    }
9153}
9154
9155fn apply_parallel_group_path(
9156    command: &mut serde_json::Map<String, Value>,
9157    path: &[ParallelGroupMetadata],
9158) {
9159    let Some(inner) = path.last() else {
9160        return;
9161    };
9162    command.insert(
9163        "parallel_group_id".to_string(),
9164        json!(inner.parallel_group_id),
9165    );
9166    command.insert(
9167        "parallel_group_kind".to_string(),
9168        json!(inner.parallel_group_kind),
9169    );
9170    command.insert(
9171        "parallel_group_base_sequence".to_string(),
9172        json!(inner.parallel_group_base_sequence),
9173    );
9174    command.insert(
9175        "parallel_group_size".to_string(),
9176        json!(inner.parallel_group_size),
9177    );
9178    command.insert(
9179        "parallel_group_index".to_string(),
9180        json!(inner.parallel_group_index),
9181    );
9182    command.insert("parallel_group_path".to_string(), json!(path));
9183}
9184
9185fn ensure_parallel_path_matches(
9186    sequence: u64,
9187    recorded: Option<&[ParallelGroupMetadata]>,
9188    expected: &[ParallelGroupMetadata],
9189) -> Result<()> {
9190    match (recorded, expected.is_empty()) {
9191        (None, true) => Ok(()),
9192        (Some(recorded), false) if recorded == expected => Ok(()),
9193        (None, false) => Err(invalid_recorded_history(
9194            "parallel_group_metadata_missing",
9195            sequence,
9196            &serde_json::to_string(expected).unwrap_or_default(),
9197            "<missing>",
9198            "recorded parallel member is missing its durable group path",
9199        )),
9200        (Some(recorded), true) => Err(invalid_recorded_history(
9201            "parallel_group_shape_mismatch",
9202            sequence,
9203            "sequential command",
9204            &serde_json::to_string(recorded).unwrap_or_default(),
9205            "recorded command belonged to a parallel group but current code schedules it sequentially",
9206        )),
9207        (Some(recorded), false) => Err(invalid_recorded_history(
9208            "parallel_group_shape_mismatch",
9209            sequence,
9210            &serde_json::to_string(recorded).unwrap_or_default(),
9211            &serde_json::to_string(expected).unwrap_or_default(),
9212            "recorded parallel-group identity or path changed during replay",
9213        )),
9214    }
9215}
9216
9217#[derive(Clone, Debug)]
9218enum ParallelShape {
9219    Leaf,
9220    Group(Vec<ParallelShape>),
9221}
9222
9223struct ParallelDescriptor {
9224    operation: ParallelOperation,
9225    offset: usize,
9226    member_path: Vec<usize>,
9227    group_path: Vec<ParallelGroupMetadata>,
9228}
9229
9230fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
9231    operations
9232        .iter()
9233        .map(|operation| match operation {
9234            ParallelOperation::Group(children) => parallel_leaf_count(children),
9235            _ => 1,
9236        })
9237        .sum()
9238}
9239
9240fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
9241    match operation {
9242        ParallelOperation::Activity { .. } => Some("activity"),
9243        ParallelOperation::ChildWorkflow { .. } => Some("child"),
9244        ParallelOperation::Timer(_) => Some("timer"),
9245        ParallelOperation::Group(children) => parallel_group_kind(children),
9246    }
9247}
9248
9249fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
9250    let mut kind = None;
9251    for operation in operations {
9252        let Some(operation_kind) = parallel_operation_kind(operation) else {
9253            continue;
9254        };
9255        match kind {
9256            None => kind = Some(operation_kind),
9257            Some(current) if current == operation_kind => {}
9258            Some(_) => return Some("mixed"),
9259        }
9260    }
9261    kind
9262}
9263
9264fn validate_parallel_operations(
9265    operations: &[ParallelOperation],
9266    member_path: &mut Vec<usize>,
9267    root: bool,
9268) -> Result<()> {
9269    let leaves = parallel_leaf_count(operations);
9270    if leaves > MAX_PARALLEL_OPERATIONS {
9271        return Err(Error::InvalidParallelGroup(ParallelGroupError {
9272            reason: "fan_out_limit_exceeded",
9273            member_path: member_path.clone(),
9274            message: format!(
9275                "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
9276            ),
9277        }));
9278    }
9279    if !root && operations.is_empty() {
9280        return Err(Error::InvalidParallelGroup(ParallelGroupError {
9281            reason: "nested_group_empty",
9282            member_path: member_path.clone(),
9283            message: "a nested group must contain at least one durable leaf".to_string(),
9284        }));
9285    }
9286
9287    for (index, operation) in operations.iter().enumerate() {
9288        member_path.push(index);
9289        match operation {
9290            ParallelOperation::Activity {
9291                options, arguments, ..
9292            } => {
9293                options
9294                    .validate()
9295                    .map_err(|error| Error::InvalidActivityOptions(error))?;
9296                if let Err(error) = arguments {
9297                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
9298                        reason: "arguments_invalid",
9299                        member_path: member_path.clone(),
9300                        message: error.to_string(),
9301                    }));
9302                }
9303            }
9304            ParallelOperation::ChildWorkflow {
9305                options, arguments, ..
9306            } => {
9307                validate_parallel_child_options(options)?;
9308                if let Err(error) = arguments {
9309                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
9310                        reason: "arguments_invalid",
9311                        member_path: member_path.clone(),
9312                        message: error.to_string(),
9313                    }));
9314                }
9315            }
9316            ParallelOperation::Timer(duration)
9317                if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
9318            {
9319                return Err(Error::TimerDurationOverflow);
9320            }
9321            ParallelOperation::Timer(_) => {}
9322            ParallelOperation::Group(children) => {
9323                validate_parallel_operations(children, member_path, false)?;
9324            }
9325        }
9326        member_path.pop();
9327    }
9328    Ok(())
9329}
9330
9331fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
9332    if options.task_queue.trim().is_empty() {
9333        return Err(Error::InvalidChildWorkflowOptions(
9334            "task_queue must not be empty".to_string(),
9335        ));
9336    }
9337    for (name, value) in [
9338        (
9339            "execution_timeout_seconds",
9340            options.execution_timeout_seconds,
9341        ),
9342        ("run_timeout_seconds", options.run_timeout_seconds),
9343    ] {
9344        if value == Some(0) {
9345            return Err(Error::InvalidChildWorkflowOptions(format!(
9346                "{name} must be at least 1"
9347            )));
9348        }
9349    }
9350    if options
9351        .retry_policy
9352        .as_ref()
9353        .is_some_and(|policy| policy.max_attempts == Some(0))
9354    {
9355        return Err(Error::InvalidChildWorkflowOptions(
9356            "retry_policy.max_attempts must be at least 1".to_string(),
9357        ));
9358    }
9359    Ok(())
9360}
9361
9362fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
9363    ParallelShape::Group(
9364        operations
9365            .iter()
9366            .map(|operation| match operation {
9367                ParallelOperation::Group(children) => parallel_shape(children),
9368                _ => ParallelShape::Leaf,
9369            })
9370            .collect(),
9371    )
9372}
9373
9374fn parallel_descriptors(
9375    operations: Vec<ParallelOperation>,
9376    base_sequence: u64,
9377) -> Result<Vec<ParallelDescriptor>> {
9378    let size = parallel_leaf_count(&operations);
9379    let kind = parallel_group_kind(&operations).unwrap_or("activity");
9380    let mut descriptors = Vec::with_capacity(size);
9381    let mut cursor = 0;
9382
9383    for (index, operation) in operations.into_iter().enumerate() {
9384        match operation {
9385            ParallelOperation::Group(children) => {
9386                let child_base = base_sequence
9387                    .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
9388                    .ok_or(Error::TimerDurationOverflow)?;
9389                for mut descriptor in parallel_descriptors(children, child_base)? {
9390                    let outer_index = cursor + descriptor.offset;
9391                    descriptor.group_path.insert(
9392                        0,
9393                        parallel_group_entry(base_sequence, size, outer_index, kind),
9394                    );
9395                    descriptor.member_path.insert(0, index);
9396                    descriptor.offset = outer_index;
9397                    descriptors.push(descriptor);
9398                }
9399                cursor = descriptors.len();
9400            }
9401            operation => {
9402                descriptors.push(ParallelDescriptor {
9403                    operation,
9404                    offset: cursor,
9405                    member_path: vec![index],
9406                    group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
9407                });
9408                cursor += 1;
9409            }
9410        }
9411    }
9412    Ok(descriptors)
9413}
9414
9415enum ParallelLeafCall {
9416    Activity(ActivityCall),
9417    ChildWorkflow(ChildWorkflowCall),
9418    Timer(TimerCall),
9419}
9420
9421impl ParallelLeafCall {
9422    fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
9423        match self {
9424            Self::Activity(call) => Pin::new(call)
9425                .poll_avro_value(cx)
9426                .map_ok(ParallelAvroResult::Activity),
9427            Self::ChildWorkflow(call) => Pin::new(call)
9428                .poll_avro_value(cx)
9429                .map_ok(ParallelAvroResult::ChildWorkflow),
9430            Self::Timer(call) => Pin::new(call)
9431                .poll(cx)
9432                .map_ok(|()| ParallelAvroResult::Timer),
9433        }
9434    }
9435}
9436
9437struct ParallelLeaf {
9438    call: ParallelLeafCall,
9439    member_path: Vec<usize>,
9440    group_path: Vec<ParallelGroupMetadata>,
9441    result: Option<ParallelAvroResult>,
9442}
9443
9444/// Future returned by [`WorkflowContext::parallel`].
9445pub struct ParallelCall {
9446    ctx: WorkflowContext,
9447    operations: Option<Vec<ParallelOperation>>,
9448    shape: Option<ParallelShape>,
9449    leaves: Vec<ParallelLeaf>,
9450}
9451
9452impl ParallelCall {
9453    fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
9454        Self {
9455            ctx,
9456            operations: Some(operations),
9457            shape: None,
9458            leaves: Vec::new(),
9459        }
9460    }
9461
9462    fn initialize(&mut self) -> Result<()> {
9463        let operations = self.operations.take().unwrap_or_default();
9464        validate_parallel_operations(&operations, &mut Vec::new(), true)?;
9465        self.shape = Some(parallel_shape(&operations));
9466        if operations.is_empty() {
9467            return Ok(());
9468        }
9469
9470        let base_sequence = {
9471            let state = self
9472                .ctx
9473                .state
9474                .lock()
9475                .map_err(|_| Error::WorkflowStatePoisoned)?;
9476            if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
9477                recorded.sequence()
9478            } else {
9479                let last = state
9480                    .recorded_commands
9481                    .last()
9482                    .map(RecordedCommand::sequence)
9483                    .unwrap_or(0);
9484                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
9485                    .and_then(|sequence| sequence.checked_add(1))
9486                    .ok_or_else(|| {
9487                        Error::InvalidParallelGroup(ParallelGroupError {
9488                            reason: "sequence_overflow",
9489                            member_path: Vec::new(),
9490                            message: "parallel group sequence identity overflowed u64".to_string(),
9491                        })
9492                    })?
9493            }
9494        };
9495
9496        self.leaves = parallel_descriptors(operations, base_sequence)?
9497            .into_iter()
9498            .map(|descriptor| {
9499                let path = descriptor.group_path.clone();
9500                let call = match descriptor.operation {
9501                    ParallelOperation::Activity {
9502                        activity_type,
9503                        options,
9504                        arguments,
9505                    } => ParallelLeafCall::Activity(ActivityCall {
9506                        ctx: self.ctx.clone(),
9507                        activity_type,
9508                        options,
9509                        args: Some(arguments),
9510                        scheduled: false,
9511                        parallel_group_path: path,
9512                    }),
9513                    ParallelOperation::ChildWorkflow {
9514                        workflow_type,
9515                        options,
9516                        arguments,
9517                    } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
9518                        ctx: self.ctx.clone(),
9519                        workflow_type,
9520                        options,
9521                        args: Some(arguments),
9522                        scheduled: false,
9523                        matched_pending: false,
9524                        parallel_group_path: path,
9525                    }),
9526                    ParallelOperation::Timer(duration) => {
9527                        let delay_seconds = duration
9528                            .as_secs()
9529                            .checked_add(u64::from(duration.subsec_nanos() > 0));
9530                        ParallelLeafCall::Timer(TimerCall {
9531                            ctx: self.ctx.clone(),
9532                            delay_seconds,
9533                            scheduled: false,
9534                            matched_pending: false,
9535                            parallel_group_path: path,
9536                        })
9537                    }
9538                    ParallelOperation::Group(_) => {
9539                        unreachable!("parallel descriptors contain only durable leaves")
9540                    }
9541                };
9542                ParallelLeaf {
9543                    call,
9544                    member_path: descriptor.member_path,
9545                    group_path: descriptor.group_path,
9546                    result: None,
9547                }
9548            })
9549            .collect();
9550        Ok(())
9551    }
9552
9553    fn poll_avro_value(
9554        mut self: Pin<&mut Self>,
9555        cx: &mut TaskContext<'_>,
9556    ) -> Poll<Result<Vec<ParallelAvroResult>>> {
9557        if self.operations.is_some() {
9558            if let Err(error) = self.initialize() {
9559                return Poll::Ready(Err(error));
9560            }
9561        }
9562        if self.leaves.is_empty() {
9563            return Poll::Ready(Ok(Vec::new()));
9564        }
9565
9566        let mut failures = Vec::new();
9567        let mut pending = false;
9568        for (index, leaf) in self.leaves.iter_mut().enumerate() {
9569            if leaf.result.is_some() {
9570                continue;
9571            }
9572            match leaf.call.poll_avro_value(cx) {
9573                Poll::Ready(Ok(result)) => leaf.result = Some(result),
9574                Poll::Ready(Err(error)) => failures.push((index, error)),
9575                Poll::Pending => pending = true,
9576            }
9577        }
9578
9579        if !failures.is_empty() {
9580            if let Some(position) = failures
9581                .iter()
9582                .position(|(_, error)| workflow_task_integrity_error(error))
9583            {
9584                return Poll::Ready(Err(failures.remove(position).1));
9585            }
9586            failures.sort_by_key(|(index, _)| *index);
9587            let (failed_index, cause) = failures.remove(0);
9588            let failed = &self.leaves[failed_index];
9589            let completed = self
9590                .leaves
9591                .iter()
9592                .filter_map(|leaf| {
9593                    leaf.result
9594                        .clone()
9595                        .and_then(|result| result.into_json_result().ok())
9596                        .map(|result| ParallelCompletion {
9597                            member_path: leaf.member_path.clone(),
9598                            result,
9599                        })
9600                })
9601                .collect();
9602            let group_id = failed
9603                .group_path
9604                .first()
9605                .map(|entry| entry.parallel_group_id.clone())
9606                .unwrap_or_default();
9607            return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
9608                group_id,
9609                member_path: failed.member_path.clone(),
9610                group_path: failed.group_path.clone(),
9611                completed,
9612                cause: Box::new(cause),
9613            })));
9614        }
9615        if pending {
9616            return Poll::Pending;
9617        }
9618
9619        let mut flat_results = self
9620            .leaves
9621            .iter_mut()
9622            .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
9623            .collect::<Vec<_>>()
9624            .into_iter();
9625        let results = parallel_results_for_shape(
9626            self.shape.as_ref().expect("initialized parallel shape"),
9627            &mut flat_results,
9628        );
9629        Poll::Ready(Ok(match results {
9630            ParallelAvroResult::Group(results) => results,
9631            ParallelAvroResult::Activity(_)
9632            | ParallelAvroResult::ChildWorkflow(_)
9633            | ParallelAvroResult::Timer => unreachable!("root parallel shape is a group"),
9634        }))
9635    }
9636}
9637
9638fn parallel_results_for_shape(
9639    shape: &ParallelShape,
9640    flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
9641) -> ParallelAvroResult {
9642    match shape {
9643        ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
9644        ParallelShape::Group(children) => ParallelAvroResult::Group(
9645            children
9646                .iter()
9647                .map(|child| parallel_results_for_shape(child, flat_results))
9648                .collect(),
9649        ),
9650    }
9651}
9652
9653impl Future for ParallelCall {
9654    type Output = Result<Vec<ParallelResult>>;
9655
9656    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9657        self.poll_avro_value(cx)
9658            .map_ok(|results| {
9659                results
9660                    .into_iter()
9661                    .map(ParallelAvroResult::into_json_result)
9662                    .collect::<Result<Vec<_>>>()
9663            })
9664            .map_ok(|result| result)
9665            .flatten_result()
9666    }
9667}
9668
9669trait PollNestedResultExt<T> {
9670    fn flatten_result(self) -> Poll<Result<T>>;
9671}
9672
9673impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
9674    fn flatten_result(self) -> Poll<Result<T>> {
9675        match self {
9676            Poll::Ready(Ok(result)) => Poll::Ready(result),
9677            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
9678            Poll::Pending => Poll::Pending,
9679        }
9680    }
9681}
9682
9683struct SagaCompensation {
9684    activity_type: String,
9685    options: ActivityOptions,
9686    arguments: AvroValue,
9687    registration_order: usize,
9688}
9689
9690/// Workflow-local deterministic saga compensation helper.
9691///
9692/// Register each compensation only after its forward step succeeds. Passing
9693/// the forward `Result` to [`Saga::finish`] runs compensations sequentially in
9694/// reverse registration order after any failure, including cooperative
9695/// cancellation. Each compensation is an ordinary durable activity, so replay,
9696/// duplicate delivery, and worker restart use existing history semantics.
9697pub struct Saga {
9698    ctx: WorkflowContext,
9699    compensations: Vec<SagaCompensation>,
9700}
9701
9702impl Saga {
9703    fn new(ctx: WorkflowContext) -> Self {
9704        Self {
9705            ctx,
9706            compensations: Vec::new(),
9707        }
9708    }
9709
9710    pub fn add_compensation<T: Serialize>(
9711        &mut self,
9712        activity_type: impl Into<String>,
9713        args: T,
9714    ) -> Result<&mut Self> {
9715        self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
9716    }
9717
9718    pub fn add_compensation_with_options<T: Serialize>(
9719        &mut self,
9720        activity_type: impl Into<String>,
9721        options: ActivityOptions,
9722        args: T,
9723    ) -> Result<&mut Self> {
9724        let activity_type = activity_type.into();
9725        if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
9726            return Err(Error::Codec(
9727                "saga compensation activity type must be non-empty without surrounding whitespace"
9728                    .to_string(),
9729            ));
9730        }
9731        options.validate().map_err(Error::InvalidActivityOptions)?;
9732        let arguments = AvroValue::from_serialize(&args)?;
9733        let registration_order = self.compensations.len() + 1;
9734        self.compensations.push(SagaCompensation {
9735            activity_type,
9736            options,
9737            arguments,
9738            registration_order,
9739        });
9740        Ok(self)
9741    }
9742
9743    /// Compensate `initiating_failure` and return the failure that must remain.
9744    pub async fn compensate(mut self, initiating_failure: Error) -> Error {
9745        while let Some(compensation) = self.compensations.pop() {
9746            if let Err(compensation_failure) = self
9747                .ctx
9748                .activity_with_options(
9749                    compensation.activity_type.clone(),
9750                    compensation.options,
9751                    compensation.arguments,
9752                )
9753                .await
9754            {
9755                if workflow_task_integrity_error(&compensation_failure) {
9756                    return compensation_failure;
9757                }
9758                return Error::SagaCompensationFailed(SagaCompensationFailure {
9759                    initiating_failure: Box::new(initiating_failure),
9760                    compensation_failure: Box::new(compensation_failure),
9761                    compensation_activity_type: compensation.activity_type,
9762                    compensation_registration_order: compensation.registration_order,
9763                });
9764            }
9765        }
9766        initiating_failure
9767    }
9768
9769    /// Return a successful forward value or compensate and preserve its failure.
9770    pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
9771        match outcome {
9772            Ok(value) => Ok(value),
9773            Err(error) => Err(self.compensate(error).await),
9774        }
9775    }
9776}
9777
9778pub struct ActivityCall {
9779    ctx: WorkflowContext,
9780    activity_type: String,
9781    options: ActivityOptions,
9782    args: Option<Result<AvroValue>>,
9783    scheduled: bool,
9784    parallel_group_path: Vec<ParallelGroupMetadata>,
9785}
9786
9787impl ActivityCall {
9788    fn poll_avro_value(
9789        mut self: Pin<&mut Self>,
9790        _cx: &mut TaskContext<'_>,
9791    ) -> Poll<Result<AvroValue>> {
9792        let ctx = self.ctx.clone();
9793        let mut state = match ctx.state.lock() {
9794            Ok(state) => state,
9795            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9796        };
9797
9798        if self.scheduled {
9799            return Poll::Pending;
9800        }
9801
9802        let options = match self.options.validate() {
9803            Ok(options) => options,
9804            Err(error) => {
9805                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
9806            }
9807        };
9808        let task_queue = options
9809            .task_queue
9810            .clone()
9811            .unwrap_or_else(|| state.task_queue.clone());
9812        let current_recorded_options = RecordedActivityOptions {
9813            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
9814            // Rust schedules ordinary durable activities. The server records a
9815            // non-null mode only for a specialized execution primitive.
9816            execution_mode: RecordedSnapshotValue::Known(None),
9817            retry_policy: current_activity_retry_snapshot(&options),
9818        };
9819
9820        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9821            let sequence = recorded.sequence();
9822            match recorded {
9823                RecordedCommand::Activity {
9824                    activity_type,
9825                    options: recorded_options,
9826                    outcome,
9827                    parallel_group_path,
9828                    ..
9829                } => {
9830                    if let Err(error) = ensure_parallel_path_matches(
9831                        sequence,
9832                        parallel_group_path.as_deref(),
9833                        &self.parallel_group_path,
9834                    ) {
9835                        return Poll::Ready(Err(error));
9836                    }
9837                    if let Some(recorded_type) = activity_type {
9838                        if recorded_type != self.activity_type {
9839                            return Poll::Ready(Err(Error::NonDeterministicReplay(
9840                                ReplayFailure::new(
9841                                    "recorded_command_detail_mismatch",
9842                                    Some(sequence),
9843                                    Some(format!("activity:{recorded_type}")),
9844                                    Some(format!("activity:{}", self.activity_type)),
9845                                    "recorded activity type differs from the current workflow command",
9846                                ),
9847                            )));
9848                        }
9849                    }
9850                    if let Some(recorded_options) = recorded_options {
9851                        if !recorded_options
9852                            .task_queue
9853                            .matches_current(&current_recorded_options.task_queue)
9854                        {
9855                            return Poll::Ready(Err(Error::NonDeterministicReplay(
9856                                ReplayFailure::new(
9857                                    "activity_task_queue_mismatch",
9858                                    Some(sequence),
9859                                    Some(activity_options_description(&recorded_options)),
9860                                    Some(activity_options_description(&current_recorded_options)),
9861                                    "recorded activity task queue differs from the current workflow command",
9862                                ),
9863                            )));
9864                        }
9865                        if !recorded_options
9866                            .execution_mode
9867                            .matches_current(&current_recorded_options.execution_mode)
9868                        {
9869                            return Poll::Ready(Err(Error::NonDeterministicReplay(
9870                                ReplayFailure::new(
9871                                    "activity_execution_mode_mismatch",
9872                                    Some(sequence),
9873                                    Some(activity_options_description(&recorded_options)),
9874                                    Some(activity_options_description(&current_recorded_options)),
9875                                    "recorded activity execution mode differs from the current workflow command",
9876                                ),
9877                            )));
9878                        }
9879                        if !recorded_options
9880                            .retry_policy
9881                            .matches_current(&current_recorded_options.retry_policy)
9882                        {
9883                            return Poll::Ready(Err(Error::NonDeterministicReplay(
9884                                ReplayFailure::new(
9885                                    "activity_retry_policy_mismatch",
9886                                    Some(sequence),
9887                                    Some(activity_options_description(&recorded_options)),
9888                                    Some(activity_options_description(&current_recorded_options)),
9889                                    "recorded activity retry policy differs from the current workflow command",
9890                                ),
9891                            )));
9892                        }
9893                    }
9894                    state.command_cursor += 1;
9895                    if let Some(outcome) = outcome {
9896                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
9897                    }
9898                    state.matched_recorded_pending = true;
9899                    self.scheduled = true;
9900                    return Poll::Pending;
9901                }
9902                other => {
9903                    return Poll::Ready(Err(command_mismatch(
9904                        &other,
9905                        format!("activity:{}", self.activity_type),
9906                    )));
9907                }
9908            }
9909        }
9910
9911        if !self.scheduled {
9912            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
9913                Ok(args) => args,
9914                Err(error) => return Poll::Ready(Err(error)),
9915            };
9916            let arguments = normalize_avro_arguments(args);
9917            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
9918                Ok(envelope) => envelope,
9919                Err(error) => return Poll::Ready(Err(error)),
9920            };
9921
9922            let mut command = serde_json::Map::from_iter([
9923                ("type".to_string(), json!("schedule_activity")),
9924                (
9925                    "activity_type".to_string(),
9926                    json!(self.activity_type.clone()),
9927                ),
9928                ("queue".to_string(), json!(task_queue)),
9929                ("arguments".to_string(), envelope),
9930            ]);
9931            for (field, value) in [
9932                ("start_to_close_timeout", options.start_to_close_timeout),
9933                (
9934                    "schedule_to_start_timeout",
9935                    options.schedule_to_start_timeout,
9936                ),
9937                (
9938                    "schedule_to_close_timeout",
9939                    options.schedule_to_close_timeout,
9940                ),
9941                ("heartbeat_timeout", options.heartbeat_timeout),
9942            ] {
9943                if let Some(value) = value {
9944                    command.insert(field.to_string(), json!(value));
9945                }
9946            }
9947            if let Some(retry_policy) = options.retry_policy {
9948                command.insert("retry_policy".to_string(), retry_policy);
9949            }
9950            apply_parallel_group_path(&mut command, &self.parallel_group_path);
9951            state.commands.push(Value::Object(command));
9952            self.scheduled = true;
9953        }
9954
9955        Poll::Pending
9956    }
9957}
9958
9959impl Future for ActivityCall {
9960    type Output = Result<Value>;
9961
9962    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9963        match self.poll_avro_value(cx) {
9964            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
9965            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
9966            Poll::Pending => Poll::Pending,
9967        }
9968    }
9969}
9970
9971/// Future returned by [`WorkflowContext::sleep`].
9972pub struct TimerCall {
9973    ctx: WorkflowContext,
9974    delay_seconds: Option<u64>,
9975    scheduled: bool,
9976    matched_pending: bool,
9977    parallel_group_path: Vec<ParallelGroupMetadata>,
9978}
9979
9980impl Future for TimerCall {
9981    type Output = Result<()>;
9982
9983    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9984        if self.matched_pending {
9985            return Poll::Pending;
9986        }
9987
9988        let ctx = self.ctx.clone();
9989        let Some(requested_delay) = self.delay_seconds else {
9990            return Poll::Ready(Err(Error::TimerDurationOverflow));
9991        };
9992        let mut state = match ctx.state.lock() {
9993            Ok(state) => state,
9994            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9995        };
9996
9997        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9998            match recorded {
9999                RecordedCommand::Timer {
10000                    sequence,
10001                    delay_seconds,
10002                    fired,
10003                    parallel_group_path,
10004                    ..
10005                } => {
10006                    if let Err(error) = ensure_parallel_path_matches(
10007                        sequence,
10008                        parallel_group_path.as_deref(),
10009                        &self.parallel_group_path,
10010                    ) {
10011                        return Poll::Ready(Err(error));
10012                    }
10013                    if delay_seconds != requested_delay {
10014                        return Poll::Ready(Err(Error::NonDeterministicReplay(
10015                            ReplayFailure::new(
10016                                "timer_delay_mismatch",
10017                                Some(sequence),
10018                                Some(format!("timer:{delay_seconds}s")),
10019                                Some(format!("timer:{requested_delay}s")),
10020                                "recorded timer delay differs from the current workflow command",
10021                            ),
10022                        )));
10023                    }
10024                    state.command_cursor += 1;
10025                    if fired {
10026                        return Poll::Ready(Ok(()));
10027                    }
10028                    state.matched_recorded_pending = true;
10029                    self.scheduled = true;
10030                    self.matched_pending = true;
10031                    return Poll::Pending;
10032                }
10033                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
10034            }
10035        }
10036
10037        if !self.scheduled {
10038            let mut command = serde_json::Map::from_iter([
10039                ("type".to_string(), json!("start_timer")),
10040                ("delay_seconds".to_string(), json!(requested_delay)),
10041            ]);
10042            apply_parallel_group_path(&mut command, &self.parallel_group_path);
10043            state.commands.push(Value::Object(command));
10044            self.scheduled = true;
10045        }
10046
10047        Poll::Pending
10048    }
10049}
10050
10051/// Future returned by [`WorkflowContext::wait_condition`].
10052pub struct ConditionWaitCall {
10053    ctx: WorkflowContext,
10054    options: ConditionWaitOptions,
10055    predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
10056    occurrence_id: Option<String>,
10057    opened_wait: bool,
10058}
10059
10060impl Future for ConditionWaitCall {
10061    type Output = Result<ConditionWaitResult>;
10062
10063    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10064        if self.opened_wait {
10065            return Poll::Pending;
10066        }
10067
10068        let options = match self.options.validate() {
10069            Ok(options) => options,
10070            Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
10071        };
10072        let ctx = self.ctx.clone();
10073        let occurrence_id = match self.occurrence_id.as_ref() {
10074            Some(occurrence_id) => occurrence_id.clone(),
10075            None => {
10076                let mut state = match ctx.state.lock() {
10077                    Ok(state) => state,
10078                    Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10079                };
10080                let ordinal = state.condition_wait_occurrence_counter;
10081                state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
10082                    Some(next) => next,
10083                    None => {
10084                        return Poll::Ready(Err(Error::WorkerLoop(
10085                            "condition wait occurrence counter overflowed".to_string(),
10086                        )))
10087                    }
10088                };
10089                let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
10090                drop(state);
10091                self.occurrence_id = Some(occurrence_id.clone());
10092                occurrence_id
10093            }
10094        };
10095
10096        let recorded_result = {
10097            let mut state = match ctx.state.lock() {
10098                Ok(state) => state,
10099                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10100            };
10101            let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
10102                drop(state);
10103                return self.poll_new_condition(options);
10104            };
10105            if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
10106                return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
10107            }
10108
10109            let mut cursor = state.command_cursor;
10110            let mut result = None;
10111            loop {
10112                let Some(RecordedCommand::ConditionWait {
10113                    sequence,
10114                    occurrence_id: recorded_occurrence_id,
10115                    condition_key,
10116                    predicate_identity,
10117                    timeout_seconds,
10118                    result: recorded_result,
10119                    ..
10120                }) = state.recorded_commands.get(cursor)
10121                else {
10122                    break;
10123                };
10124
10125                if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
10126                    break;
10127                }
10128                if let Err(error) = validate_recorded_condition_wait(
10129                    *sequence,
10130                    recorded_occurrence_id,
10131                    condition_key.as_deref(),
10132                    predicate_identity,
10133                    *timeout_seconds,
10134                    &occurrence_id,
10135                    &options,
10136                ) {
10137                    return Poll::Ready(Err(error));
10138                }
10139                if result == Some(ConditionWaitResult::TimedOut) {
10140                    return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
10141                        "condition_wait_reopened_after_timeout",
10142                        Some(*sequence),
10143                        Some("timed-out condition is terminal".to_string()),
10144                        Some("another physical wait-open".to_string()),
10145                        "condition history reopened one logical wait after its durable timeout",
10146                    ))));
10147                }
10148                result = *recorded_result;
10149                cursor += 1;
10150            }
10151            state.command_cursor = cursor;
10152            result
10153        };
10154
10155        if let Some(result) = recorded_result {
10156            return Poll::Ready(Ok(result));
10157        }
10158
10159        self.poll_open_condition(options)
10160    }
10161}
10162
10163impl ConditionWaitCall {
10164    fn poll_new_condition(
10165        self: Pin<&mut Self>,
10166        options: ValidatedConditionWaitOptions,
10167    ) -> Poll<Result<ConditionWaitResult>> {
10168        self.poll_open_condition(options)
10169    }
10170
10171    fn poll_open_condition(
10172        mut self: Pin<&mut Self>,
10173        options: ValidatedConditionWaitOptions,
10174    ) -> Poll<Result<ConditionWaitResult>> {
10175        match (self.predicate)() {
10176            Ok(true) => return Poll::Ready(Ok(ConditionWaitResult::Satisfied)),
10177            Ok(false) => {}
10178            Err(error) => return Poll::Ready(Err(error)),
10179        }
10180        if options.timeout_seconds == Some(0) {
10181            return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
10182        }
10183
10184        let ctx = self.ctx.clone();
10185        let mut state = match ctx.state.lock() {
10186            Ok(state) => state,
10187            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10188        };
10189        let mut command = serde_json::Map::from_iter([
10190            ("type".to_string(), json!("open_condition_wait")),
10191            (
10192                "condition_wait_occurrence_id".to_string(),
10193                json!(self.occurrence_id.as_deref().unwrap_or_default()),
10194            ),
10195            ("condition_key".to_string(), json!(options.condition_key)),
10196            (
10197                "condition_definition_fingerprint".to_string(),
10198                json!(options.predicate_identity),
10199            ),
10200        ]);
10201        if let Some(timeout_seconds) = options.timeout_seconds {
10202            command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
10203        }
10204        state.commands.push(Value::Object(command));
10205        drop(state);
10206        self.opened_wait = true;
10207        Poll::Pending
10208    }
10209}
10210
10211fn validate_recorded_condition_wait(
10212    sequence: u64,
10213    recorded_occurrence_id: &str,
10214    recorded_key: Option<&str>,
10215    recorded_predicate_identity: &str,
10216    recorded_timeout_seconds: Option<u64>,
10217    current_occurrence_id: &str,
10218    current: &ValidatedConditionWaitOptions,
10219) -> Result<()> {
10220    if recorded_occurrence_id != current_occurrence_id {
10221        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10222            "condition_wait_occurrence_mismatch",
10223            Some(sequence),
10224            Some(recorded_occurrence_id.to_string()),
10225            Some(current_occurrence_id.to_string()),
10226            "recorded condition occurrence differs from the current authored wait position",
10227        )));
10228    }
10229    if recorded_key != Some(current.condition_key.as_str()) {
10230        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10231            "condition_wait_key_mismatch",
10232            Some(sequence),
10233            recorded_key.map(str::to_string),
10234            Some(current.condition_key.clone()),
10235            "recorded condition identity differs from the current workflow wait",
10236        )));
10237    }
10238    if recorded_predicate_identity != current.predicate_identity {
10239        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10240            "condition_wait_predicate_mismatch",
10241            Some(sequence),
10242            Some(recorded_predicate_identity.to_string()),
10243            Some(current.predicate_identity.clone()),
10244            "recorded condition predicate behavior differs from current workflow code",
10245        )));
10246    }
10247    if recorded_timeout_seconds != current.timeout_seconds {
10248        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10249            "condition_wait_timeout_mismatch",
10250            Some(sequence),
10251            recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
10252            current.timeout_seconds.map(|seconds| format!("{seconds}s")),
10253            "recorded condition timeout differs from the current workflow wait",
10254        )));
10255    }
10256    Ok(())
10257}
10258
10259/// Future returned by [`WorkflowContext::start_child_workflow`].
10260pub struct ChildWorkflowCall {
10261    ctx: WorkflowContext,
10262    workflow_type: String,
10263    options: ChildWorkflowOptions,
10264    args: Option<Result<AvroValue>>,
10265    scheduled: bool,
10266    matched_pending: bool,
10267    parallel_group_path: Vec<ParallelGroupMetadata>,
10268}
10269
10270impl ChildWorkflowCall {
10271    fn poll_avro_value(
10272        mut self: Pin<&mut Self>,
10273        _cx: &mut TaskContext<'_>,
10274    ) -> Poll<Result<ChildWorkflowAvroResult>> {
10275        if self.matched_pending {
10276            return Poll::Pending;
10277        }
10278
10279        let ctx = self.ctx.clone();
10280        let mut state = match ctx.state.lock() {
10281            Ok(state) => state,
10282            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10283        };
10284
10285        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10286            let sequence = recorded.sequence();
10287            match recorded {
10288                RecordedCommand::ChildWorkflow {
10289                    workflow_type,
10290                    outcome,
10291                    parallel_group_path,
10292                    ..
10293                } => {
10294                    if let Err(error) = ensure_parallel_path_matches(
10295                        sequence,
10296                        parallel_group_path.as_deref(),
10297                        &self.parallel_group_path,
10298                    ) {
10299                        return Poll::Ready(Err(error));
10300                    }
10301                    if let Some(recorded_type) = workflow_type {
10302                        if recorded_type != self.workflow_type {
10303                            return Poll::Ready(Err(Error::NonDeterministicReplay(
10304                                ReplayFailure::new(
10305                                    "recorded_command_detail_mismatch",
10306                                    Some(sequence),
10307                                    Some(format!("child workflow:{recorded_type}")),
10308                                    Some(format!("child workflow:{}", self.workflow_type)),
10309                                    "recorded child workflow type differs from the current workflow command",
10310                                ),
10311                            )));
10312                        }
10313                    }
10314                    state.command_cursor += 1;
10315                    if let Some(outcome) = outcome {
10316                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
10317                    }
10318                    state.matched_recorded_pending = true;
10319                    self.scheduled = true;
10320                    self.matched_pending = true;
10321                    return Poll::Pending;
10322                }
10323                other => {
10324                    return Poll::Ready(Err(command_mismatch(
10325                        &other,
10326                        format!("child workflow:{}", self.workflow_type),
10327                    )));
10328                }
10329            }
10330        }
10331
10332        if !self.scheduled {
10333            if self.options.task_queue.trim().is_empty() {
10334                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
10335                    "task_queue must not be empty".to_string(),
10336                )));
10337            }
10338            for (name, value) in [
10339                (
10340                    "execution_timeout_seconds",
10341                    self.options.execution_timeout_seconds,
10342                ),
10343                ("run_timeout_seconds", self.options.run_timeout_seconds),
10344            ] {
10345                if value == Some(0) {
10346                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
10347                        "{name} must be at least 1"
10348                    ))));
10349                }
10350            }
10351
10352            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
10353                Ok(args) => args,
10354                Err(error) => return Poll::Ready(Err(error)),
10355            };
10356            let arguments = match encode_typed_envelope(
10357                &normalize_avro_arguments(args),
10358                &state.payload_codec,
10359            ) {
10360                Ok(arguments) => arguments,
10361                Err(error) => return Poll::Ready(Err(error)),
10362            };
10363            let mut command = json!({
10364                "type": "start_child_workflow",
10365                "workflow_type": self.workflow_type,
10366                "queue": self.options.task_queue,
10367                "parent_close_policy": self.options.parent_close_policy.as_str(),
10368                "arguments": arguments,
10369            });
10370            let object = command
10371                .as_object_mut()
10372                .expect("child workflow command is always an object");
10373            if let Some(policy) = &self.options.retry_policy {
10374                let mut retry_policy = serde_json::Map::new();
10375                if let Some(max_attempts) = policy.max_attempts {
10376                    if max_attempts == 0 {
10377                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
10378                            "retry_policy.max_attempts must be at least 1".to_string(),
10379                        )));
10380                    }
10381                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
10382                }
10383                if !policy.backoff_seconds.is_empty() {
10384                    retry_policy
10385                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
10386                }
10387                if !policy.non_retryable_error_types.is_empty() {
10388                    retry_policy.insert(
10389                        "non_retryable_error_types".to_string(),
10390                        json!(policy.non_retryable_error_types),
10391                    );
10392                }
10393                if retry_policy.is_empty() {
10394                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
10395                        "retry_policy must configure at least one field".to_string(),
10396                    )));
10397                }
10398                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
10399            }
10400            if let Some(seconds) = self.options.execution_timeout_seconds {
10401                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
10402            }
10403            if let Some(seconds) = self.options.run_timeout_seconds {
10404                object.insert("run_timeout_seconds".to_string(), json!(seconds));
10405            }
10406            apply_parallel_group_path(object, &self.parallel_group_path);
10407            state.commands.push(command);
10408            self.scheduled = true;
10409        }
10410
10411        Poll::Pending
10412    }
10413}
10414
10415impl Future for ChildWorkflowCall {
10416    type Output = Result<ChildWorkflowResult>;
10417
10418    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10419        match self.poll_avro_value(cx) {
10420            Poll::Ready(Ok(result)) => match result.result.into_json() {
10421                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
10422                    parent: result.parent,
10423                    child: result.child,
10424                    child_workflow_type: result.child_workflow_type,
10425                    result: projected,
10426                })),
10427                Err(error) => Poll::Ready(Err(error)),
10428            },
10429            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
10430            Poll::Pending => Poll::Pending,
10431        }
10432    }
10433}
10434
10435fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
10436    Error::NonDeterministicReplay(ReplayFailure::new(
10437        "recorded_command_mismatch",
10438        Some(recorded.sequence()),
10439        Some(recorded.shape().to_string()),
10440        Some(actual.into()),
10441        "current workflow command does not match the recorded durable command sequence",
10442    ))
10443}
10444
10445pub struct SignalCall {
10446    ctx: WorkflowContext,
10447    signal_name: String,
10448    runtime_reserved_allowed: bool,
10449    opened_wait: bool,
10450    matched_pending: bool,
10451}
10452
10453impl SignalCall {
10454    fn poll_avro_value(
10455        mut self: Pin<&mut Self>,
10456        _cx: &mut TaskContext<'_>,
10457    ) -> Poll<Result<Vec<AvroValue>>> {
10458        if self.matched_pending {
10459            return Poll::Pending;
10460        }
10461        if !self.runtime_reserved_allowed {
10462            if let Err(error) = validate_user_signal_name(&self.signal_name) {
10463                return Poll::Ready(Err(error));
10464            }
10465        }
10466
10467        let ctx = self.ctx.clone();
10468        let mut state = match ctx.state.lock() {
10469            Ok(state) => state,
10470            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10471        };
10472
10473        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10474            match recorded {
10475                RecordedCommand::SignalWait {
10476                    sequence,
10477                    signal_name,
10478                    value,
10479                } => {
10480                    if signal_name != self.signal_name {
10481                        return Poll::Ready(Err(Error::NonDeterministicReplay(
10482                            ReplayFailure::new(
10483                                "recorded_command_detail_mismatch",
10484                                Some(sequence),
10485                                Some(format!("signal wait:{signal_name}")),
10486                                Some(format!("signal wait:{}", self.signal_name)),
10487                                "recorded signal name differs from the current workflow command",
10488                            ),
10489                        )));
10490                    }
10491
10492                    state.command_cursor += 1;
10493                    if let Some(value) = value {
10494                        return Poll::Ready(Ok(value));
10495                    }
10496                    if state
10497                        .resume_signal
10498                        .as_ref()
10499                        .is_some_and(|signal| signal.signal_name == self.signal_name)
10500                    {
10501                        let signal = state
10502                            .resume_signal
10503                            .take()
10504                            .expect("matching resume signal is present");
10505                        return Poll::Ready(Ok(signal.arguments));
10506                    }
10507
10508                    state.matched_recorded_pending = true;
10509                    self.opened_wait = true;
10510                    self.matched_pending = true;
10511                    return Poll::Pending;
10512                }
10513                other => {
10514                    return Poll::Ready(Err(command_mismatch(
10515                        &other,
10516                        format!("signal wait:{}", self.signal_name),
10517                    )));
10518                }
10519            }
10520        }
10521
10522        if state
10523            .resume_signal
10524            .as_ref()
10525            .is_some_and(|signal| signal.signal_name == self.signal_name)
10526        {
10527            let signal = state
10528                .resume_signal
10529                .take()
10530                .expect("matching resume signal is present");
10531            return Poll::Ready(Ok(signal.arguments));
10532        }
10533
10534        if !self.opened_wait {
10535            state.commands.push(json!({
10536                "type": "open_signal_wait",
10537                "signal_name": self.signal_name
10538            }));
10539            self.opened_wait = true;
10540        }
10541
10542        Poll::Pending
10543    }
10544}
10545
10546impl Future for SignalCall {
10547    type Output = Result<Vec<Value>>;
10548
10549    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10550        match self.poll_avro_value(cx) {
10551            Poll::Ready(Ok(values)) => Poll::Ready(
10552                values
10553                    .into_iter()
10554                    .map(AvroValue::into_json)
10555                    .collect::<Result<Vec<_>>>(),
10556            ),
10557            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
10558            Poll::Pending => Poll::Pending,
10559        }
10560    }
10561}
10562
10563#[derive(Clone, Debug)]
10564pub struct ActivityContext {
10565    client: Client,
10566    pub task_id: String,
10567    pub activity_attempt_id: String,
10568    pub lease_owner: String,
10569    pub activity_type: String,
10570    pub attempt_number: u64,
10571    pub task_queue: String,
10572    pub worker_id: String,
10573}
10574
10575impl ActivityContext {
10576    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
10577        self.client
10578            .heartbeat_activity_task(
10579                &self.task_id,
10580                &self.activity_attempt_id,
10581                &self.lease_owner,
10582                details,
10583            )
10584            .await
10585    }
10586}
10587
10588fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
10589    validate_payload_codec(codec)?;
10590    match value {
10591        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
10592            value, codec,
10593        )?)),
10594        None => Ok(AvroValue::Array(Vec::new())),
10595    }
10596}
10597
10598fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
10599    let Some(signal_name) = task
10600        .signal_name
10601        .as_deref()
10602        .filter(|value| !value.is_empty())
10603    else {
10604        return Ok(None);
10605    };
10606    let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
10607    let AvroValue::Array(arguments) = decoded else {
10608        unreachable!("normalize_avro_arguments always returns an array");
10609    };
10610
10611    Ok(Some(ResumeSignal {
10612        signal_name: signal_name.to_string(),
10613        arguments,
10614    }))
10615}
10616
10617fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
10618    validate_payload_codec(&task.payload_codec)?;
10619    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
10620    validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
10621    for event in &task.history_events {
10622        validate_history_event_payloads(event, &task.payload_codec)?;
10623    }
10624    Ok(())
10625}
10626
10627fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
10628    validate_payload_codec(&task.payload_codec)?;
10629    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
10630}
10631
10632fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
10633    validate_payload_codec(&task.payload_codec)?;
10634    validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
10635    validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
10636    for event in &task.history_events {
10637        validate_history_event_payloads(event, &task.payload_codec)?;
10638    }
10639
10640    let Some(export) = task.history_export.as_ref() else {
10641        return Ok(());
10642    };
10643    let export_codec = match export.get("payloads") {
10644        Some(payloads) => declared_payload_codec(payloads, "codec")?,
10645        None => None,
10646    }
10647    .unwrap_or(&task.payload_codec);
10648    validate_payload_codec(export_codec)?;
10649
10650    if let Some(events) = export.get("history_events").and_then(Value::as_array) {
10651        for event in events {
10652            let event_type = event
10653                .get("event_type")
10654                .or_else(|| event.get("type"))
10655                .and_then(Value::as_str)
10656                .unwrap_or_default();
10657            if let Some(payload) = event.get("payload") {
10658                validate_history_payloads(event_type, payload, export_codec)?;
10659            }
10660        }
10661    }
10662    for signal in export
10663        .get("signals")
10664        .and_then(Value::as_array)
10665        .into_iter()
10666        .flatten()
10667    {
10668        let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
10669        validate_payload_codec(codec)?;
10670        validate_optional_inbound_payload(signal.get("arguments"), codec)?;
10671    }
10672    for activity in export
10673        .get("activities")
10674        .and_then(Value::as_array)
10675        .into_iter()
10676        .flatten()
10677    {
10678        let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
10679        validate_payload_codec(codec)?;
10680        validate_optional_inbound_payload(activity.get("arguments"), codec)?;
10681        validate_optional_inbound_payload(activity.get("result"), codec)?;
10682    }
10683    Ok(())
10684}
10685
10686fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
10687    validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
10688}
10689
10690fn validate_history_payloads(
10691    event_type: &str,
10692    payload: &Value,
10693    fallback_codec: &str,
10694) -> Result<()> {
10695    let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
10696    validate_payload_codec(codec)?;
10697    for field in history_payload_fields(event_type) {
10698        validate_optional_inbound_payload(payload.get(*field), codec)?;
10699    }
10700    Ok(())
10701}
10702
10703const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
10704
10705fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
10706    match event_type {
10707        "ActivityCompleted" => &["result"],
10708        "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
10709        "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
10710        "UpdateCompleted" | "SideEffectRecorded" => &["result"],
10711        "ChildRunCompleted" => &["result", "output"],
10712        "WorkflowCompleted" => &["output"],
10713        "ServiceCallStarted"
10714        | "ServiceCallCompleted"
10715        | "ServiceCallFailed"
10716        | "ServiceCallCancelled" => &["request_payload", "response_payload"],
10717        _ => &[],
10718    }
10719}
10720
10721fn signal_history_payload(payload: &Value) -> Option<&Value> {
10722    SIGNAL_HISTORY_PAYLOAD_FIELDS
10723        .iter()
10724        .find_map(|field| payload.get(*field))
10725}
10726
10727fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
10728    match value.get(field) {
10729        None => Ok(None),
10730        Some(Value::String(codec)) => Ok(Some(codec)),
10731        Some(_) => Err(invalid_payload_envelope()),
10732    }
10733}
10734
10735fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
10736    validate_payload_codec(codec)?;
10737    if let Some(value) = value.filter(|value| !value.is_null()) {
10738        decode_wire_avro_value(value, codec)?;
10739    }
10740    Ok(())
10741}
10742
10743fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
10744    let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
10745        invalid_recorded_history(
10746            "parallel_group_metadata_invalid",
10747            sequence,
10748            "non-empty parallel_group_id",
10749            &payload.to_string(),
10750            "parallel-group history is missing its stable identity",
10751        )
10752    })?;
10753    let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
10754        invalid_recorded_history(
10755            "parallel_group_metadata_invalid",
10756            sequence,
10757            "activity, child, timer, or mixed group kind",
10758            &payload.to_string(),
10759            "parallel-group history is missing its group kind",
10760        )
10761    })?;
10762    if !matches!(kind.as_str(), "activity" | "child" | "timer" | "mixed") {
10763        return Err(invalid_recorded_history(
10764            "parallel_group_metadata_invalid",
10765            sequence,
10766            "activity, child, timer, or mixed group kind",
10767            &kind,
10768            "parallel-group history contains an unsupported group kind",
10769        ));
10770    }
10771    let base_sequence = payload
10772        .get("parallel_group_base_sequence")
10773        .and_then(value_as_u64)
10774        .filter(|value| *value > 0)
10775        .ok_or_else(|| {
10776            invalid_recorded_history(
10777                "parallel_group_metadata_invalid",
10778                sequence,
10779                "positive parallel_group_base_sequence",
10780                &payload.to_string(),
10781                "parallel-group history contains an invalid base sequence",
10782            )
10783        })?;
10784    let size = payload
10785        .get("parallel_group_size")
10786        .and_then(value_as_u64)
10787        .and_then(|value| usize::try_from(value).ok())
10788        .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
10789        .ok_or_else(|| {
10790            invalid_recorded_history(
10791                "parallel_group_metadata_invalid",
10792                sequence,
10793                "bounded positive parallel_group_size",
10794                &payload.to_string(),
10795                "parallel-group history contains an invalid group size",
10796            )
10797        })?;
10798    let index = payload
10799        .get("parallel_group_index")
10800        .and_then(value_as_u64)
10801        .and_then(|value| usize::try_from(value).ok())
10802        .filter(|value| *value < size)
10803        .ok_or_else(|| {
10804            invalid_recorded_history(
10805                "parallel_group_metadata_invalid",
10806                sequence,
10807                "parallel_group_index within group bounds",
10808                &payload.to_string(),
10809                "parallel-group history contains an invalid member index",
10810            )
10811        })?;
10812    if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
10813        return Err(invalid_recorded_history(
10814            "parallel_group_metadata_invalid",
10815            sequence,
10816            "base sequence plus member index equals workflow sequence",
10817            &payload.to_string(),
10818            "parallel-group path does not preserve durable workflow position",
10819        ));
10820    }
10821    let expected_id = format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind));
10822    if group_id != expected_id {
10823        return Err(invalid_recorded_history(
10824            "parallel_group_metadata_invalid",
10825            sequence,
10826            &expected_id,
10827            &group_id,
10828            "parallel-group history contains an incompatible stable group ID",
10829        ));
10830    }
10831    Ok(ParallelGroupMetadata {
10832        parallel_group_id: group_id,
10833        parallel_group_kind: kind,
10834        parallel_group_base_sequence: base_sequence,
10835        parallel_group_size: size,
10836        parallel_group_index: index,
10837    })
10838}
10839
10840fn recorded_parallel_group_path(
10841    events: &[&HistoryEvent],
10842    sequence: u64,
10843) -> Result<Option<Vec<ParallelGroupMetadata>>> {
10844    let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
10845    for event in events {
10846        let payload = &event.payload;
10847        let has_metadata = payload.get("parallel_group_path").is_some()
10848            || payload.get("parallel_group_id").is_some()
10849            || payload.get("parallel_group_kind").is_some()
10850            || payload.get("parallel_group_base_sequence").is_some()
10851            || payload.get("parallel_group_size").is_some()
10852            || payload.get("parallel_group_index").is_some();
10853        if !has_metadata {
10854            continue;
10855        }
10856
10857        let top_level = recorded_parallel_group_entry(payload, sequence)?;
10858        let path = match payload.get("parallel_group_path") {
10859            None => vec![top_level.clone()],
10860            Some(Value::Array(entries)) if !entries.is_empty() => entries
10861                .iter()
10862                .map(|entry| recorded_parallel_group_entry(entry, sequence))
10863                .collect::<Result<Vec<_>>>()?,
10864            Some(value) => {
10865                return Err(invalid_recorded_history(
10866                    "parallel_group_metadata_invalid",
10867                    sequence,
10868                    "non-empty parallel_group_path list",
10869                    &value.to_string(),
10870                    "parallel-group history contains an invalid group path",
10871                ));
10872            }
10873        };
10874        if path.last() != Some(&top_level) {
10875            return Err(invalid_recorded_history(
10876                "parallel_group_metadata_invalid",
10877                sequence,
10878                &serde_json::to_string(&path.last()).unwrap_or_default(),
10879                &serde_json::to_string(&top_level).unwrap_or_default(),
10880                "parallel-group top-level fields do not match the innermost path entry",
10881            ));
10882        }
10883        if recorded.as_ref().is_some_and(|existing| existing != &path) {
10884            return Err(invalid_recorded_history(
10885                "parallel_group_history_conflict",
10886                sequence,
10887                &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
10888                &serde_json::to_string(&path).unwrap_or_default(),
10889                "parallel-group metadata changed between scheduling and resolution history",
10890            ));
10891        }
10892        recorded = Some(path);
10893    }
10894    Ok(recorded)
10895}
10896
10897fn recorded_commands(
10898    events: &[HistoryEvent],
10899    fallback_codec: &str,
10900    parent: WorkflowIdentity,
10901) -> Result<Vec<RecordedCommand>> {
10902    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
10903    let mut last_new_sequence = None;
10904
10905    for event in events {
10906        let is_activity = matches!(
10907            event.event_type.as_str(),
10908            "ActivityScheduled"
10909                | "ActivityStarted"
10910                | "ActivityHeartbeatRecorded"
10911                | "ActivityRetryScheduled"
10912                | "ActivityCompleted"
10913                | "ActivityFailed"
10914                | "ActivityCancelled"
10915                | "ActivityTimedOut"
10916        );
10917        let is_workflow_timer = matches!(
10918            event.event_type.as_str(),
10919            "TimerScheduled" | "TimerCancelled" | "TimerFired"
10920        ) && !is_internal_timer_event(event);
10921        let is_child_workflow = matches!(
10922            event.event_type.as_str(),
10923            "ChildWorkflowScheduled"
10924                | "ChildRunCompleted"
10925                | "ChildRunFailed"
10926                | "ChildRunCancelled"
10927                | "ChildRunTerminated"
10928        );
10929        let is_signal_wait = is_recorded_signal_wait_event(event);
10930        let is_condition_wait = is_recorded_condition_wait_event(event);
10931        let is_search_attributes = event.event_type == "SearchAttributesUpserted";
10932        let is_side_effect = event.event_type == "SideEffectRecorded";
10933        let is_version_marker = event.event_type == "VersionMarkerRecorded";
10934        let is_memo = event.event_type == "MemoUpserted";
10935        if !is_activity
10936            && !is_workflow_timer
10937            && !is_child_workflow
10938            && !is_signal_wait
10939            && !is_condition_wait
10940            && !is_search_attributes
10941            && !is_side_effect
10942            && !is_version_marker
10943            && !is_memo
10944        {
10945            continue;
10946        }
10947
10948        let sequence = durable_event_sequence(event).ok_or_else(|| {
10949            Error::NonDeterministicReplay(ReplayFailure::new(
10950                "durable_command_sequence_missing",
10951                None,
10952                Some("positive workflow sequence".to_string()),
10953                Some(event.event_type.clone()),
10954                "durable command history event has no workflow sequence",
10955            ))
10956        })?;
10957        if sequence == 0 {
10958            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10959                "durable_command_sequence_invalid",
10960                Some(sequence),
10961                Some("positive workflow sequence".to_string()),
10962                Some(sequence.to_string()),
10963                "durable command history uses an invalid workflow sequence",
10964            )));
10965        }
10966        if !events_by_sequence.contains_key(&sequence) {
10967            if let Some(previous) = last_new_sequence {
10968                if sequence < previous {
10969                    return Err(invalid_recorded_history(
10970                        "durable_command_sequence_mismatch",
10971                        sequence,
10972                        &format!("workflow sequence greater than {previous}"),
10973                        &sequence.to_string(),
10974                        "durable commands are not strictly ordered by their recorded workflow sequence",
10975                    ));
10976                }
10977            }
10978            last_new_sequence = Some(sequence);
10979        }
10980        events_by_sequence.entry(sequence).or_default().push(event);
10981    }
10982
10983    let commands: Vec<RecordedCommand> = events_by_sequence
10984        .into_iter()
10985        .map(|(sequence, sequence_events)| {
10986            let activity_events: Vec<_> = sequence_events
10987                .iter()
10988                .copied()
10989                .filter(|event| event.event_type.starts_with("Activity"))
10990                .collect();
10991            let timer_events: Vec<_> = sequence_events
10992                .iter()
10993                .copied()
10994                .filter(|event| event.event_type.starts_with("Timer"))
10995                .collect();
10996            let child_events: Vec<_> = sequence_events
10997                .iter()
10998                .copied()
10999                .filter(|event| {
11000                    event.event_type == "ChildWorkflowScheduled"
11001                        || event.event_type.starts_with("ChildRun")
11002                })
11003                .collect();
11004            let signal_wait_events: Vec<_> = sequence_events
11005                .iter()
11006                .copied()
11007                .filter(|event| is_recorded_signal_wait_event(event))
11008                .collect();
11009            let condition_wait_events: Vec<_> = sequence_events
11010                .iter()
11011                .copied()
11012                .filter(|event| is_recorded_condition_wait_event(event))
11013                .collect();
11014            let search_attribute_events: Vec<_> = sequence_events
11015                .iter()
11016                .copied()
11017                .filter(|event| event.event_type == "SearchAttributesUpserted")
11018                .collect();
11019            let side_effect_events: Vec<_> = sequence_events
11020                .iter()
11021                .copied()
11022                .filter(|event| event.event_type == "SideEffectRecorded")
11023                .collect();
11024            let version_marker_events: Vec<_> = sequence_events
11025                .iter()
11026                .copied()
11027                .filter(|event| event.event_type == "VersionMarkerRecorded")
11028                .collect();
11029            let memo_events: Vec<_> = sequence_events
11030                .iter()
11031                .copied()
11032                .filter(|event| event.event_type == "MemoUpserted")
11033                .collect();
11034
11035            let command_kind_count = usize::from(!activity_events.is_empty())
11036                + usize::from(!timer_events.is_empty())
11037                + usize::from(!child_events.is_empty())
11038                + usize::from(!signal_wait_events.is_empty())
11039                + usize::from(!condition_wait_events.is_empty())
11040                + usize::from(!search_attribute_events.is_empty())
11041                + usize::from(!side_effect_events.is_empty())
11042                + usize::from(!version_marker_events.is_empty())
11043                + usize::from(!memo_events.is_empty());
11044            if command_kind_count > 1 {
11045                let actual = [
11046                    (!activity_events.is_empty()).then_some("activity"),
11047                    (!timer_events.is_empty()).then_some("timer"),
11048                    (!child_events.is_empty()).then_some("child workflow"),
11049                    (!signal_wait_events.is_empty()).then_some("signal wait"),
11050                    (!condition_wait_events.is_empty()).then_some("condition wait"),
11051                    (!search_attribute_events.is_empty()).then_some("search-attribute update"),
11052                    (!side_effect_events.is_empty()).then_some("side effect"),
11053                    (!version_marker_events.is_empty()).then_some("version marker"),
11054                    (!memo_events.is_empty()).then_some("memo upsert"),
11055                ]
11056                .into_iter()
11057                .flatten()
11058                .collect::<Vec<_>>()
11059                .join(" and ");
11060                return Err(invalid_recorded_history(
11061                    "durable_command_sequence_collision",
11062                    sequence,
11063                    "one durable command kind",
11064                    &actual,
11065                    "one workflow sequence records more than one durable command kind",
11066                ));
11067            }
11068
11069            if !activity_events.is_empty() {
11070                let parallel_group_path =
11071                    recorded_parallel_group_path(&activity_events, sequence)?;
11072                let scheduled_count = activity_events
11073                    .iter()
11074                    .filter(|event| event.event_type == "ActivityScheduled")
11075                    .count();
11076                if scheduled_count > 1 {
11077                    return Err(invalid_recorded_history(
11078                        "duplicate_activity_schedule",
11079                        sequence,
11080                        "at most one ActivityScheduled event",
11081                        "multiple ActivityScheduled events",
11082                        "activity history schedules more than one command at one workflow sequence",
11083                    ));
11084                }
11085                let activity_type = activity_events.iter().find_map(|event| {
11086                    event
11087                        .payload
11088                        .get("activity_type")
11089                        .or_else(|| event.payload.get("activity_name"))
11090                        .and_then(Value::as_str)
11091                        .map(str::to_string)
11092                });
11093                if activity_events.iter().filter_map(|event| {
11094                    event
11095                        .payload
11096                        .get("activity_type")
11097                        .or_else(|| event.payload.get("activity_name"))
11098                        .and_then(Value::as_str)
11099                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
11100                    return Err(invalid_recorded_history(
11101                        "activity_identity_mismatch",
11102                        sequence,
11103                        activity_type.as_deref().unwrap_or("one activity identity"),
11104                        "conflicting activity identities",
11105                        "activity lifecycle events at one workflow sequence disagree on identity",
11106                    ));
11107                }
11108                let terminal: Vec<_> = activity_events
11109                    .iter()
11110                    .copied()
11111                    .filter(|event| {
11112                        matches!(
11113                            event.event_type.as_str(),
11114                            "ActivityCompleted"
11115                                | "ActivityFailed"
11116                                | "ActivityCancelled"
11117                                | "ActivityTimedOut"
11118                        )
11119                    })
11120                    .collect();
11121                let duplicate_delivery = terminal.first().is_some_and(|first| {
11122                    terminal.iter().all(|event| {
11123                        event.event_type == first.event_type && event.payload == first.payload
11124                    })
11125                });
11126                if terminal.len() > 1 && !duplicate_delivery {
11127                    return Err(invalid_recorded_history(
11128                        "duplicate_activity_terminal_event",
11129                        sequence,
11130                        "at most one terminal activity event",
11131                        "multiple terminal activity events",
11132                        "activity history settles one command more than once",
11133                    ));
11134                }
11135                let outcome = terminal
11136                    .first()
11137                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
11138                    .transpose()?;
11139                let options = activity_events
11140                    .iter()
11141                    .find(|event| event.event_type == "ActivityScheduled")
11142                    .and_then(|event| event.payload.get("activity"))
11143                    .and_then(Value::as_object)
11144                    .map(|activity| RecordedActivityOptions {
11145                        task_queue: recorded_optional_string(activity, "queue"),
11146                        execution_mode: recorded_optional_string(activity, "execution_mode"),
11147                        retry_policy: recorded_activity_retry_snapshot(
11148                            activity.get("retry_policy"),
11149                        ),
11150                    });
11151                return Ok(RecordedCommand::Activity {
11152                    sequence,
11153                    activity_type,
11154                    options,
11155                    outcome,
11156                    parallel_group_path,
11157                });
11158            }
11159
11160            if !child_events.is_empty() {
11161                let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
11162                let scheduled: Vec<_> = child_events
11163                    .iter()
11164                    .copied()
11165                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
11166                    .collect();
11167                if scheduled.len() != 1 {
11168                    return Err(invalid_recorded_history(
11169                        "child_workflow_schedule_missing_or_duplicate",
11170                        sequence,
11171                        "one ChildWorkflowScheduled event",
11172                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
11173                        "child workflow replay requires exactly one recorded schedule event",
11174                    ));
11175                }
11176                let workflow_type = child_events.iter().find_map(|event| {
11177                    event
11178                        .payload
11179                        .get("child_workflow_type")
11180                        .or_else(|| event.payload.get("workflow_type"))
11181                        .and_then(Value::as_str)
11182                        .filter(|value| !value.is_empty())
11183                        .map(str::to_string)
11184                });
11185                if child_events
11186                    .iter()
11187                    .filter_map(|event| {
11188                        event
11189                            .payload
11190                            .get("child_workflow_type")
11191                            .or_else(|| event.payload.get("workflow_type"))
11192                            .and_then(Value::as_str)
11193                    })
11194                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
11195                {
11196                    return Err(invalid_recorded_history(
11197                        "child_workflow_identity_mismatch",
11198                        sequence,
11199                        workflow_type
11200                            .as_deref()
11201                            .unwrap_or("one child workflow type"),
11202                        "conflicting child workflow types",
11203                        "child workflow lifecycle events at one sequence disagree on type",
11204                    ));
11205                }
11206                let mut outcomes = child_workflow_outcomes(
11207                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
11208                    fallback_codec,
11209                    parent.clone(),
11210                )?;
11211                let terminal_events = child_events
11212                    .iter()
11213                    .copied()
11214                    .filter(|event| event.event_type.starts_with("ChildRun"))
11215                    .collect::<Vec<_>>();
11216                let duplicate_delivery = terminal_events.first().is_some_and(|first| {
11217                    terminal_events.iter().all(|event| {
11218                        event.event_type == first.event_type && event.payload == first.payload
11219                    })
11220                });
11221                if outcomes.len() > 1 && !duplicate_delivery {
11222                    return Err(invalid_recorded_history(
11223                        "duplicate_child_workflow_terminal_event",
11224                        sequence,
11225                        "at most one terminal child event",
11226                        "multiple terminal child events",
11227                        "child workflow history settles one command more than once",
11228                    ));
11229                }
11230                return Ok(RecordedCommand::ChildWorkflow {
11231                    sequence,
11232                    workflow_type,
11233                    outcome: outcomes.pop(),
11234                    parallel_group_path,
11235                });
11236            }
11237
11238            if !signal_wait_events.is_empty() {
11239                let opened: Vec<_> = signal_wait_events
11240                    .iter()
11241                    .copied()
11242                    .filter(|event| event.event_type == "SignalWaitOpened")
11243                    .collect();
11244                if opened.len() != 1 {
11245                    return Err(invalid_recorded_history(
11246                        "signal_wait_open_missing_or_duplicate",
11247                        sequence,
11248                        "one SignalWaitOpened event",
11249                        &format!("{} SignalWaitOpened events", opened.len()),
11250                        "signal replay requires exactly one canonical wait-open event",
11251                    ));
11252                }
11253
11254                let applied: Vec<_> = signal_wait_events
11255                    .iter()
11256                    .copied()
11257                    .filter(|event| event.event_type == "SignalApplied")
11258                    .collect();
11259                if applied.len() > 1 {
11260                    return Err(invalid_recorded_history(
11261                        "duplicate_signal_wait_apply",
11262                        sequence,
11263                        "at most one SignalApplied event",
11264                        "multiple SignalApplied events",
11265                        "signal history applies one durable wait more than once",
11266                    ));
11267                }
11268
11269                let signal_names = signal_wait_events
11270                    .iter()
11271                    .map(|event| required_signal_wait_name(event, sequence))
11272                    .collect::<Result<Vec<_>>>()?;
11273                let signal_name = signal_names
11274                    .first()
11275                    .expect("signal wait events are not empty")
11276                    .clone();
11277                if signal_names.iter().any(|candidate| candidate != &signal_name) {
11278                    return Err(invalid_recorded_history(
11279                        "signal_wait_identity_mismatch",
11280                        sequence,
11281                        &signal_name,
11282                        "conflicting signal names",
11283                        "signal wait lifecycle events at one workflow sequence disagree on identity",
11284                    ));
11285                }
11286                let value = applied
11287                    .first()
11288                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
11289                    .transpose()?;
11290                return Ok(RecordedCommand::SignalWait {
11291                    sequence,
11292                    signal_name,
11293                    value,
11294                });
11295            }
11296
11297            if !condition_wait_events.is_empty() {
11298                return recorded_condition_wait(
11299                    sequence,
11300                    &condition_wait_events,
11301                    events,
11302                );
11303            }
11304
11305            if !search_attribute_events.is_empty() {
11306                if search_attribute_events.len() != 1 {
11307                    return Err(invalid_recorded_history(
11308                        "duplicate_search_attribute_update",
11309                        sequence,
11310                        "one SearchAttributesUpserted event",
11311                        &format!(
11312                            "{} SearchAttributesUpserted events",
11313                            search_attribute_events.len()
11314                        ),
11315                        "search-attribute history records one workflow command more than once",
11316                    ));
11317                }
11318                let payload = &search_attribute_events[0].payload;
11319                let attributes = payload
11320                    .get("attributes")
11321                    .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
11322                    .cloned()
11323                    .ok_or_else(|| {
11324                        invalid_recorded_history(
11325                            "search_attribute_update_missing",
11326                            sequence,
11327                            "non-empty attributes object",
11328                            "missing or invalid attributes",
11329                            "search-attribute history is missing its recorded mutation",
11330                        )
11331                    })?;
11332                let attribute_types =
11333                    recorded_search_attribute_types(payload, &attributes, sequence)?;
11334                return Ok(RecordedCommand::SearchAttributes {
11335                    sequence,
11336                    attributes,
11337                    attribute_types,
11338                });
11339            }
11340
11341            if !side_effect_events.is_empty() {
11342                if side_effect_events.len() != 1 {
11343                    return Err(invalid_recorded_history(
11344                        "duplicate_side_effect_record",
11345                        sequence,
11346                        "one SideEffectRecorded event",
11347                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
11348                        "side-effect history records one workflow command more than once",
11349                    ));
11350                }
11351                let event = side_effect_events[0];
11352                let result = event.payload.get("result").ok_or_else(|| {
11353                    invalid_recorded_history(
11354                        "side_effect_result_missing",
11355                        sequence,
11356                        "recorded result payload",
11357                        "missing result",
11358                        "side-effect history is missing its recorded value",
11359                    )
11360                })?;
11361                let has_published_envelope = result.as_str().is_some()
11362                    || result.as_object().is_some_and(|envelope| {
11363                        envelope.get("codec").and_then(Value::as_str).is_some()
11364                            && envelope.get("blob").and_then(Value::as_str).is_some()
11365                    });
11366                if !has_published_envelope {
11367                    return Err(invalid_recorded_history(
11368                        "side_effect_payload_malformed",
11369                        sequence,
11370                        "payload blob or {codec, blob} envelope",
11371                        &result.to_string(),
11372                        "side-effect history result does not use a published payload envelope",
11373                    ));
11374                }
11375                let codec = event
11376                    .payload
11377                    .get("payload_codec")
11378                    .and_then(Value::as_str)
11379                    .unwrap_or(fallback_codec);
11380                let value = decode_wire_avro_value(result, codec).map_err(|error| {
11381                    if error.to_string().contains("unsupported_payload_codec") {
11382                        return error;
11383                    }
11384
11385                    invalid_recorded_history(
11386                        "side_effect_payload_incompatible",
11387                        sequence,
11388                        &format!("valid {codec} payload envelope"),
11389                        &error.to_string(),
11390                        "side-effect history payload cannot be decoded with its recorded codec",
11391                    )
11392                })?;
11393                return Ok(RecordedCommand::SideEffect { sequence, value });
11394            }
11395
11396            if !version_marker_events.is_empty() {
11397                if version_marker_events.len() != 1 {
11398                    return Err(invalid_recorded_history(
11399                        "duplicate_version_marker_record",
11400                        sequence,
11401                        "one VersionMarkerRecorded event",
11402                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
11403                        "version-marker history records one workflow command more than once",
11404                    ));
11405                }
11406                let payload = &version_marker_events[0].payload;
11407                let change_id = payload
11408                    .get("change_id")
11409                    .and_then(Value::as_str)
11410                    .filter(|value| !value.is_empty())
11411                    .map(str::to_string)
11412                    .ok_or_else(|| {
11413                        invalid_recorded_history(
11414                            "version_marker_field_missing",
11415                            sequence,
11416                            "non-empty change_id",
11417                            "missing or invalid change_id",
11418                            "version-marker history is missing its stable change ID",
11419                        )
11420                    })?;
11421                let version = required_version_i32(payload, "version", sequence)?;
11422                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
11423                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
11424                if min_supported > max_supported || version < min_supported || version > max_supported {
11425                    return Err(invalid_recorded_history(
11426                        "version_marker_history_range_invalid",
11427                        sequence,
11428                        "min_supported <= version <= max_supported",
11429                        &format!("{min_supported} <= {version} <= {max_supported}"),
11430                        "recorded version marker contains an internally incompatible range",
11431                    ));
11432                }
11433                return Ok(RecordedCommand::VersionMarker {
11434                    sequence,
11435                    change_id,
11436                    version,
11437                });
11438            }
11439
11440            if !memo_events.is_empty() {
11441                if memo_events.len() != 1 {
11442                    return Err(invalid_recorded_history(
11443                        "duplicate_memo_upsert_record",
11444                        sequence,
11445                        "one MemoUpserted event",
11446                        &format!("{} MemoUpserted events", memo_events.len()),
11447                        "memo history records one workflow update more than once",
11448                    ));
11449                }
11450                let payload = &memo_events[0].payload;
11451                let entries = payload.get("entries").cloned().ok_or_else(|| {
11452                    invalid_recorded_history(
11453                        "memo_entries_missing",
11454                        sequence,
11455                        "memo entries object",
11456                        "missing entries",
11457                        "MemoUpserted history is missing replay identity entries",
11458                    )
11459                })?;
11460                let entries = decode_memo_history_map(&entries, true).map_err(|error| {
11461                    invalid_recorded_history(
11462                        "memo_entries_invalid",
11463                        sequence,
11464                        "valid canonical memo entries",
11465                        &error.to_string(),
11466                        "MemoUpserted history contains invalid replay identity entries",
11467                    )
11468                })?;
11469                let merged = payload.get("merged").cloned().ok_or_else(|| {
11470                    invalid_recorded_history(
11471                        "memo_merged_projection_missing",
11472                        sequence,
11473                        "merged memo projection",
11474                        "missing merged",
11475                        "MemoUpserted history is missing its merged projection",
11476                    )
11477                })?;
11478                decode_memo_history_map(&merged, false).map_err(|error| {
11479                    invalid_recorded_history(
11480                        "memo_merged_projection_invalid",
11481                        sequence,
11482                        "valid merged memo projection",
11483                        &error.to_string(),
11484                        "MemoUpserted history contains an invalid merged projection",
11485                    )
11486                })?;
11487
11488                return Ok(RecordedCommand::Memo { sequence, entries });
11489            }
11490            let scheduled: Vec<_> = timer_events
11491                .iter()
11492                .copied()
11493                .filter(|event| event.event_type == "TimerScheduled")
11494                .collect();
11495            let fired: Vec<_> = timer_events
11496                .iter()
11497                .copied()
11498                .filter(|event| event.event_type == "TimerFired")
11499                .collect();
11500            if scheduled.len() != 1 {
11501                return Err(invalid_recorded_history(
11502                    "timer_schedule_missing_or_duplicate",
11503                    sequence,
11504                    "one TimerScheduled event",
11505                    &format!("{} TimerScheduled events", scheduled.len()),
11506                    "timer replay requires exactly one recorded schedule event",
11507                ));
11508            }
11509            if fired.len() > 1 {
11510                return Err(invalid_recorded_history(
11511                    "duplicate_timer_fire",
11512                    sequence,
11513                    "at most one TimerFired event",
11514                    "multiple TimerFired events",
11515                    "timer history contains more than one fire event for a workflow sequence",
11516                ));
11517            }
11518
11519            let scheduled = scheduled[0];
11520            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
11521            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
11522            if let Some(fired) = fired.first() {
11523                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
11524                if fired_timer_id != timer_id {
11525                    return Err(invalid_recorded_history(
11526                        "timer_identity_mismatch",
11527                        sequence,
11528                        &timer_id,
11529                        &fired_timer_id,
11530                        "TimerFired does not correspond to the recorded TimerScheduled event",
11531                    ));
11532                }
11533                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
11534                if fired_delay != delay_seconds {
11535                    return Err(invalid_recorded_history(
11536                        "timer_history_delay_mismatch",
11537                        sequence,
11538                        &delay_seconds.to_string(),
11539                        &fired_delay.to_string(),
11540                        "TimerScheduled and TimerFired record different delays",
11541                    ));
11542                }
11543            }
11544
11545            Ok(RecordedCommand::Timer {
11546                sequence,
11547                delay_seconds,
11548                fired: !fired.is_empty(),
11549                parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
11550            })
11551        })
11552        .collect::<Result<_>>()?;
11553
11554    let mut marker_sequences = HashMap::new();
11555    for command in &commands {
11556        if let RecordedCommand::VersionMarker {
11557            sequence,
11558            change_id,
11559            ..
11560        } = command
11561        {
11562            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
11563                return Err(invalid_recorded_history(
11564                    "duplicate_version_marker",
11565                    *sequence,
11566                    &format!("one marker for change ID {change_id:?}"),
11567                    &format!("markers at sequences {first_sequence} and {sequence}"),
11568                    "workflow history contains duplicate markers for one stable change ID",
11569                ));
11570            }
11571        }
11572    }
11573
11574    Ok(commands)
11575}
11576
11577fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
11578    payload
11579        .get(field)
11580        .and_then(Value::as_i64)
11581        .and_then(|value| i32::try_from(value).ok())
11582        .ok_or_else(|| {
11583            invalid_recorded_history(
11584                "version_marker_field_missing",
11585                sequence,
11586                &format!("integer {field}"),
11587                "missing or out-of-range integer",
11588                "version-marker history is missing a required integer field",
11589            )
11590        })
11591}
11592
11593fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
11594    event
11595        .payload
11596        .get("sequence")
11597        .or_else(|| event.payload.get("workflow_sequence"))
11598        .or_else(|| event.raw.get("sequence"))
11599        .or_else(|| event.raw.get("workflow_sequence"))
11600        .and_then(value_as_u64)
11601}
11602
11603fn is_internal_timer_event(event: &HistoryEvent) -> bool {
11604    matches!(
11605        event
11606            .payload
11607            .get("timer_kind")
11608            .or_else(|| event.raw.get("timer_kind"))
11609            .and_then(Value::as_str),
11610        Some("condition_timeout" | "signal_timeout")
11611    )
11612}
11613
11614fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
11615    matches!(
11616        event.event_type.as_str(),
11617        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
11618    )
11619}
11620
11621fn recorded_condition_wait(
11622    sequence: u64,
11623    condition_events: &[&HistoryEvent],
11624    all_events: &[HistoryEvent],
11625) -> Result<RecordedCommand> {
11626    let opened = condition_events
11627        .iter()
11628        .copied()
11629        .filter(|event| event.event_type == "ConditionWaitOpened")
11630        .collect::<Vec<_>>();
11631    if opened.len() != 1 {
11632        return Err(invalid_recorded_history(
11633            "condition_wait_open_missing_or_duplicate",
11634            sequence,
11635            "one ConditionWaitOpened event",
11636            &format!("{} ConditionWaitOpened events", opened.len()),
11637            "condition replay requires exactly one canonical wait-open event",
11638        ));
11639    }
11640    let terminal = condition_events
11641        .iter()
11642        .copied()
11643        .filter(|event| {
11644            matches!(
11645                event.event_type.as_str(),
11646                "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
11647            )
11648        })
11649        .collect::<Vec<_>>();
11650    if terminal.len() > 1 {
11651        return Err(invalid_recorded_history(
11652            "duplicate_condition_wait_terminal_event",
11653            sequence,
11654            "at most one condition terminal event",
11655            "multiple condition terminal events",
11656            "condition history settles one durable wait more than once",
11657        ));
11658    }
11659
11660    let opened = opened[0];
11661    let condition_wait_id = required_condition_wait_id(opened, sequence)?;
11662    let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
11663    for event in condition_events
11664        .iter()
11665        .copied()
11666        .filter(|event| !std::ptr::eq(*event, opened))
11667    {
11668        let event_wait_id = required_condition_wait_id(event, sequence)?;
11669        if event_wait_id != condition_wait_id {
11670            return Err(invalid_recorded_history(
11671                "condition_wait_id_mismatch",
11672                sequence,
11673                &condition_wait_id,
11674                &event_wait_id,
11675                "condition lifecycle events at one sequence disagree on wait identity",
11676            ));
11677        }
11678        let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
11679        if event_occurrence_id != occurrence_id {
11680            return Err(invalid_recorded_history(
11681                "condition_wait_occurrence_history_mismatch",
11682                sequence,
11683                &occurrence_id,
11684                &event_occurrence_id,
11685                "condition lifecycle events at one sequence disagree on authored occurrence identity",
11686            ));
11687        }
11688    }
11689
11690    let condition_key = optional_non_empty_history_string(opened, "condition_key");
11691    let predicate_identity = opened
11692        .payload
11693        .get("condition_definition_fingerprint")
11694        .and_then(Value::as_str)
11695        .filter(|value| !value.is_empty())
11696        .map(str::to_string)
11697        .ok_or_else(|| {
11698            invalid_recorded_history(
11699                "condition_wait_predicate_fingerprint_missing",
11700                sequence,
11701                "non-empty condition_definition_fingerprint",
11702                &opened.event_type,
11703                "canonical condition history is missing its predicate identity",
11704            )
11705        })?;
11706    let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
11707    for event in condition_events
11708        .iter()
11709        .copied()
11710        .filter(|event| !std::ptr::eq(*event, opened))
11711    {
11712        for (field, opened_value) in [
11713            ("condition_key", condition_key.as_deref()),
11714            (
11715                "condition_definition_fingerprint",
11716                Some(predicate_identity.as_str()),
11717            ),
11718        ] {
11719            if let Some(value) = optional_non_empty_history_string(event, field) {
11720                if opened_value.is_some_and(|opened_value| opened_value != value) {
11721                    return Err(invalid_recorded_history(
11722                        "condition_wait_definition_history_mismatch",
11723                        sequence,
11724                        opened_value.unwrap_or_default(),
11725                        &value,
11726                        "condition lifecycle events disagree on the recorded definition",
11727                    ));
11728                }
11729            }
11730        }
11731        if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
11732            if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
11733                return Err(invalid_recorded_history(
11734                    "condition_wait_definition_history_mismatch",
11735                    sequence,
11736                    &format!("{}s", timeout_seconds.unwrap_or_default()),
11737                    &format!("{event_timeout}s"),
11738                    "condition lifecycle events disagree on the recorded timeout",
11739                ));
11740            }
11741        }
11742    }
11743
11744    let timeout_timer_events = all_events
11745        .iter()
11746        .filter(|event| {
11747            matches!(
11748                event.event_type.as_str(),
11749                "TimerScheduled" | "TimerCancelled" | "TimerFired"
11750            ) && event.payload.get("timer_kind").and_then(Value::as_str)
11751                == Some("condition_timeout")
11752                && event
11753                    .payload
11754                    .get("condition_wait_id")
11755                    .and_then(Value::as_str)
11756                    == Some(condition_wait_id.as_str())
11757        })
11758        .collect::<Vec<_>>();
11759    let scheduled = timeout_timer_events
11760        .iter()
11761        .copied()
11762        .filter(|event| event.event_type == "TimerScheduled")
11763        .collect::<Vec<_>>();
11764    let fired = timeout_timer_events
11765        .iter()
11766        .copied()
11767        .filter(|event| event.event_type == "TimerFired")
11768        .collect::<Vec<_>>();
11769    if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
11770        return Err(invalid_recorded_history(
11771            "condition_wait_timeout_history_invalid",
11772            sequence,
11773            "one timeout schedule and at most one fire",
11774            &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
11775            "condition timeout history has a missing or duplicate lifecycle event",
11776        ));
11777    }
11778    if let Some(scheduled) = scheduled.first() {
11779        let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
11780        let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
11781        if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
11782            return Err(invalid_recorded_history(
11783                "condition_wait_timeout_delay_mismatch",
11784                sequence,
11785                &format!("{}s", timeout_seconds.unwrap_or_default()),
11786                &format!("{delay_seconds}s"),
11787                "condition timeout timer differs from the wait definition",
11788            ));
11789        }
11790        if let Some(fired) = fired.first() {
11791            let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
11792            let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
11793            if fired_timer_id != timer_id || fired_delay != delay_seconds {
11794                return Err(invalid_recorded_history(
11795                    "condition_wait_timeout_identity_mismatch",
11796                    sequence,
11797                    &format!("{timer_id}:{delay_seconds}s"),
11798                    &format!("{fired_timer_id}:{fired_delay}s"),
11799                    "condition timeout fire does not match its durable schedule",
11800                ));
11801            }
11802        }
11803    }
11804
11805    let result = terminal.first().map(|event| {
11806        if event.event_type == "ConditionWaitTimedOut" {
11807            ConditionWaitResult::TimedOut
11808        } else {
11809            ConditionWaitResult::Satisfied
11810        }
11811    });
11812    let result = if !fired.is_empty() {
11813        if result == Some(ConditionWaitResult::Satisfied) {
11814            return Err(invalid_recorded_history(
11815                "condition_wait_terminal_conflict",
11816                sequence,
11817                "one satisfied or timed-out outcome",
11818                "satisfied event and fired timeout",
11819                "condition history records conflicting terminal outcomes",
11820            ));
11821        }
11822        Some(ConditionWaitResult::TimedOut)
11823    } else {
11824        result
11825    };
11826
11827    Ok(RecordedCommand::ConditionWait {
11828        sequence,
11829        occurrence_id,
11830        condition_key,
11831        predicate_identity,
11832        timeout_seconds,
11833        result,
11834    })
11835}
11836
11837fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
11838    event
11839        .payload
11840        .get("condition_wait_occurrence_id")
11841        .and_then(Value::as_str)
11842        .filter(|value| !value.is_empty())
11843        .map(str::to_string)
11844        .ok_or_else(|| {
11845            invalid_recorded_history(
11846                "condition_wait_occurrence_id_missing",
11847                sequence,
11848                "non-empty condition_wait_occurrence_id",
11849                &event.event_type,
11850                "condition history is missing authored occurrence identity",
11851            )
11852        })
11853}
11854
11855fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
11856    event
11857        .payload
11858        .get("condition_wait_id")
11859        .and_then(Value::as_str)
11860        .filter(|value| !value.is_empty())
11861        .map(str::to_string)
11862        .ok_or_else(|| {
11863            invalid_recorded_history(
11864                "condition_wait_id_missing",
11865                sequence,
11866                "non-empty condition_wait_id",
11867                &event.event_type,
11868                "canonical condition history is missing its durable wait identity",
11869            )
11870        })
11871}
11872
11873fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
11874    event
11875        .payload
11876        .get(field)
11877        .and_then(Value::as_str)
11878        .filter(|value| !value.is_empty())
11879        .map(str::to_string)
11880}
11881
11882fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
11883    match event.payload.get(field) {
11884        None | Some(Value::Null) => Ok(None),
11885        Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
11886            invalid_recorded_history(
11887                "condition_wait_definition_invalid",
11888                sequence,
11889                &format!("non-negative integer {field}"),
11890                &value.to_string(),
11891                "condition history contains an invalid numeric definition field",
11892            )
11893        }),
11894    }
11895}
11896
11897fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
11898    event
11899        .payload
11900        .get("signal_name")
11901        .or_else(|| event.raw.get("signal_name"))
11902        .and_then(Value::as_str)
11903        .filter(|value| !value.is_empty())
11904        .map(str::to_string)
11905        .ok_or_else(|| {
11906            invalid_recorded_history(
11907                "signal_wait_name_missing",
11908                sequence,
11909                "non-empty signal_name",
11910                &event.event_type,
11911                "canonical signal-wait history is missing its signal identity",
11912            )
11913        })
11914}
11915
11916fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
11917    matches!(
11918        event.event_type.as_str(),
11919        "SignalWaitOpened" | "SignalApplied"
11920    )
11921}
11922
11923fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
11924    event
11925        .payload
11926        .get(field)
11927        .and_then(Value::as_str)
11928        .filter(|value| !value.is_empty())
11929        .map(str::to_string)
11930        .ok_or_else(|| {
11931            invalid_recorded_history(
11932                "timer_history_field_missing",
11933                sequence,
11934                field,
11935                &event.event_type,
11936                "timer history is missing a required identity field",
11937            )
11938        })
11939}
11940
11941fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
11942    event
11943        .payload
11944        .get(field)
11945        .and_then(value_as_u64)
11946        .ok_or_else(|| {
11947            invalid_recorded_history(
11948                "timer_history_field_missing",
11949                sequence,
11950                field,
11951                &event.event_type,
11952                "timer history is missing a required numeric field",
11953            )
11954        })
11955}
11956
11957fn recorded_search_attribute_types(
11958    payload: &Value,
11959    attributes: &Value,
11960    sequence: u64,
11961) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
11962    let Some(raw_types) = payload.get("attribute_types") else {
11963        // This is the explicit compatibility rule for histories recorded
11964        // before typed identity was persisted. Values still constrain replay;
11965        // the unknown type snapshot does not assert a typed match.
11966        return Ok(RecordedSnapshotValue::Unknown);
11967    };
11968    let Some(raw_types) = raw_types.as_object() else {
11969        return Err(invalid_recorded_history(
11970            "search_attribute_types_malformed",
11971            sequence,
11972            "canonical attribute type map",
11973            &raw_types.to_string(),
11974            "search-attribute history contains malformed type identity",
11975        ));
11976    };
11977    let attribute_keys = attributes
11978        .as_object()
11979        .expect("recorded search attributes were validated as an object");
11980    let mut types = BTreeMap::new();
11981    for (key, value) in raw_types {
11982        let Some(attribute_type) = value.as_str() else {
11983            return Err(invalid_recorded_history(
11984                "search_attribute_types_malformed",
11985                sequence,
11986                "canonical string type name",
11987                &value.to_string(),
11988                "search-attribute history contains a non-string type identity",
11989            ));
11990        };
11991        if !attribute_keys.contains_key(key)
11992            || !matches!(
11993                attribute_type,
11994                "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
11995            )
11996        {
11997            return Err(invalid_recorded_history(
11998                "search_attribute_types_malformed",
11999                sequence,
12000                "canonical types for keys present in attributes",
12001                &format!("{key}:{attribute_type}"),
12002                "search-attribute history contains unsupported or orphaned type identity",
12003            ));
12004        }
12005        types.insert(key.clone(), attribute_type.to_string());
12006    }
12007    Ok(RecordedSnapshotValue::Known(types))
12008}
12009
12010fn invalid_recorded_history(
12011    reason: &str,
12012    sequence: u64,
12013    expected: &str,
12014    actual: &str,
12015    message: &str,
12016) -> Error {
12017    Error::NonDeterministicReplay(ReplayFailure::new(
12018        reason,
12019        Some(sequence),
12020        Some(expected.to_string()),
12021        Some(actual.to_string()),
12022        message,
12023    ))
12024}
12025
12026type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
12027
12028fn activity_outcome(
12029    event: &HistoryEvent,
12030    fallback_codec: &str,
12031    recorded_activity_type: Option<String>,
12032) -> Result<ActivityOutcome> {
12033    if event.event_type == "ActivityCompleted" {
12034        let codec = event
12035            .payload
12036            .get("payload_codec")
12037            .and_then(Value::as_str)
12038            .unwrap_or(fallback_codec);
12039        return Ok(Ok(decode_wire_avro_value(
12040            event.payload.get("result").unwrap_or(&Value::Null),
12041            codec,
12042        )?));
12043    }
12044
12045    let payload = &event.payload;
12046    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
12047        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
12048        "ActivityCancelled" => (
12049            ActivityFailureKind::Cancelled,
12050            "cancelled",
12051            "activity was cancelled",
12052        ),
12053        "ActivityTimedOut" => (
12054            ActivityFailureKind::TimedOut,
12055            "timeout",
12056            "activity timed out",
12057        ),
12058        _ => unreachable!("activity_outcome is called only for terminal activity events"),
12059    };
12060    let exception = payload
12061        .get("exception")
12062        .filter(|value| !value.is_null())
12063        .cloned();
12064    let failure_category = payload_string(payload, "failure_category");
12065    let timeout_kind = payload_string(payload, "timeout_kind");
12066    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
12067        ActivityFailureKind::Failed => failure_category
12068            .clone()
12069            .unwrap_or_else(|| fallback_reason.to_string()),
12070        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
12071        ActivityFailureKind::TimedOut => timeout_kind
12072            .clone()
12073            .unwrap_or_else(|| fallback_reason.to_string()),
12074    });
12075    let message = payload_string(payload, "message")
12076        .or_else(|| {
12077            exception
12078                .as_ref()
12079                .and_then(|value| payload_string(value, "message"))
12080        })
12081        .unwrap_or_else(|| fallback_message.to_string());
12082
12083    Ok(Err(ActivityFailure {
12084        kind,
12085        reason,
12086        message,
12087        activity_execution_id: payload_string(payload, "activity_execution_id"),
12088        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
12089        activity_type: payload_string(payload, "activity_type")
12090            .or_else(|| payload_string(payload, "activity_name"))
12091            .or(recorded_activity_type),
12092        activity_class: payload_string(payload, "activity_class"),
12093        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
12094        failure_id: payload_string(payload, "failure_id"),
12095        failure_category,
12096        timeout_kind,
12097        non_retryable: payload
12098            .get("non_retryable")
12099            .and_then(Value::as_bool)
12100            .unwrap_or(false),
12101        exception_type: payload_string(payload, "exception_type").or_else(|| {
12102            exception
12103                .as_ref()
12104                .and_then(|value| payload_string(value, "type"))
12105        }),
12106        exception_class: payload_string(payload, "exception_class").or_else(|| {
12107            exception
12108                .as_ref()
12109                .and_then(|value| payload_string(value, "class"))
12110        }),
12111        code: payload
12112            .get("code")
12113            .filter(|value| !value.is_null())
12114            .cloned(),
12115        exception,
12116    }))
12117}
12118
12119type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
12120
12121fn child_workflow_outcomes(
12122    events: &[HistoryEvent],
12123    fallback_codec: &str,
12124    parent: WorkflowIdentity,
12125) -> Result<Vec<ChildWorkflowOutcome>> {
12126    let mut outcomes = Vec::new();
12127
12128    for event in events {
12129        let kind = match event.event_type.as_str() {
12130            "ChildRunCompleted" => None,
12131            "ChildRunFailed" => Some((
12132                ChildWorkflowFailureKind::Failed,
12133                "child_workflow",
12134                "child workflow failed",
12135            )),
12136            "ChildRunCancelled" => Some((
12137                ChildWorkflowFailureKind::Cancelled,
12138                "cancelled",
12139                "child workflow was cancelled",
12140            )),
12141            "ChildRunTerminated" => Some((
12142                ChildWorkflowFailureKind::Terminated,
12143                "terminated",
12144                "child workflow was terminated",
12145            )),
12146            _ => continue,
12147        };
12148        let payload = &event.payload;
12149        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
12150        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
12151        let child_workflow_type = payload_string(payload, "child_workflow_type");
12152
12153        if let Some((kind, reason, fallback_message)) = kind {
12154            let exception = payload
12155                .get("exception")
12156                .filter(|value| !value.is_null())
12157                .cloned();
12158            let message = payload_string(payload, "message")
12159                .or_else(|| {
12160                    exception
12161                        .as_ref()
12162                        .and_then(|value| payload_string(value, "message"))
12163                })
12164                .unwrap_or_else(|| fallback_message.to_string());
12165            let exception_type = payload_string(payload, "exception_type").or_else(|| {
12166                exception
12167                    .as_ref()
12168                    .and_then(|value| payload_string(value, "type"))
12169            });
12170            let exception_class = payload_string(payload, "exception_class").or_else(|| {
12171                exception
12172                    .as_ref()
12173                    .and_then(|value| payload_string(value, "class"))
12174            });
12175            outcomes.push(Err(ChildWorkflowFailure {
12176                kind,
12177                reason: reason.to_string(),
12178                message,
12179                parent_workflow_id: parent.workflow_id.clone(),
12180                parent_workflow_run_id: parent.run_id.clone(),
12181                child_workflow_id,
12182                child_workflow_run_id,
12183                child_workflow_type,
12184                failure_id: payload_string(payload, "failure_id"),
12185                failure_category: payload_string(payload, "failure_category"),
12186                exception_type,
12187                exception_class,
12188                non_retryable: payload
12189                    .get("non_retryable")
12190                    .and_then(Value::as_bool)
12191                    .unwrap_or(false),
12192                code: payload
12193                    .get("code")
12194                    .filter(|value| !value.is_null())
12195                    .cloned(),
12196                exception,
12197            }));
12198            continue;
12199        }
12200
12201        let codec = payload
12202            .get("payload_codec")
12203            .and_then(Value::as_str)
12204            .unwrap_or(fallback_codec);
12205        let result = payload
12206            .get("result")
12207            .or_else(|| payload.get("output"))
12208            .unwrap_or(&Value::Null);
12209        outcomes.push(Ok(ChildWorkflowAvroResult {
12210            parent: parent.clone(),
12211            child: WorkflowIdentity {
12212                workflow_id: child_workflow_id,
12213                run_id: child_workflow_run_id,
12214            },
12215            child_workflow_type,
12216            result: decode_wire_avro_value(result, codec)?,
12217        }));
12218    }
12219
12220    Ok(outcomes)
12221}
12222
12223fn payload_string(payload: &Value, key: &str) -> Option<String> {
12224    payload
12225        .get(key)
12226        .and_then(Value::as_str)
12227        .filter(|value| !value.is_empty())
12228        .map(str::to_string)
12229}
12230
12231fn workflow_failure_command(error: &Error) -> Value {
12232    let (exception_type, exception_class, properties) = match error {
12233        Error::ActivityFailed(failure) => (
12234            match failure.kind {
12235                ActivityFailureKind::Failed => "ActivityFailed",
12236                ActivityFailureKind::Cancelled => "ActivityCancelled",
12237                ActivityFailureKind::TimedOut => "ActivityTimedOut",
12238            },
12239            "durable_workflow::ActivityFailure",
12240            json!({
12241                "reason": failure.reason,
12242                "activity_execution_id": failure.activity_execution_id,
12243                "activity_attempt_id": failure.activity_attempt_id,
12244                "activity_type": failure.activity_type,
12245                "activity_class": failure.activity_class,
12246                "attempt_number": failure.attempt_number,
12247                "failure_id": failure.failure_id,
12248                "failure_category": failure.failure_category,
12249                "timeout_kind": failure.timeout_kind,
12250                "activity_non_retryable": failure.non_retryable,
12251                "activity_exception_type": failure.exception_type,
12252                "activity_exception_class": failure.exception_class,
12253                "activity_code": failure.code,
12254                "activity_exception": failure.exception,
12255            }),
12256        ),
12257        Error::ChildWorkflowFailed(failure) => (
12258            match failure.kind {
12259                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
12260                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
12261                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
12262            },
12263            "durable_workflow::ChildWorkflowFailure",
12264            json!({
12265                "reason": failure.reason,
12266                "parent_workflow_id": failure.parent_workflow_id,
12267                "parent_workflow_run_id": failure.parent_workflow_run_id,
12268                "child_workflow_id": failure.child_workflow_id,
12269                "child_workflow_run_id": failure.child_workflow_run_id,
12270                "child_workflow_type": failure.child_workflow_type,
12271                "failure_id": failure.failure_id,
12272                "failure_category": failure.failure_category,
12273                "child_exception_type": failure.exception_type,
12274                "child_exception_class": failure.exception_class,
12275                "child_non_retryable": failure.non_retryable,
12276                "child_code": failure.code,
12277                "child_exception": failure.exception,
12278            }),
12279        ),
12280        Error::ParallelFailed(failure) => (
12281            "ParallelFailed",
12282            "durable_workflow::ParallelFailure",
12283            json!({
12284                "parallel_group_id": failure.group_id,
12285                "parallel_member_path": failure.member_path,
12286                "parallel_group_path": failure.group_path,
12287                "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
12288                "cause_type": workflow_error_type(&failure.cause),
12289                "cause_message": failure.cause.to_string(),
12290            }),
12291        ),
12292        Error::SagaCompensationFailed(failure) => (
12293            "SagaCompensationFailed",
12294            "durable_workflow::SagaCompensationFailure",
12295            json!({
12296                "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
12297                "initiating_failure_message": failure.initiating_failure.to_string(),
12298                "compensation_activity_type": failure.compensation_activity_type,
12299                "compensation_registration_order": failure.compensation_registration_order,
12300                "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
12301                "compensation_failure_message": failure.compensation_failure.to_string(),
12302            }),
12303        ),
12304        Error::WorkflowCancellationRequested(_) => (
12305            "WorkflowCancellationRequested",
12306            "durable_workflow::WorkflowCancellationRequested",
12307            json!({"reason": "cancelled"}),
12308        ),
12309        Error::NonDeterministicReplay(_) => (
12310            "NonDeterministicReplay",
12311            "durable_workflow::Error",
12312            Value::Null,
12313        ),
12314        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
12315    };
12316    let non_retryable = match error {
12317        Error::ActivityFailed(failure) => failure.non_retryable,
12318        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
12319        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
12320        Error::SagaCompensationFailed(failure) => {
12321            workflow_error_non_retryable(&failure.compensation_failure)
12322        }
12323        Error::WorkflowCancellationRequested(_) => true,
12324        Error::NonDeterministicReplay(_) => true,
12325        _ => false,
12326    };
12327
12328    json!({
12329        "type": "fail_workflow",
12330        "message": error.to_string(),
12331        "exception_type": exception_type,
12332        "exception_class": exception_class,
12333        "non_retryable": non_retryable,
12334        "exception": {
12335            "type": exception_type,
12336            "class": exception_class,
12337            "message": error.to_string(),
12338            "properties": properties,
12339        }
12340    })
12341}
12342
12343fn workflow_error_type(error: &Error) -> &'static str {
12344    match error {
12345        Error::ActivityFailed(failure) => match failure.kind {
12346            ActivityFailureKind::Failed => "ActivityFailed",
12347            ActivityFailureKind::Cancelled => "ActivityCancelled",
12348            ActivityFailureKind::TimedOut => "ActivityTimedOut",
12349        },
12350        Error::ChildWorkflowFailed(failure) => match failure.kind {
12351            ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
12352            ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
12353            ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
12354        },
12355        Error::ParallelFailed(_) => "ParallelFailed",
12356        Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
12357        Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
12358        Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
12359        _ => "RustWorkflowError",
12360    }
12361}
12362
12363fn workflow_error_non_retryable(error: &Error) -> bool {
12364    match error {
12365        Error::ActivityFailed(failure) => failure.non_retryable,
12366        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
12367        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
12368        Error::SagaCompensationFailed(failure) => {
12369            workflow_error_non_retryable(&failure.compensation_failure)
12370        }
12371        Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
12372        _ => false,
12373    }
12374}
12375
12376fn workflow_task_integrity_error(error: &Error) -> bool {
12377    matches!(
12378        error,
12379        Error::NonDeterministicReplay(_)
12380            | Error::Protocol(_)
12381            | Error::MissingWorkflowCommandIdentity
12382            | Error::WorkflowStatePoisoned
12383    )
12384}
12385
12386fn decode_signal_event_arguments(
12387    event: &HistoryEvent,
12388    fallback_codec: &str,
12389) -> Result<Vec<AvroValue>> {
12390    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
12391    validate_payload_codec(codec)?;
12392    let raw = signal_history_payload(&event.payload);
12393    let decoded = match raw.filter(|value| !value.is_null()) {
12394        Some(value) => decode_wire_avro_value(value, codec)?,
12395        None => AvroValue::Array(Vec::new()),
12396    };
12397    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
12398        unreachable!("normalize_avro_arguments always returns an array");
12399    };
12400    Ok(arguments)
12401}
12402
12403fn decode_update_event_arguments(
12404    event: &HistoryEvent,
12405    fallback_codec: &str,
12406) -> Result<Vec<AvroValue>> {
12407    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
12408    validate_payload_codec(codec)?;
12409    let decoded = match event
12410        .payload
12411        .get("arguments")
12412        .filter(|value| !value.is_null())
12413    {
12414        Some(value) => decode_wire_avro_value(value, codec)?,
12415        None => AvroValue::Array(Vec::new()),
12416    };
12417    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
12418        unreachable!("normalize_avro_arguments always returns an array");
12419    };
12420    Ok(arguments)
12421}
12422
12423fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
12424    let Some(export_events) = task
12425        .history_export
12426        .as_ref()
12427        .and_then(|export| export.get("history_events"))
12428        .and_then(Value::as_array)
12429    else {
12430        return Ok(());
12431    };
12432
12433    if export_events.len() > task.history_events.len() {
12434        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
12435    }
12436
12437    Ok(())
12438}
12439
12440fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
12441    let Some(export) = task.history_export.as_ref() else {
12442        return Ok(());
12443    };
12444    let signals = export
12445        .get("signals")
12446        .and_then(Value::as_array)
12447        .cloned()
12448        .unwrap_or_default();
12449    let activities = export
12450        .get("activities")
12451        .and_then(Value::as_array)
12452        .cloned()
12453        .unwrap_or_default();
12454    let export_codec = export
12455        .get("payloads")
12456        .and_then(|payloads| payloads.get("codec"))
12457        .and_then(Value::as_str)
12458        .unwrap_or(&task.payload_codec)
12459        .to_string();
12460    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
12461
12462    for event in &mut task.history_events {
12463        if event.event_type == "ActivityCompleted" {
12464            let sequence = event
12465                .payload
12466                .get("sequence")
12467                .or_else(|| event.payload.get("workflow_sequence"))
12468                .and_then(value_as_u64);
12469            let Some(activity) = sequence.and_then(|sequence| {
12470                activities.iter().find(|activity| {
12471                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
12472                })
12473            }) else {
12474                continue;
12475            };
12476            let Some(payload) = event.payload.as_object_mut() else {
12477                continue;
12478            };
12479            if missing_payload(payload.get("result")) {
12480                if let Some(result) = activity
12481                    .get("result")
12482                    .filter(|value| !missing_payload(Some(value)))
12483                {
12484                    payload.insert("result".to_string(), result.clone());
12485                }
12486            }
12487            for field in ["payload_codec", "activity_type"] {
12488                if payload
12489                    .get(field)
12490                    .and_then(Value::as_str)
12491                    .unwrap_or_default()
12492                    .is_empty()
12493                {
12494                    if let Some(value) = activity.get(field) {
12495                        payload.insert(field.to_string(), value.clone());
12496                    }
12497                }
12498            }
12499            continue;
12500        }
12501
12502        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
12503            continue;
12504        }
12505        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
12506        let command_id = event
12507            .payload
12508            .get("workflow_command_id")
12509            .or_else(|| event.raw.get("workflow_command_id"))
12510            .and_then(Value::as_str);
12511        let signal_name = event
12512            .payload
12513            .get("signal_name")
12514            .and_then(Value::as_str)
12515            .unwrap_or_default()
12516            .to_string();
12517        let matched = signals
12518            .iter()
12519            .find(|signal| {
12520                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
12521            })
12522            .or_else(|| {
12523                signals.iter().find(|signal| {
12524                    command_id.is_some()
12525                        && signal.get("command_id").and_then(Value::as_str) == command_id
12526                })
12527            })
12528            .or_else(|| {
12529                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
12530                let signal = signals
12531                    .iter()
12532                    .filter(|signal| {
12533                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
12534                    })
12535                    .nth(*offset);
12536                if signal.is_some() {
12537                    *offset += 1;
12538                }
12539                signal
12540            });
12541        let Some(signal) = matched else {
12542            continue;
12543        };
12544        let signal_codec = signal
12545            .get("payload_codec")
12546            .and_then(Value::as_str)
12547            .unwrap_or(&export_codec);
12548        let Some(payload) = event.payload.as_object_mut() else {
12549            continue;
12550        };
12551        if missing_payload(payload.get("arguments")) {
12552            if let Some(arguments) = signal
12553                .get("arguments")
12554                .filter(|value| !missing_payload(Some(value)))
12555            {
12556                let envelope = match arguments {
12557                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
12558                    other => other.clone(),
12559                };
12560                payload.insert("arguments".to_string(), envelope);
12561            }
12562        }
12563        if payload
12564            .get("payload_codec")
12565            .and_then(Value::as_str)
12566            .unwrap_or_default()
12567            .is_empty()
12568        {
12569            payload.insert("payload_codec".to_string(), json!(signal_codec));
12570        }
12571    }
12572
12573    Ok(())
12574}
12575
12576fn missing_payload(value: Option<&Value>) -> bool {
12577    match value {
12578        None | Some(Value::Null) => true,
12579        Some(Value::String(value)) => value.is_empty(),
12580        Some(_) => false,
12581    }
12582}
12583
12584fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
12585    let export_signals = task
12586        .history_export
12587        .as_ref()
12588        .and_then(|export| export.get("signals"))
12589        .and_then(Value::as_array)
12590        .cloned()
12591        .unwrap_or_default();
12592    let export_codec = task
12593        .history_export
12594        .as_ref()
12595        .and_then(|export| export.get("payloads"))
12596        .and_then(|payloads| payloads.get("codec"))
12597        .and_then(Value::as_str)
12598        .unwrap_or(&task.payload_codec);
12599    let mut name_offsets: HashMap<String, usize> = HashMap::new();
12600    let mut signals = Vec::new();
12601
12602    for event in &task.history_events {
12603        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
12604            continue;
12605        }
12606
12607        let name = event
12608            .payload
12609            .get("signal_name")
12610            .and_then(Value::as_str)
12611            .unwrap_or_default();
12612        if name.is_empty() {
12613            continue;
12614        }
12615        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
12616        let command_id = event
12617            .payload
12618            .get("workflow_command_id")
12619            .or_else(|| event.raw.get("workflow_command_id"))
12620            .and_then(Value::as_str);
12621        let matched_export = export_signals
12622            .iter()
12623            .find(|candidate| {
12624                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
12625            })
12626            .or_else(|| {
12627                export_signals.iter().find(|candidate| {
12628                    command_id.is_some()
12629                        && candidate.get("command_id").and_then(Value::as_str) == command_id
12630                })
12631            })
12632            .or_else(|| {
12633                let offset = name_offsets.entry(name.to_string()).or_default();
12634                let candidate = export_signals
12635                    .iter()
12636                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
12637                    .nth(*offset);
12638                if candidate.is_some() {
12639                    *offset += 1;
12640                }
12641                candidate
12642            });
12643        let codec = event
12644            .payload
12645            .get("payload_codec")
12646            .and_then(Value::as_str)
12647            .or_else(|| {
12648                matched_export
12649                    .and_then(|signal| signal.get("payload_codec"))
12650                    .and_then(Value::as_str)
12651            })
12652            .unwrap_or(export_codec);
12653        let raw_arguments = signal_history_payload(&event.payload)
12654            .filter(|value| !value.is_null())
12655            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
12656        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
12657        let workflow_sequence = event
12658            .payload
12659            .get("workflow_sequence")
12660            .and_then(value_as_u64)
12661            .or_else(|| {
12662                matched_export
12663                    .and_then(|signal| signal.get("workflow_sequence"))
12664                    .and_then(value_as_u64)
12665            });
12666
12667        signals.push(QuerySignal {
12668            id: signal_id.map(str::to_string).or_else(|| {
12669                matched_export
12670                    .and_then(|signal| signal.get("id"))
12671                    .and_then(Value::as_str)
12672                    .map(str::to_string)
12673            }),
12674            name: name.to_string(),
12675            arguments,
12676            avro_arguments,
12677            workflow_sequence,
12678        });
12679    }
12680
12681    if signals.is_empty() {
12682        for signal in export_signals {
12683            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
12684                continue;
12685            }
12686            let Some(name) = signal.get("name").and_then(Value::as_str) else {
12687                continue;
12688            };
12689            let codec = signal
12690                .get("payload_codec")
12691                .and_then(Value::as_str)
12692                .unwrap_or(export_codec);
12693            let (arguments, avro_arguments) =
12694                decode_query_signal_arguments(signal.get("arguments"), codec)?;
12695            signals.push(QuerySignal {
12696                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
12697                name: name.to_string(),
12698                arguments,
12699                avro_arguments,
12700                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
12701            });
12702        }
12703        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
12704    }
12705
12706    Ok(signals)
12707}
12708
12709fn decode_query_signal_arguments(
12710    raw: Option<&Value>,
12711    codec: &str,
12712) -> Result<(Vec<Value>, Vec<AvroValue>)> {
12713    validate_payload_codec(codec)?;
12714    let decoded = match raw.filter(|value| !value.is_null()) {
12715        Some(value) => decode_wire_avro_value(value, codec)?,
12716        None => AvroValue::Array(Vec::new()),
12717    };
12718    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
12719        unreachable!("normalize_avro_arguments always returns an array");
12720    };
12721    let arguments = avro_arguments
12722        .iter()
12723        .cloned()
12724        .map(AvroValue::into_json)
12725        .collect::<Result<Vec<_>>>()?;
12726    Ok((arguments, avro_arguments))
12727}
12728
12729fn value_as_u64(value: &Value) -> Option<u64> {
12730    value
12731        .as_u64()
12732        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
12733}
12734
12735#[cfg(test)]
12736mod tests {
12737    use super::*;
12738    use std::{
12739        io::{Read, Write},
12740        net::{SocketAddr, TcpListener, TcpStream},
12741        sync::atomic::AtomicUsize,
12742        thread,
12743    };
12744
12745    #[derive(Clone, Copy, Debug)]
12746    enum InvalidTaskPayloadCodec {
12747        Missing,
12748        Null,
12749        NonString,
12750    }
12751
12752    impl InvalidTaskPayloadCodec {
12753        fn label(self) -> &'static str {
12754            match self {
12755                Self::Missing => "missing",
12756                Self::Null => "null",
12757                Self::NonString => "non-string",
12758            }
12759        }
12760
12761        fn apply(self, task: &mut Value) {
12762            let task = task.as_object_mut().expect("task fixture object");
12763            match self {
12764                Self::Missing => {
12765                    task.remove("payload_codec");
12766                }
12767                Self::Null => {
12768                    task.insert("payload_codec".to_string(), Value::Null);
12769                }
12770                Self::NonString => {
12771                    task.insert("payload_codec".to_string(), json!(42));
12772                }
12773            }
12774        }
12775    }
12776
12777    fn fixture_envelope(value: Value) -> Value {
12778        encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
12779    }
12780
12781    fn fixture_blob(value: Value) -> String {
12782        encode_payload(&value, DEFAULT_CODEC)
12783            .expect("encode Avro test fixture")
12784            .blob
12785    }
12786
12787    #[test]
12788    fn client_builder_rejects_the_sdk_owned_api_suffix() {
12789        for base_url in [
12790            "http://127.0.0.1:8080/api",
12791            "http://localhost:8080/api/",
12792            "https://runtime.example.test/namespaces/orders/api",
12793        ] {
12794            let error = Client::builder(base_url)
12795                .build()
12796                .expect_err("SDK-owned /api suffix must be rejected during build");
12797
12798            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
12799            assert!(
12800                error.to_string().contains("SDK appends /api automatically"),
12801                "the validation error must explain how to fix the endpoint"
12802            );
12803        }
12804    }
12805
12806    #[test]
12807    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
12808        for (base_url, expected) in [
12809            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
12810            (
12811                "http://localhost:8080/durable-workflow/",
12812                "http://localhost:8080/durable-workflow",
12813            ),
12814            (
12815                "https://runtime.example.test/namespaces/orders",
12816                "https://runtime.example.test/namespaces/orders",
12817            ),
12818            (
12819                "https://runtime.example.test/gateway/api/namespaces/orders",
12820                "https://runtime.example.test/gateway/api/namespaces/orders",
12821            ),
12822            (
12823                "https://api.example.test/runtime/orders/",
12824                "https://api.example.test/runtime/orders",
12825            ),
12826        ] {
12827            let client = Client::builder(base_url)
12828                .build()
12829                .expect("Server and Cloud runtime base URL must remain valid");
12830
12831            assert_eq!(client.base_url, expected);
12832        }
12833    }
12834
12835    #[test]
12836    fn workflow_completion_uses_the_additive_command_protocol_floor() {
12837        assert_eq!(
12838            workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
12839            WORKER_PROTOCOL_VERSION
12840        );
12841        assert_eq!(
12842            workflow_completion_protocol_version(&[json!({
12843                "type": "upsert_search_attributes",
12844                "attributes": {"OrderStatus": "waiting"},
12845            })]),
12846            SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
12847        );
12848        assert_eq!(
12849            workflow_completion_protocol_version(&[json!({
12850                "type": "upsert_search_attributes",
12851                "attributes": {"OrderStatus": "waiting"},
12852                "attribute_types": {"OrderStatus": "keyword"},
12853            })]),
12854            TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
12855        );
12856        assert_eq!(
12857            workflow_completion_protocol_version(&[
12858                json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
12859                json!({"type": "open_condition_wait", "condition_key": "ready"}),
12860            ]),
12861            MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
12862        );
12863        assert_eq!(
12864            workflow_completion_protocol_version(&[
12865                json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
12866                json!({"type": "open_condition_wait", "condition_key": "ready"}),
12867            ]),
12868            CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
12869        );
12870        assert_eq!(
12871            workflow_completion_protocol_version(&[json!({
12872                "type": "open_condition_wait",
12873                "condition_wait_occurrence_id": "rust:condition-wait:0",
12874                "condition_key": "ready",
12875            })]),
12876            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
12877        );
12878        assert_eq!(
12879            workflow_completion_protocol_version_with_message_streams(
12880                &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
12881                true,
12882            ),
12883            MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
12884        );
12885        assert_eq!(
12886            workflow_completion_protocol_version_with_message_streams(
12887                &[json!({
12888                    "type": "open_condition_wait",
12889                    "condition_wait_occurrence_id": "rust:condition-wait:0",
12890                    "condition_key": "ready",
12891                })],
12892                true,
12893            ),
12894            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
12895        );
12896    }
12897
12898    fn typed_fidelity_probe() -> AvroValue {
12899        AvroValue::Map(BTreeMap::from([
12900            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
12901            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
12902            (
12903                "numeric".to_string(),
12904                AvroValue::Map(BTreeMap::from([
12905                    ("0".to_string(), AvroValue::String("zero".to_string())),
12906                    ("1".to_string(), AvroValue::String("one".to_string())),
12907                ])),
12908            ),
12909            (
12910                "nested".to_string(),
12911                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
12912                    "enabled".to_string(),
12913                    AvroValue::Boolean(true),
12914                )]))]),
12915            ),
12916            (
12917                "projection_collisions".to_string(),
12918                AvroValue::Array(projection_collision_probe()),
12919            ),
12920        ]))
12921    }
12922
12923    fn projection_collision_probe() -> Vec<AvroValue> {
12924        vec![
12925            AvroValue::Map(BTreeMap::from([
12926                ("$type".to_string(), AvroValue::String("bytes".to_string())),
12927                (
12928                    "base64".to_string(),
12929                    AvroValue::String("ordinary user text".to_string()),
12930                ),
12931            ])),
12932            AvroValue::Map(BTreeMap::from([
12933                ("$type".to_string(), AvroValue::String("map".to_string())),
12934                (
12935                    "entries".to_string(),
12936                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
12937                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
12938                        (
12939                            "value".to_string(),
12940                            AvroValue::String("user map".to_string()),
12941                        ),
12942                    ]))]),
12943                ),
12944            ])),
12945        ]
12946    }
12947
12948    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12949    struct TypedContract {
12950        nested: TypedNested,
12951        mode: TypedMode,
12952        optional: Option<String>,
12953        absent: Option<String>,
12954        items: Vec<i64>,
12955        labels: BTreeMap<String, String>,
12956        bytes: serde_bytes::ByteBuf,
12957        signed: i64,
12958        finite: f64,
12959    }
12960
12961    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12962    struct TypedNested {
12963        enabled: bool,
12964    }
12965
12966    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12967    enum TypedMode {
12968        Detailed { label: String },
12969    }
12970
12971    fn typed_contract() -> TypedContract {
12972        TypedContract {
12973            nested: TypedNested { enabled: true },
12974            mode: TypedMode::Detailed {
12975                label: "compiler-checked".to_string(),
12976            },
12977            optional: Some("present".to_string()),
12978            absent: None,
12979            items: vec![i64::MIN, 0, i64::MAX],
12980            labels: BTreeMap::from([
12981                ("language".to_string(), "rust".to_string()),
12982                ("wire".to_string(), "avro".to_string()),
12983            ]),
12984            bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
12985            signed: -9_223_372_036_854_775_000,
12986            finite: 12.5,
12987        }
12988    }
12989
12990    #[derive(Clone, Debug, Default, PartialEq)]
12991    struct ReplayCounterState {
12992        loaded: Option<String>,
12993        count: i64,
12994        finished: bool,
12995    }
12996
12997    fn replay_counter_worker() -> Worker {
12998        let client = Client::new("http://127.0.0.1:8080").expect("client");
12999        let mut worker = Worker::new(client, "rust-workers");
13000        worker.register_replayed_workflow(
13001            "replay-counter",
13002            ReplayCounterState::default,
13003            |ctx, _input, state| async move {
13004                let loaded = ctx.activity("load-counter", json!([])).await?;
13005                state.update(|current| {
13006                    current.loaded = loaded.as_str().map(str::to_string);
13007                })?;
13008                for _ in 0..2 {
13009                    let signal = ctx.wait_signal("increment").await?;
13010                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
13011                    state.update(|current| current.count += amount)?;
13012                }
13013                state.update(|current| current.finished = true)?;
13014                state.read(|current| Ok(json!(current.count)))?
13015            },
13016        );
13017        worker.register_replayed_query::<ReplayCounterState, _, _>(
13018            "replay-counter",
13019            "current",
13020            |_ctx, state, _args| async move {
13021                Ok(json!({
13022                    "loaded": state.loaded,
13023                    "count": state.count,
13024                    "finished": state.finished,
13025                }))
13026            },
13027        );
13028        worker.register_replayed_query::<ReplayCounterState, _, _>(
13029            "replay-counter",
13030            "detached-mutation",
13031            |_ctx, state, _args| async move {
13032                let mut detached = (*state).clone();
13033                detached.count = 999;
13034                Ok(json!(detached.count))
13035            },
13036        );
13037        worker.register_replayed_query::<ReplayCounterState, _, _>(
13038            "replay-counter",
13039            "failed-mutation",
13040            |_ctx, state, _args| async move {
13041                let mut detached = (*state).clone();
13042                detached.count = 999;
13043                Err(Error::WorkerLoop("query refused".to_string()))
13044            },
13045        );
13046        worker
13047    }
13048
13049    fn replay_counter_query(
13050        query_name: &str,
13051        history_events: Value,
13052        run_status: &str,
13053    ) -> QueryTask {
13054        let arguments = fixture_envelope(json!([]));
13055        serde_json::from_value(json!({
13056            "query_task_id": format!("query-{query_name}"),
13057            "workflow_type": "replay-counter",
13058            "query_name": query_name,
13059            "payload_codec": DEFAULT_CODEC,
13060            "workflow_arguments": arguments.clone(),
13061            "query_arguments": arguments,
13062            "history_events": history_events,
13063            "run_status": run_status,
13064        }))
13065        .expect("query task")
13066    }
13067
13068    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
13069        workflow_context_with_codec(history, DEFAULT_CODEC)
13070    }
13071
13072    fn workflow_context_with_codec(
13073        history: Vec<HistoryEvent>,
13074        payload_codec: &str,
13075    ) -> WorkflowContext {
13076        WorkflowContext {
13077            state: Arc::new(Mutex::new(
13078                WorkflowState::new_with_identity(
13079                    history,
13080                    None,
13081                    None,
13082                    "rust-workers".to_string(),
13083                    payload_codec.to_string(),
13084                    None,
13085                )
13086                .expect("valid workflow history"),
13087            )),
13088        }
13089    }
13090
13091    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
13092        HistoryEvent {
13093            event_type: event_type.to_string(),
13094            payload,
13095            raw: HashMap::new(),
13096        }
13097    }
13098
13099    fn parallel_path_entry(
13100        kind: &str,
13101        base: u64,
13102        size: usize,
13103        index: usize,
13104    ) -> ParallelGroupMetadata {
13105        parallel_group_entry(base, size, index, kind)
13106    }
13107
13108    fn parallel_history_event(
13109        event_type: &str,
13110        sequence: u64,
13111        identity_field: &str,
13112        identity: &str,
13113        path: Vec<ParallelGroupMetadata>,
13114        result: Option<Value>,
13115    ) -> HistoryEvent {
13116        let mut payload = serde_json::Map::from_iter([
13117            ("sequence".to_string(), json!(sequence)),
13118            (identity_field.to_string(), json!(identity)),
13119        ]);
13120        let inner = path.last().expect("parallel history path");
13121        apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
13122        payload.insert("parallel_group_path".to_string(), json!(path));
13123        if let Some(result) = result {
13124            let field = if event_type == "ChildRunCompleted" {
13125                "result"
13126            } else {
13127                "result"
13128            };
13129            payload.insert(field.to_string(), fixture_envelope(result));
13130            payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
13131        }
13132        history_event(event_type, Value::Object(payload))
13133    }
13134
13135    fn nested_parallel_operations() -> Vec<ParallelOperation> {
13136        vec![
13137            ParallelOperation::activity("first", json!([])),
13138            ParallelOperation::group(vec![
13139                ParallelOperation::child_workflow(
13140                    "second",
13141                    ChildWorkflowOptions::new("child-workers"),
13142                    json!([]),
13143                ),
13144                ParallelOperation::activity("third", json!([])),
13145            ]),
13146        ]
13147    }
13148
13149    fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
13150        let outer = [
13151            parallel_path_entry("mixed", 1, 3, 0),
13152            parallel_path_entry("mixed", 1, 3, 1),
13153            parallel_path_entry("mixed", 1, 3, 2),
13154        ];
13155        [
13156            vec![outer[0].clone()],
13157            vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
13158            vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
13159        ]
13160    }
13161
13162    #[test]
13163    fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
13164        let ctx = workflow_context(Vec::new());
13165        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
13166        let mut task_context = TaskContext::from_waker(noop_waker_ref());
13167
13168        assert!(matches!(
13169            call.as_mut().poll(&mut task_context),
13170            Poll::Pending
13171        ));
13172        let commands = ctx.take_commands().expect("parallel commands");
13173        assert_eq!(
13174            commands
13175                .iter()
13176                .map(|command| command["type"].as_str().unwrap_or_default())
13177                .collect::<Vec<_>>(),
13178            [
13179                "schedule_activity",
13180                "start_child_workflow",
13181                "schedule_activity"
13182            ]
13183        );
13184        let paths = nested_parallel_paths();
13185        for (command, path) in commands.iter().zip(paths) {
13186            assert_eq!(command["parallel_group_path"], json!(path));
13187            assert_eq!(
13188                command["parallel_group_id"],
13189                json!(path.last().expect("inner group").parallel_group_id)
13190            );
13191        }
13192    }
13193
13194    fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
13195        let paths = nested_parallel_paths();
13196        let third = parallel_history_event(
13197            "ActivityCompleted",
13198            3,
13199            "activity_type",
13200            "third",
13201            paths[2].clone(),
13202            Some(json!("three")),
13203        );
13204        vec![
13205            parallel_history_event(
13206                "ActivityCompleted",
13207                1,
13208                "activity_type",
13209                "first",
13210                paths[0].clone(),
13211                Some(json!("one")),
13212            ),
13213            parallel_history_event(
13214                "ChildWorkflowScheduled",
13215                2,
13216                "child_workflow_type",
13217                "second",
13218                paths[1].clone(),
13219                None,
13220            ),
13221            parallel_history_event(
13222                "ChildRunCompleted",
13223                2,
13224                "child_workflow_type",
13225                "second",
13226                paths[1].clone(),
13227                Some(json!("two")),
13228            ),
13229            third.clone(),
13230            third,
13231        ]
13232    }
13233
13234    #[test]
13235    fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
13236        for _restart_or_completed_replay in 0..2 {
13237            let ctx = workflow_context(completed_nested_parallel_history());
13238            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
13239            let mut task_context = TaskContext::from_waker(noop_waker_ref());
13240            let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
13241                panic!("completed nested parallel history must replay");
13242            };
13243            assert_eq!(
13244                results,
13245                vec![
13246                    ParallelResult::Activity(json!("one")),
13247                    ParallelResult::Group(vec![
13248                        ParallelResult::ChildWorkflow(ChildWorkflowResult {
13249                            parent: WorkflowIdentity {
13250                                workflow_id: None,
13251                                run_id: None,
13252                            },
13253                            child: WorkflowIdentity {
13254                                workflow_id: None,
13255                                run_id: None,
13256                            },
13257                            child_workflow_type: Some("second".to_string()),
13258                            result: json!("two"),
13259                        }),
13260                        ParallelResult::Activity(json!("three")),
13261                    ]),
13262                ]
13263            );
13264            assert!(ctx.take_commands().expect("commands").is_empty());
13265            ctx.ensure_history_consumed().expect("history consumed");
13266        }
13267    }
13268
13269    #[test]
13270    fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
13271        let paths = nested_parallel_paths();
13272        let history = vec![
13273            parallel_history_event(
13274                "ActivityCompleted",
13275                1,
13276                "activity_type",
13277                "first",
13278                paths[0].clone(),
13279                Some(json!("one")),
13280            ),
13281            parallel_history_event(
13282                "ChildWorkflowScheduled",
13283                2,
13284                "child_workflow_type",
13285                "second",
13286                paths[1].clone(),
13287                None,
13288            ),
13289            parallel_history_event(
13290                "ChildRunFailed",
13291                2,
13292                "child_workflow_type",
13293                "second",
13294                paths[1].clone(),
13295                None,
13296            ),
13297            parallel_history_event(
13298                "ActivityCompleted",
13299                3,
13300                "activity_type",
13301                "third",
13302                paths[2].clone(),
13303                Some(json!("late")),
13304            ),
13305        ];
13306        let ctx = workflow_context(history);
13307        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
13308        let mut task_context = TaskContext::from_waker(noop_waker_ref());
13309        let outcome = call.as_mut().poll(&mut task_context);
13310        let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
13311            panic!("one failed child must return a typed partial failure: {outcome:?}");
13312        };
13313        assert_eq!(failure.member_path, [1, 0]);
13314        assert_eq!(failure.group_id, "parallel-calls:1:3");
13315        assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
13316        assert_eq!(
13317            failure
13318                .completed
13319                .iter()
13320                .map(|completion| completion.member_path.clone())
13321                .collect::<Vec<_>>(),
13322            [vec![0], vec![1, 1]]
13323        );
13324    }
13325
13326    #[test]
13327    fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
13328        let paths = nested_parallel_paths();
13329        let history = vec![
13330            parallel_history_event(
13331                "ActivityScheduled",
13332                1,
13333                "activity_type",
13334                "first",
13335                paths[0].clone(),
13336                None,
13337            ),
13338            parallel_history_event(
13339                "ChildWorkflowScheduled",
13340                2,
13341                "child_workflow_type",
13342                "second",
13343                paths[1].clone(),
13344                None,
13345            ),
13346            parallel_history_event(
13347                "ActivityScheduled",
13348                3,
13349                "activity_type",
13350                "third",
13351                paths[2].clone(),
13352                None,
13353            ),
13354        ];
13355        for _restart in 0..2 {
13356            let ctx = workflow_context(history.clone());
13357            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
13358            let mut task_context = TaskContext::from_waker(noop_waker_ref());
13359            let outcome = call.as_mut().poll(&mut task_context);
13360            assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
13361            assert!(ctx.take_commands().expect("commands").is_empty());
13362        }
13363    }
13364
13365    async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
13366        let mut saga = ctx.saga();
13367        let outcome = async {
13368            let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
13369            saga.add_compensation("trip.cancel-flight", json!([flight]))?;
13370            let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
13371            saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
13372            ctx.activity("trip.charge", json!([])).await?;
13373            Ok(json!({"status": "booked"}))
13374        }
13375        .await;
13376        saga.finish(outcome).await
13377    }
13378
13379    fn saga_activity(
13380        event_type: &str,
13381        sequence: u64,
13382        activity_type: &str,
13383        result: Option<Value>,
13384    ) -> HistoryEvent {
13385        let mut payload = json!({
13386            "sequence": sequence,
13387            "activity_type": activity_type,
13388            "message": format!("{activity_type} failed"),
13389            "exception_type": "PlannedFailure",
13390            "non_retryable": true,
13391        });
13392        if let Some(result) = result {
13393            payload["result"] = fixture_envelope(result);
13394        }
13395        history_event(event_type, payload)
13396    }
13397
13398    #[test]
13399    fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
13400        let completed_hotel_compensation = saga_activity(
13401            "ActivityCompleted",
13402            4,
13403            "trip.cancel-hotel",
13404            Some(Value::Null),
13405        );
13406        let history = vec![
13407            saga_activity(
13408                "ActivityCompleted",
13409                1,
13410                "trip.reserve-flight",
13411                Some(json!("flight-1")),
13412            ),
13413            saga_activity(
13414                "ActivityCompleted",
13415                2,
13416                "trip.reserve-hotel",
13417                Some(json!("hotel-1")),
13418            ),
13419            saga_activity("ActivityFailed", 3, "trip.charge", None),
13420            completed_hotel_compensation.clone(),
13421            completed_hotel_compensation,
13422        ];
13423
13424        for _restart in 0..2 {
13425            let ctx = workflow_context(history.clone());
13426            let mut future = Box::pin(trip_saga(ctx.clone()));
13427            let mut task_context = TaskContext::from_waker(noop_waker_ref());
13428            assert!(matches!(
13429                future.as_mut().poll(&mut task_context),
13430                Poll::Pending
13431            ));
13432            let commands = ctx.take_commands().expect("compensation command");
13433            assert_eq!(commands.len(), 1);
13434            assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
13435        }
13436    }
13437
13438    #[test]
13439    fn saga_compensation_failure_preserves_both_typed_failures() {
13440        let history = vec![
13441            saga_activity(
13442                "ActivityCompleted",
13443                1,
13444                "trip.reserve-flight",
13445                Some(json!("flight-1")),
13446            ),
13447            saga_activity(
13448                "ActivityCompleted",
13449                2,
13450                "trip.reserve-hotel",
13451                Some(json!("hotel-1")),
13452            ),
13453            saga_activity("ActivityFailed", 3, "trip.charge", None),
13454            saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
13455        ];
13456        let ctx = workflow_context(history);
13457        let mut future = Box::pin(trip_saga(ctx));
13458        let mut task_context = TaskContext::from_waker(noop_waker_ref());
13459        let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
13460            future.as_mut().poll(&mut task_context)
13461        else {
13462            panic!("compensation failure must remain structured");
13463        };
13464        assert!(matches!(
13465            *failure.initiating_failure,
13466            Error::ActivityFailed(_)
13467        ));
13468        assert!(matches!(
13469            *failure.compensation_failure,
13470            Error::ActivityFailed(_)
13471        ));
13472        assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
13473        assert_eq!(failure.compensation_registration_order, 2);
13474    }
13475
13476    #[test]
13477    fn saga_compensates_cooperative_cancellation() {
13478        let ctx = workflow_context(vec![saga_activity(
13479            "ActivityCompleted",
13480            1,
13481            "trip.reserve-flight",
13482            Some(json!("flight-1")),
13483        )]);
13484        ctx.state.lock().expect("state").cancel_requested = true;
13485        let run = {
13486            let ctx = ctx.clone();
13487            async move {
13488                let mut saga = ctx.saga();
13489                let outcome = async {
13490                    let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
13491                    saga.add_compensation("trip.cancel-flight", json!([flight]))?;
13492                    ctx.throw_if_cancellation_requested()?;
13493                    Ok(json!("unexpected"))
13494                }
13495                .await;
13496                saga.finish(outcome).await
13497            }
13498        };
13499        let mut future = Box::pin(run);
13500        let mut task_context = TaskContext::from_waker(noop_waker_ref());
13501        assert!(matches!(
13502            future.as_mut().poll(&mut task_context),
13503            Poll::Pending
13504        ));
13505        let commands = ctx.take_commands().expect("cancellation compensation");
13506        assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
13507    }
13508
13509    fn workflow_task(
13510        workflow_type: &str,
13511        history_events: Vec<HistoryEvent>,
13512        payload_codec: &str,
13513    ) -> WorkflowTask {
13514        WorkflowTask {
13515            task_id: format!("wft-{workflow_type}"),
13516            workflow_command_id: None,
13517            workflow_id: Some(format!("wf-{workflow_type}")),
13518            run_id: Some(format!("run-{workflow_type}")),
13519            workflow_type: workflow_type.to_string(),
13520            cancel_requested: false,
13521            payload_codec: payload_codec.to_string(),
13522            arguments: Some(
13523                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
13524            ),
13525            total_history_events: Some(history_events.len() as u64),
13526            history_size_bytes: None,
13527            continue_as_new_recommended: None,
13528            history_budget_pressure: None,
13529            history_events,
13530            next_history_page_token: None,
13531            workflow_task_attempt: 1,
13532            workflow_signal_id: None,
13533            signal_name: None,
13534            signal_arguments: None,
13535            workflow_update_id: None,
13536            update_name: None,
13537            lease_owner: Some("rust-worker".to_string()),
13538        }
13539    }
13540
13541    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
13542    struct SideEffectProbe {
13543        request_id: String,
13544        attempt: u32,
13545    }
13546
13547    #[test]
13548    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
13549        let calls = AtomicUsize::new(0);
13550        let ctx = workflow_context(Vec::new());
13551        let value = ctx
13552            .side_effect(|| {
13553                calls.fetch_add(1, Ordering::SeqCst);
13554                SideEffectProbe {
13555                    request_id: "request-42".to_string(),
13556                    attempt: 3,
13557                }
13558            })
13559            .expect("first side effect");
13560        assert_eq!(value.attempt, 3);
13561        assert_eq!(calls.load(Ordering::SeqCst), 1);
13562        let commands = ctx.take_commands().expect("commands");
13563        assert_eq!(commands.len(), 1);
13564        assert_eq!(commands[0]["type"], "record_side_effect");
13565        assert_eq!(
13566            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
13567            serde_json::to_value(&value).expect("value")
13568        );
13569
13570        let replay = workflow_context(vec![history_event(
13571            "SideEffectRecorded",
13572            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
13573        )]);
13574        let replayed: SideEffectProbe = replay
13575            .side_effect(|| {
13576                calls.fetch_add(1, Ordering::SeqCst);
13577                panic!("committed side-effect callbacks must not run during replay")
13578            })
13579            .expect("replayed side effect");
13580        assert_eq!(replayed, value);
13581        assert_eq!(calls.load(Ordering::SeqCst), 1);
13582        assert!(replay.take_commands().expect("commands").is_empty());
13583        replay.ensure_history_consumed().expect("history consumed");
13584    }
13585
13586    #[test]
13587    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
13588        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
13589        let value = ctx
13590            .side_effect(|| SideEffectProbe {
13591                request_id: "avro-request".to_string(),
13592                attempt: 1,
13593            })
13594            .expect("Avro side effect");
13595        let uuid = ctx.uuid_v4().expect("deterministic UUID");
13596        let commands = ctx.take_commands().expect("commands");
13597        assert_eq!(commands.len(), 2);
13598        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
13599        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
13600        assert_eq!(
13601            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
13602            serde_json::to_value(&value).expect("value")
13603        );
13604
13605        let replay = workflow_context_with_codec(
13606            vec![
13607                history_event(
13608                    "SideEffectRecorded",
13609                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
13610                ),
13611                history_event(
13612                    "SideEffectRecorded",
13613                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
13614                ),
13615            ],
13616            DEFAULT_CODEC,
13617        );
13618        let replayed: SideEffectProbe = replay
13619            .side_effect(|| panic!("Avro callback must not run"))
13620            .expect("replayed Avro value");
13621        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
13622        assert_eq!(replayed, value);
13623        assert_eq!(replayed_uuid, uuid);
13624        assert!(replay.take_commands().expect("commands").is_empty());
13625    }
13626
13627    #[test]
13628    fn typed_side_effect_replay_preserves_bytes_and_maps() {
13629        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
13630        let value = ctx
13631            .side_effect_avro_value(typed_fidelity_probe)
13632            .expect("typed side effect");
13633        let commands = ctx.take_commands().expect("side-effect command");
13634        assert_eq!(
13635            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
13636                .expect("recorded side effect"),
13637            value
13638        );
13639
13640        let replay = workflow_context_with_codec(
13641            vec![history_event(
13642                "SideEffectRecorded",
13643                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
13644            )],
13645            DEFAULT_CODEC,
13646        );
13647        assert_eq!(
13648            replay
13649                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
13650                .expect("replayed typed side effect"),
13651            value
13652        );
13653    }
13654
13655    #[test]
13656    fn ordered_side_effects_share_the_durable_command_stream() {
13657        let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
13658        let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
13659        let ctx = workflow_context(vec![
13660            history_event(
13661                "SideEffectRecorded",
13662                json!({"sequence": 1, "result": first}),
13663            ),
13664            history_event(
13665                "SideEffectRecorded",
13666                json!({"sequence": 2, "result": second}),
13667            ),
13668        ]);
13669        let first: String = ctx
13670            .side_effect(|| panic!("first callback must not run"))
13671            .expect("first replay");
13672        let second: i32 = ctx
13673            .side_effect(|| panic!("second callback must not run"))
13674            .expect("second replay");
13675        assert_eq!(first, "first");
13676        assert_eq!(second, 29);
13677        ctx.ensure_history_consumed().expect("ordered history");
13678
13679        let reordered = workflow_context(vec![history_event(
13680            "VersionMarkerRecorded",
13681            json!({
13682                "sequence": 1,
13683                "change_id": "before-side-effect",
13684                "version": 1,
13685                "min_supported": 1,
13686                "max_supported": 1,
13687            }),
13688        )]);
13689        let error = reordered
13690            .side_effect(|| "new".to_string())
13691            .expect_err("command reordering must fail");
13692        assert!(matches!(
13693            error,
13694            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13695                if reason == "recorded_command_mismatch"
13696        ));
13697    }
13698
13699    #[test]
13700    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
13701        let ctx = workflow_context(Vec::new());
13702        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
13703        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
13704        assert!(ctx.patched("new-search").expect("patch"));
13705        ctx.deprecate_patch("new-search").expect("deprecate patch");
13706        let commands = ctx.take_commands().expect("commands");
13707        assert_eq!(commands.len(), 2);
13708        assert_eq!(commands[0]["type"], "record_version_marker");
13709        assert_eq!(commands[0]["version"], 2);
13710        assert_eq!(commands[1]["change_id"], "new-search");
13711
13712        let replay = workflow_context(vec![history_event(
13713            "VersionMarkerRecorded",
13714            json!({
13715                "sequence": 1,
13716                "change_id": "checkout-v2",
13717                "version": 2,
13718                "min_supported": 1,
13719                "max_supported": 2,
13720            }),
13721        )]);
13722        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
13723        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
13724        assert!(replay.take_commands().expect("commands").is_empty());
13725        replay.ensure_history_consumed().expect("history consumed");
13726    }
13727
13728    #[test]
13729    fn version_markers_reject_incompatible_or_malformed_history() {
13730        let incompatible = workflow_context(vec![history_event(
13731            "VersionMarkerRecorded",
13732            json!({
13733                "sequence": 1,
13734                "change_id": "checkout-v2",
13735                "version": 1,
13736                "min_supported": 1,
13737                "max_supported": 2,
13738            }),
13739        )]);
13740        let error = incompatible
13741            .get_version("checkout-v2", 2, 3)
13742            .expect_err("old version is unsupported");
13743        assert!(matches!(
13744            error,
13745            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13746                if reason == "version_marker_incompatible_range"
13747        ));
13748
13749        for (history, reason) in [
13750            (
13751                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
13752                "side_effect_result_missing",
13753            ),
13754            (
13755                vec![history_event(
13756                    "SideEffectRecorded",
13757                    json!({
13758                        "sequence": 1,
13759                        "result": {"codec": "avro", "blob": "not-base64"},
13760                    }),
13761                )],
13762                "side_effect_payload_incompatible",
13763            ),
13764            (
13765                vec![history_event(
13766                    "SideEffectRecorded",
13767                    json!({"sequence": 1, "result": {"unwrapped": true}}),
13768                )],
13769                "side_effect_payload_malformed",
13770            ),
13771            (
13772                vec![history_event(
13773                    "VersionMarkerRecorded",
13774                    json!({
13775                        "sequence": 1,
13776                        "change_id": "change",
13777                        "version": 1,
13778                        "min_supported": 2,
13779                        "max_supported": 1,
13780                    }),
13781                )],
13782                "version_marker_history_range_invalid",
13783            ),
13784        ] {
13785            let error = WorkflowState::new(
13786                history,
13787                "rust-workers".to_string(),
13788                DEFAULT_CODEC.to_string(),
13789                None,
13790            )
13791            .expect_err("malformed history must fail");
13792            assert!(matches!(
13793                error,
13794                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
13795                    if actual == reason
13796            ));
13797        }
13798    }
13799
13800    #[test]
13801    fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
13802        let history = vec![history_event(
13803            "SearchAttributesUpserted",
13804            json!({
13805                "sequence": 1,
13806                "attributes": {"customer_tier": "gold"},
13807                "attribute_types": {"customer_tier": "keyword"},
13808                "merged": {"customer_tier": "gold"}
13809            }),
13810        )];
13811
13812        let matching = workflow_context(history.clone());
13813        matching
13814            .upsert_search_attributes(
13815                SearchAttributeUpdate::new()
13816                    .keyword("customer_tier", "gold")
13817                    .expect("keyword update"),
13818            )
13819            .expect("matching typed update must replay");
13820        matching
13821            .ensure_history_consumed()
13822            .expect("history consumed");
13823
13824        let changed_type = workflow_context(history.clone());
13825        let error = changed_type
13826            .upsert_search_attributes(
13827                SearchAttributeUpdate::new()
13828                    .string("customer_tier", "gold")
13829                    .expect("string update"),
13830            )
13831            .expect_err("same JSON value with a different declaration must be nondeterministic");
13832        let Error::NonDeterministicReplay(failure) = error else {
13833            panic!("typed identity drift must be a replay failure");
13834        };
13835        assert_eq!(failure.reason, "search_attribute_type_mismatch");
13836        assert_eq!(failure.sequence, Some(1));
13837
13838        let changed_value = workflow_context(history);
13839        let error = changed_value
13840            .upsert_search_attributes(
13841                SearchAttributeUpdate::new()
13842                    .keyword("customer_tier", "platinum")
13843                    .expect("keyword update"),
13844            )
13845            .expect_err("changed values must be nondeterministic");
13846        let Error::NonDeterministicReplay(failure) = error else {
13847            panic!("value drift must be a replay failure");
13848        };
13849        assert_eq!(failure.reason, "search_attribute_value_mismatch");
13850    }
13851
13852    #[test]
13853    fn legacy_search_attribute_history_keeps_type_identity_unknown() {
13854        let history = vec![history_event(
13855            "SearchAttributesUpserted",
13856            json!({
13857                "sequence": 1,
13858                "attributes": {"customer_tier": "gold"},
13859                "merged": {"customer_tier": "gold"}
13860            }),
13861        )];
13862
13863        for update in [
13864            SearchAttributeUpdate::new()
13865                .keyword("customer_tier", "gold")
13866                .expect("keyword update"),
13867            SearchAttributeUpdate::new()
13868                .string("customer_tier", "gold")
13869                .expect("string update"),
13870        ] {
13871            let restarted = workflow_context(history.clone());
13872            restarted
13873                .upsert_search_attributes(update)
13874                .expect("legacy history constrains values but has unknown type identity");
13875            restarted
13876                .ensure_history_consumed()
13877                .expect("history consumed");
13878        }
13879    }
13880
13881    #[test]
13882    fn search_attribute_command_emits_canonical_types() {
13883        let ctx = workflow_context(Vec::new());
13884        ctx.upsert_search_attributes(
13885            SearchAttributeUpdate::new()
13886                .keyword("customer_tier", "gold")
13887                .expect("keyword update")
13888                .int("attempts", 3)
13889                .expect("int update")
13890                .delete("obsolete")
13891                .expect("delete update"),
13892        )
13893        .expect("valid search attributes");
13894
13895        assert_eq!(
13896            ctx.take_commands().expect("commands"),
13897            vec![json!({
13898                "type": "upsert_search_attributes",
13899                "attributes": {
13900                    "attempts": 3,
13901                    "customer_tier": "gold",
13902                    "obsolete": null
13903                },
13904                "attribute_types": {
13905                    "attempts": "int",
13906                    "customer_tier": "keyword"
13907                }
13908            })]
13909        );
13910    }
13911
13912    #[test]
13913    fn duplicate_side_effects_and_version_markers_are_rejected() {
13914        let duplicate_side_effect = WorkflowState::new(
13915            vec![
13916                history_event(
13917                    "SideEffectRecorded",
13918                    json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
13919                ),
13920                history_event(
13921                    "SideEffectRecorded",
13922                    json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
13923                ),
13924            ],
13925            "rust-workers".to_string(),
13926            DEFAULT_CODEC.to_string(),
13927            None,
13928        )
13929        .expect_err("duplicate side effect");
13930        assert!(matches!(
13931            duplicate_side_effect,
13932            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13933                if reason == "duplicate_side_effect_record"
13934        ));
13935
13936        let marker = |sequence| {
13937            history_event(
13938                "VersionMarkerRecorded",
13939                json!({
13940                    "sequence": sequence,
13941                    "change_id": "same-change",
13942                    "version": 1,
13943                    "min_supported": 1,
13944                    "max_supported": 1,
13945                }),
13946            )
13947        };
13948        let duplicate_marker = WorkflowState::new(
13949            vec![marker(1), marker(3)],
13950            "rust-workers".to_string(),
13951            DEFAULT_CODEC.to_string(),
13952            None,
13953        )
13954        .expect_err("duplicate marker");
13955        assert!(matches!(
13956            duplicate_marker,
13957            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13958                if reason == "duplicate_version_marker"
13959        ));
13960    }
13961
13962    #[test]
13963    fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
13964        let mut state = WorkflowState::new(
13965            Vec::new(),
13966            "rust-workers".to_string(),
13967            DEFAULT_CODEC.to_string(),
13968            None,
13969        )
13970        .expect("workflow state");
13971        state.workflow_command_identity = "command-7".to_string();
13972        let context = WorkflowContext {
13973            state: Arc::new(Mutex::new(state)),
13974        };
13975        let item =
13976            WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
13977
13978        context
13979            .append_workflow_stream("output", &[item], Some(10))
13980            .expect("append command");
13981        context
13982            .error_workflow_stream("output", "producer failed", None)
13983            .expect("error command");
13984        let commands = context.take_commands().expect("commands");
13985
13986        assert_eq!(commands[0]["type"], "record_side_effect");
13987        assert_eq!(
13988            commands[0]["workflow_stream"]["command_identity"],
13989            "command-7"
13990        );
13991        assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
13992        assert_eq!(
13993            commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
13994            "dw-stream:command-7:0:0"
13995        );
13996        assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
13997
13998        let recorded = history_event(
13999            "SideEffectRecorded",
14000            json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
14001        );
14002        let mut replay_state = WorkflowState::new(
14003            vec![recorded],
14004            "rust-workers".to_string(),
14005            DEFAULT_CODEC.to_string(),
14006            None,
14007        )
14008        .expect("replay state");
14009        replay_state.workflow_command_identity = "command-7".to_string();
14010        let replay_context = WorkflowContext {
14011            state: Arc::new(Mutex::new(replay_state)),
14012        };
14013        replay_context
14014            .append_workflow_stream(
14015                "output",
14016                &[WorkflowStreamAppendItem::from_reference(
14017                    "s3://bucket/item.avro",
14018                )],
14019                Some(10),
14020            )
14021            .expect("replayed append");
14022        assert!(replay_context
14023            .take_commands()
14024            .expect("replayed commands")
14025            .is_empty());
14026    }
14027
14028    #[test]
14029    fn workflow_stream_authoring_requires_server_durable_command_identity() {
14030        let context = workflow_context(Vec::new());
14031        let error = context
14032            .append_workflow_stream(
14033                "output",
14034                &[WorkflowStreamAppendItem::from_reference(
14035                    "s3://bucket/item.avro",
14036                )],
14037                None,
14038            )
14039            .expect_err("stream append without durable command identity must fail closed");
14040
14041        assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
14042        assert!(context.take_commands().expect("commands").is_empty());
14043    }
14044
14045    #[test]
14046    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
14047        fn worker(calls: Arc<AtomicUsize>) -> Worker {
14048            let client = Client::new("http://127.0.0.1:8080").expect("client");
14049            let mut worker = Worker::new(client, "rust-workers");
14050            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
14051                let calls = Arc::clone(&calls);
14052                async move {
14053                    let captured = ctx.side_effect(|| {
14054                        calls.fetch_add(1, Ordering::SeqCst);
14055                        "captured-once".to_string()
14056                    })?;
14057                    let version = ctx.get_version("cold-restart", 1, 2)?;
14058                    Ok(json!({"captured": captured, "version": version}))
14059                }
14060            });
14061            worker
14062        }
14063
14064        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
14065            WorkflowTask {
14066                task_id: "wft-side-effect-version".to_string(),
14067                workflow_command_id: None,
14068                workflow_id: Some("wf-side-effect-version".to_string()),
14069                run_id: Some("run-side-effect-version".to_string()),
14070                workflow_type: "rust.side-effect-version".to_string(),
14071                cancel_requested: false,
14072                payload_codec: DEFAULT_CODEC.to_string(),
14073                arguments: Some(
14074                    encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
14075                ),
14076                history_events,
14077                total_history_events: None,
14078                history_size_bytes: None,
14079                continue_as_new_recommended: None,
14080                history_budget_pressure: None,
14081                next_history_page_token: None,
14082                workflow_task_attempt: 1,
14083                workflow_signal_id: None,
14084                signal_name: None,
14085                signal_arguments: None,
14086                workflow_update_id: None,
14087                update_name: None,
14088                lease_owner: Some("rust-worker".to_string()),
14089            }
14090        }
14091
14092        let calls = Arc::new(AtomicUsize::new(0));
14093        let initial = worker(Arc::clone(&calls))
14094            .execute_workflow_task(task(Vec::new()))
14095            .expect("initial execution");
14096        assert_eq!(
14097            initial
14098                .iter()
14099                .map(|command| &command["type"])
14100                .collect::<Vec<_>>(),
14101            vec![
14102                "record_side_effect",
14103                "record_version_marker",
14104                "complete_workflow"
14105            ]
14106        );
14107        assert_eq!(calls.load(Ordering::SeqCst), 1);
14108
14109        let restarted = worker(Arc::clone(&calls));
14110        let replayed = restarted
14111            .execute_workflow_task(task(vec![
14112                history_event(
14113                    "SideEffectRecorded",
14114                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
14115                ),
14116                history_event(
14117                    "VersionMarkerRecorded",
14118                    json!({
14119                        "sequence": 2,
14120                        "change_id": "cold-restart",
14121                        "version": 2,
14122                        "min_supported": 1,
14123                        "max_supported": 2,
14124                    }),
14125                ),
14126            ]))
14127            .expect("cold replay");
14128        assert_eq!(replayed.len(), 1);
14129        assert_eq!(replayed[0]["type"], "complete_workflow");
14130        assert_eq!(calls.load(Ordering::SeqCst), 1);
14131    }
14132
14133    #[test]
14134    fn side_effect_replay_rejects_changed_rust_value_type() {
14135        let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
14136        let ctx = workflow_context(vec![history_event(
14137            "SideEffectRecorded",
14138            json!({"sequence": 1, "result": result}),
14139        )]);
14140        let error = ctx
14141            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
14142            .expect_err("changed type must fail replay");
14143        assert!(matches!(
14144            error,
14145            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
14146                if reason == "side_effect_type_mismatch"
14147        ));
14148    }
14149
14150    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
14151        vec![
14152            history_event(
14153                "ActivityScheduled",
14154                json!({
14155                    "sequence": 1,
14156                    "activity_type": "flaky",
14157                    "activity_execution_id": "act-1",
14158                    "activity": {
14159                        "id": "act-1",
14160                        "sequence": 1,
14161                        "type": "flaky",
14162                        "queue": "critical-activities",
14163                        "execution_mode": null,
14164                        "retry_policy": {
14165                            "snapshot_version": 1,
14166                            "max_attempts": 3,
14167                            "backoff_seconds": [2, 4],
14168                            "start_to_close_timeout": 30,
14169                            "schedule_to_start_timeout": 5,
14170                            "schedule_to_close_timeout": 90,
14171                            "heartbeat_timeout": 10,
14172                            "non_retryable_error_types": ["PermanentError"]
14173                        }
14174                    }
14175                }),
14176            ),
14177            history_event(
14178                "ActivityStarted",
14179                json!({
14180                    "sequence": 1,
14181                    "activity_type": "flaky",
14182                    "activity_execution_id": "act-1",
14183                    "activity_attempt_id": "attempt-1",
14184                    "attempt_number": 1
14185                }),
14186            ),
14187            history_event(
14188                "ActivityRetryScheduled",
14189                json!({
14190                    "sequence": 1,
14191                    "activity_type": "flaky",
14192                    "activity_execution_id": "act-1",
14193                    "activity_attempt_id": "attempt-1",
14194                    "attempt_number": 1,
14195                    "retry_after_attempt": 1,
14196                    "retry_backoff_seconds": 2,
14197                    "failure_category": "activity",
14198                    "exception_type": "TransientError"
14199                }),
14200            ),
14201            history_event(
14202                "ActivityStarted",
14203                json!({
14204                    "sequence": 1,
14205                    "activity_type": "flaky",
14206                    "activity_execution_id": "act-1",
14207                    "activity_attempt_id": "attempt-2",
14208                    "attempt_number": 2
14209                }),
14210            ),
14211            history_event(
14212                "ActivityCompleted",
14213                json!({
14214                    "sequence": 1,
14215                    "activity_type": "flaky",
14216                    "activity_execution_id": "act-1",
14217                    "activity_attempt_id": "attempt-2",
14218                    "attempt_number": 2,
14219                    "payload_codec": DEFAULT_CODEC,
14220                    "result": fixture_envelope(json!({"status":"recovered"}))
14221                }),
14222            ),
14223        ]
14224    }
14225
14226    fn retry_activity_options() -> ActivityOptions {
14227        ActivityOptions::new()
14228            .task_queue("critical-activities")
14229            .retry_policy(
14230                ActivityRetryPolicy::new(3)
14231                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
14232                    .non_retryable_error_type("PermanentError"),
14233            )
14234            .start_to_close_timeout(Duration::from_secs(30))
14235            .schedule_to_start_timeout(Duration::from_secs(5))
14236            .schedule_to_close_timeout(Duration::from_secs(90))
14237            .heartbeat_timeout(Duration::from_secs(10))
14238    }
14239
14240    #[test]
14241    fn fixed_avro_value_round_trips_json_values() {
14242        let value = json!({"greeting": "hello", "count": 3, "ok": true});
14243        let envelope = PayloadEnvelope::avro(&value).expect("encode");
14244        assert_eq!(envelope.codec, DEFAULT_CODEC);
14245        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
14246    }
14247
14248    #[tokio::test]
14249    async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
14250        let client = Client::new("http://127.0.0.1:8080").expect("client");
14251        let mut worker = Worker::new(client, "rust-workers");
14252        worker.register_typed_workflow(
14253            "typed.contract.workflow",
14254            |_ctx, input: TypedContract| async move { Ok(input) },
14255        );
14256        worker.register_typed_activity(
14257            "typed.contract.activity",
14258            |_ctx, input: TypedContract| async move { Ok(input) },
14259        );
14260
14261        let expected = typed_contract();
14262        let arguments = AvroValue::Array(vec![
14263            AvroValue::from_serialize(&expected).expect("typed request")
14264        ]);
14265        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
14266        let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
14267        workflow.arguments = Some(envelope.clone());
14268        let commands = worker
14269            .execute_workflow_task(workflow)
14270            .expect("typed workflow task");
14271        let workflow_result: TypedContract =
14272            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
14273                .expect("workflow result envelope")
14274                .deserialize()
14275                .expect("workflow result type");
14276        assert_eq!(workflow_result, expected);
14277
14278        let activity = ActivityTask {
14279            task_id: "typed-contract-activity".to_string(),
14280            activity_attempt_id: Some("typed-contract-attempt".to_string()),
14281            attempt_id: None,
14282            activity_type: "typed.contract.activity".to_string(),
14283            payload_codec: DEFAULT_CODEC.to_string(),
14284            arguments: Some(envelope),
14285            attempt_number: 1,
14286            lease_owner: Some("rust-worker".to_string()),
14287        };
14288        let activity_result: TypedContract = worker
14289            .execute_activity_task(activity)
14290            .await
14291            .expect("typed activity task")
14292            .deserialize()
14293            .expect("activity result type");
14294        assert_eq!(activity_result, expected);
14295    }
14296
14297    #[tokio::test]
14298    async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
14299        let client = Client::new("http://127.0.0.1:8080").expect("client");
14300        let mut worker = Worker::new(client, "rust-workers");
14301        worker.register_typed_workflow(
14302            "typed.shape.workflow",
14303            |_ctx, input: TypedContract| async move { Ok(input) },
14304        );
14305        worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
14306            Ok(f64::NAN)
14307        });
14308
14309        let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
14310        workflow.arguments = Some(
14311            encode_typed_envelope(
14312                &AvroValue::Array(vec![
14313                    AvroValue::String("first".to_string()),
14314                    AvroValue::String("second".to_string()),
14315                ]),
14316                DEFAULT_CODEC,
14317            )
14318            .expect("malformed typed arguments"),
14319        );
14320        let commands = worker
14321            .execute_workflow_task(workflow)
14322            .expect("shape mismatch becomes a workflow failure");
14323        let message = commands[0]["message"].as_str().expect("failure message");
14324        assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
14325        assert!(message.contains(type_name::<TypedContract>()));
14326        assert!(message.contains("task carried 2 arguments"));
14327
14328        let activity = ActivityTask {
14329            task_id: "typed-unsupported-activity".to_string(),
14330            activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
14331            attempt_id: None,
14332            activity_type: "typed.unsupported.activity".to_string(),
14333            payload_codec: DEFAULT_CODEC.to_string(),
14334            arguments: Some(
14335                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
14336                    .expect("unit arguments"),
14337            ),
14338            attempt_number: 1,
14339            lease_owner: Some("rust-worker".to_string()),
14340        };
14341        let Error::HandlerType {
14342            handler_kind,
14343            handler_name,
14344            value_kind,
14345            rust_type,
14346            message,
14347        } = worker
14348            .execute_activity_task(activity)
14349            .await
14350            .expect_err("non-finite handler output must fail")
14351        else {
14352            panic!("expected contextual handler type failure");
14353        };
14354        assert_eq!(handler_kind, HandlerKind::Activity);
14355        assert_eq!(handler_name, "typed.unsupported.activity");
14356        assert_eq!(value_kind, HandlerValueKind::Result);
14357        assert_eq!(rust_type, type_name::<f64>());
14358        assert!(message.contains("non_finite_float"));
14359    }
14360
14361    #[tokio::test]
14362    async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
14363        #[derive(Clone, Default)]
14364        struct State {
14365            observed: Option<TypedContract>,
14366        }
14367
14368        let client = Client::new("http://127.0.0.1:8080").expect("client");
14369        let mut worker = Worker::new(client, "rust-workers");
14370        worker.register_typed_replayed_workflow(
14371            "typed.contract.replayed",
14372            State::default,
14373            |ctx, input: TypedContract, state| async move {
14374                let result: TypedContract =
14375                    ctx.activity_typed("typed.contract.activity", input).await?;
14376                state.update(|current| current.observed = Some(result.clone()))?;
14377                Ok(result)
14378            },
14379        );
14380        worker.register_replayed_query::<State, _, _>(
14381            "typed.contract.replayed",
14382            "observed",
14383            |_ctx, state, _args| async move {
14384                Ok(json!(state.observed.as_ref().map(|value| value.signed)))
14385            },
14386        );
14387
14388        let expected = typed_contract();
14389        let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
14390        let workflow_arguments =
14391            encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
14392                .expect("workflow arguments");
14393        let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
14394        let task = QueryTask {
14395            query_task_id: "typed-replay-query".to_string(),
14396            query_task_attempt: 1,
14397            lease_owner: Some("rust-worker".to_string()),
14398            workflow_id: Some("typed-replay".to_string()),
14399            run_id: Some("typed-replay-run".to_string()),
14400            workflow_type: "typed.contract.replayed".to_string(),
14401            query_name: "observed".to_string(),
14402            payload_codec: DEFAULT_CODEC.to_string(),
14403            workflow_arguments: Some(workflow_arguments),
14404            query_arguments: Some(
14405                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
14406                    .expect("query arguments"),
14407            ),
14408            history_events: vec![
14409                history_event(
14410                    "ActivityScheduled",
14411                    json!({
14412                        "sequence": 1,
14413                        "activity_type": "typed.contract.activity"
14414                    }),
14415                ),
14416                history_event(
14417                    "ActivityCompleted",
14418                    json!({
14419                        "sequence": 1,
14420                        "activity_type": "typed.contract.activity",
14421                        "payload_codec": DEFAULT_CODEC,
14422                        "result": result
14423                    }),
14424                ),
14425            ],
14426            history_export: None,
14427            run_status: Some("completed".to_string()),
14428        };
14429
14430        assert_eq!(
14431            worker
14432                .execute_query_task(task)
14433                .await
14434                .expect("typed replay query")
14435                .deserialize::<i64>()
14436                .expect("query result"),
14437            expected.signed
14438        );
14439    }
14440
14441    #[tokio::test]
14442    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
14443        let client = Client::new("http://127.0.0.1:8080").expect("client");
14444        let mut worker = Worker::new(client, "rust-workers");
14445        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
14446        worker
14447            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
14448        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
14449            Ok(input)
14450        });
14451        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
14452            Ok(input)
14453        });
14454        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
14455            Ok(AvroValue::Array(
14456                ctx.wait_signal_avro_value("changed").await?,
14457            ))
14458        });
14459
14460        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
14461        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
14462
14463        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
14464        workflow.arguments = Some(envelope.clone());
14465        let commands = worker
14466            .execute_workflow_task(workflow)
14467            .expect("typed workflow task");
14468        assert_eq!(commands[0]["type"], "complete_workflow");
14469        assert_eq!(
14470            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
14471                .expect("typed workflow result"),
14472            arguments
14473        );
14474
14475        let activity = ActivityTask {
14476            task_id: "activity-typed".to_string(),
14477            activity_attempt_id: Some("attempt-typed".to_string()),
14478            attempt_id: None,
14479            activity_type: "typed.activity".to_string(),
14480            payload_codec: DEFAULT_CODEC.to_string(),
14481            arguments: Some(envelope.clone()),
14482            attempt_number: 1,
14483            lease_owner: Some("rust-worker".to_string()),
14484        };
14485        assert_eq!(
14486            worker
14487                .execute_activity_task(activity)
14488                .await
14489                .expect("typed activity result"),
14490            arguments
14491        );
14492
14493        let query = QueryTask {
14494            query_task_id: "query-typed".to_string(),
14495            query_task_attempt: 1,
14496            lease_owner: Some("rust-worker".to_string()),
14497            workflow_id: Some("typed-1".to_string()),
14498            run_id: Some("run-typed".to_string()),
14499            workflow_type: "typed.echo".to_string(),
14500            query_name: "inspect".to_string(),
14501            payload_codec: DEFAULT_CODEC.to_string(),
14502            workflow_arguments: Some(
14503                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
14504                    .expect("workflow input"),
14505            ),
14506            query_arguments: Some(envelope.clone()),
14507            history_events: Vec::new(),
14508            history_export: None,
14509            run_status: Some("running".to_string()),
14510        };
14511        assert_eq!(
14512            worker
14513                .execute_query_task(query)
14514                .await
14515                .expect("typed query result"),
14516            arguments
14517        );
14518
14519        let mut update = workflow_task(
14520            "typed.echo",
14521            vec![history_event(
14522                "UpdateAccepted",
14523                json!({
14524                    "update_id": "update-typed",
14525                    "update_name": "replace",
14526                    "arguments": envelope.clone(),
14527                }),
14528            )],
14529            DEFAULT_CODEC,
14530        );
14531        update.workflow_update_id = Some("update-typed".to_string());
14532        update.update_name = Some("replace".to_string());
14533        let commands = worker
14534            .execute_workflow_task(update)
14535            .expect("typed update task");
14536        assert_eq!(commands[0]["type"], "complete_update");
14537        assert_eq!(
14538            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
14539                .expect("typed update result"),
14540            arguments
14541        );
14542
14543        let mut signal = workflow_task(
14544            "typed.signal",
14545            vec![history_event(
14546                "SignalReceived",
14547                json!({
14548                    "signal_id": "signal-typed",
14549                    "signal_name": "changed",
14550                    "arguments": envelope.clone(),
14551                }),
14552            )],
14553            DEFAULT_CODEC,
14554        );
14555        signal.workflow_signal_id = Some("signal-typed".to_string());
14556        signal.signal_name = Some("changed".to_string());
14557        signal.signal_arguments = Some(envelope);
14558        let commands = worker
14559            .execute_workflow_task(signal)
14560            .expect("typed signal resume");
14561        assert_eq!(
14562            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
14563                .expect("typed signal result"),
14564            arguments
14565        );
14566    }
14567
14568    #[tokio::test]
14569    async fn typed_helpers_never_parse_json_inspection_projection() {
14570        let collision_values = projection_collision_probe();
14571        let expected = AvroValue::Array(collision_values.clone());
14572        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
14573
14574        let activity_context = workflow_context_with_codec(
14575            vec![history_event(
14576                "ActivityCompleted",
14577                json!({
14578                    "sequence": 1,
14579                    "activity_type": "collision.activity",
14580                    "payload_codec": DEFAULT_CODEC,
14581                    "result": envelope.clone(),
14582                }),
14583            )],
14584            DEFAULT_CODEC,
14585        );
14586        assert_eq!(
14587            activity_context
14588                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
14589                .await
14590                .expect("typed activity collision result"),
14591            expected
14592        );
14593
14594        let signal_context = workflow_context_with_codec(
14595            vec![
14596                history_event(
14597                    "SignalWaitOpened",
14598                    json!({"sequence": 1, "signal_name": "collision"}),
14599                ),
14600                history_event(
14601                    "SignalApplied",
14602                    json!({
14603                        "sequence": 1,
14604                        "signal_name": "collision",
14605                        "payload_codec": DEFAULT_CODEC,
14606                        "value": envelope.clone(),
14607                    }),
14608                ),
14609            ],
14610            DEFAULT_CODEC,
14611        );
14612        assert_eq!(
14613            signal_context
14614                .wait_signal_avro_value("collision")
14615                .await
14616                .expect("typed signal collision arguments"),
14617            collision_values
14618        );
14619
14620        let child_context = workflow_context_with_codec(
14621            vec![
14622                history_event(
14623                    "ChildWorkflowScheduled",
14624                    json!({
14625                        "sequence": 1,
14626                        "child_workflow_instance_id": "collision-child",
14627                        "child_workflow_run_id": "collision-run",
14628                        "child_workflow_type": "collision.child",
14629                    }),
14630                ),
14631                history_event(
14632                    "ChildRunCompleted",
14633                    json!({
14634                        "sequence": 1,
14635                        "child_workflow_instance_id": "collision-child",
14636                        "child_workflow_run_id": "collision-run",
14637                        "child_workflow_type": "collision.child",
14638                        "payload_codec": DEFAULT_CODEC,
14639                        "result": envelope,
14640                    }),
14641                ),
14642            ],
14643            DEFAULT_CODEC,
14644        );
14645        let child = child_context
14646            .start_child_workflow_avro_value(
14647                "collision.child",
14648                ChildWorkflowOptions::new("collision-workers"),
14649                AvroValue::Array(Vec::new()),
14650            )
14651            .await
14652            .expect("typed child collision result");
14653        assert_eq!(child.result, expected);
14654    }
14655
14656    #[tokio::test]
14657    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
14658        let client = Client::new("http://127.0.0.1:8080").expect("client");
14659        let mut worker = Worker::new(client, "rust-workers");
14660        worker.register_replayed_workflow_avro_value(
14661            "typed.replayed",
14662            || (),
14663            |_ctx, input, _state| async move { Ok(input) },
14664        );
14665        worker.register_replayed_query_avro_value::<(), _, _>(
14666            "typed.replayed",
14667            "inspect",
14668            |ctx, _state, args| async move {
14669                let mut signals = ctx.signals_avro_value("collision");
14670                let signal = signals
14671                    .pop()
14672                    .map(AvroValue::Array)
14673                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
14674                Ok(AvroValue::Array(vec![
14675                    ctx.workflow_input_avro_value().clone(),
14676                    signal,
14677                    args,
14678                ]))
14679            },
14680        );
14681        let arguments = AvroValue::Array(projection_collision_probe());
14682        let signal_arguments =
14683            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
14684        let task = QueryTask {
14685            query_task_id: "query-typed-replay".to_string(),
14686            query_task_attempt: 1,
14687            lease_owner: Some("rust-worker".to_string()),
14688            workflow_id: Some("typed-replay".to_string()),
14689            run_id: Some("run-typed-replay".to_string()),
14690            workflow_type: "typed.replayed".to_string(),
14691            query_name: "inspect".to_string(),
14692            payload_codec: DEFAULT_CODEC.to_string(),
14693            workflow_arguments: Some(
14694                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
14695            ),
14696            query_arguments: Some(
14697                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
14698            ),
14699            history_events: vec![history_event(
14700                "SignalReceived",
14701                json!({
14702                    "signal_id": "collision-signal",
14703                    "signal_name": "collision",
14704                    "workflow_sequence": 1,
14705                    "payload_codec": DEFAULT_CODEC,
14706                    "arguments": signal_arguments,
14707                }),
14708            )],
14709            history_export: None,
14710            run_status: Some("completed".to_string()),
14711        };
14712
14713        assert_eq!(
14714            worker
14715                .execute_query_task(task)
14716                .await
14717                .expect("typed replay query"),
14718            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
14719        );
14720    }
14721
14722    #[test]
14723    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
14724        let value = BTreeMap::from([(1_i32, "integer key")]);
14725        let error = PayloadEnvelope::avro(&value)
14726            .expect_err("integer map keys must fail")
14727            .to_string();
14728
14729        assert!(error.contains("invalid_map_key"));
14730    }
14731
14732    #[test]
14733    fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
14734        let envelope = PayloadEnvelope {
14735            codec: "json".to_string(),
14736            blob: r#"{"greeting":"hello"}"#.to_string(),
14737        };
14738
14739        let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
14740        let diagnostic = error.to_string();
14741        assert!(diagnostic.contains("unsupported_payload_codec"));
14742        assert!(diagnostic.contains("codec=\"avro\""));
14743        assert!(diagnostic.contains("HTTP document transport"));
14744    }
14745
14746    #[test]
14747    fn untagged_json_payload_value_fails_closed() {
14748        let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
14749            .expect_err("untagged JSON payload values must fail");
14750        let diagnostic = error.to_string();
14751        assert!(diagnostic.contains("unsupported_payload_codec"));
14752        assert!(diagnostic.contains("untagged durable payload"));
14753        assert!(diagnostic.contains("HTTP document transport"));
14754    }
14755
14756    #[test]
14757    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
14758        let envelope = PayloadEnvelope {
14759            codec: DEFAULT_CODEC.to_string(),
14760            blob: BASE64.encode([0x01]),
14761        };
14762
14763        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
14764        assert!(error.to_string().contains("invalid_payload_framing"));
14765    }
14766
14767    #[tokio::test]
14768    async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
14769        let server = MockWorkerServer::start();
14770        let client = Client::builder(server.base_url())
14771            .timeout(Duration::from_secs(2))
14772            .build()
14773            .expect("client");
14774        let invalid_commands = [
14775            json!({
14776                "type": "complete_workflow",
14777                "result": {"codec": "json", "blob": null}
14778            }),
14779            json!({
14780                "type": "schedule_activity",
14781                "arguments": {"codec": "yaml", "blob": "ignored"}
14782            }),
14783            json!({
14784                "type": "start_child_workflow",
14785                "arguments": {"codec": DEFAULT_CODEC, "blob": null}
14786            }),
14787            json!({"type": "continue_as_new", "arguments": []}),
14788            json!({"type": "complete_update"}),
14789            json!({"type": "record_side_effect", "result": null}),
14790            json!({
14791                "type": "start_service_operation",
14792                "payload_codec": DEFAULT_CODEC,
14793                "request_payload": "raw-avro-bytes"
14794            }),
14795        ];
14796
14797        for command in invalid_commands {
14798            let error = client
14799                .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
14800                .await
14801                .expect_err("invalid durable payload must fail locally");
14802            let diagnostic = error.to_string();
14803            assert!(
14804                diagnostic.contains("unsupported_payload_codec")
14805                    || diagnostic.contains("invalid_payload_envelope")
14806                    || diagnostic.contains("untagged durable payload"),
14807                "unexpected validation diagnostic: {diagnostic}"
14808            );
14809        }
14810
14811        assert_eq!(
14812            server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
14813            0,
14814            "invalid command payloads must not reach HTTP transport"
14815        );
14816    }
14817
14818    #[test]
14819    fn workflow_completion_validates_only_protocol_owned_payload_slots() {
14820        let envelope = fixture_envelope(json!({"codec": "customer-value"}));
14821        let commands = [
14822            json!({"type": "complete_workflow", "result": envelope.clone()}),
14823            json!({"type": "schedule_activity", "arguments": envelope.clone()}),
14824            json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
14825            json!({"type": "continue_as_new", "arguments": envelope.clone()}),
14826            json!({"type": "complete_update", "result": envelope.clone()}),
14827            json!({"type": "record_side_effect", "result": envelope.clone()}),
14828            json!({
14829                "type": "start_service_operation",
14830                "payload_codec": DEFAULT_CODEC,
14831                "request_payload": envelope.clone()
14832            }),
14833            json!({
14834                "type": "complete_workflow",
14835                "result": envelope,
14836                "metadata": {
14837                    "codec": "json",
14838                    "payload_codec": "customer-codec",
14839                    "result": {"codec": "yaml", "blob": null}
14840                }
14841            }),
14842        ];
14843
14844        validate_workflow_task_commands(&commands)
14845            .expect("customer metadata must not become a protocol codec declaration");
14846    }
14847
14848    #[test]
14849    fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
14850        assert_eq!(
14851            decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
14852            AvroValue::Array(Vec::new())
14853        );
14854        assert_eq!(
14855            decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
14856            AvroValue::Array(Vec::new())
14857        );
14858
14859        let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
14860        signal.signal_name = Some("empty-signal".to_string());
14861        signal.signal_arguments = None;
14862        let decoded = decode_resume_signal(&signal)
14863            .expect("valid Avro signal")
14864            .expect("named signal resumes the workflow");
14865        assert!(decoded.arguments.is_empty());
14866    }
14867
14868    #[tokio::test]
14869    async fn malformed_task_level_codecs_become_pre_handler_failures() {
14870        let client = Client::new("http://127.0.0.1:8080").expect("client");
14871        let mut worker = Worker::new(client, "rust-workers");
14872        let handler_calls = Arc::new(AtomicUsize::new(0));
14873
14874        let calls = Arc::clone(&handler_calls);
14875        worker.register_workflow("codec.workflow", move |_ctx, _args| {
14876            calls.fetch_add(1, Ordering::SeqCst);
14877            async move { Ok(Value::Null) }
14878        });
14879        let calls = Arc::clone(&handler_calls);
14880        worker.register_activity("codec.activity", move |_ctx, _args| {
14881            calls.fetch_add(1, Ordering::SeqCst);
14882            async move { Ok(Value::Null) }
14883        });
14884        let calls = Arc::clone(&handler_calls);
14885        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
14886            calls.fetch_add(1, Ordering::SeqCst);
14887            async move { Ok(Value::Null) }
14888        });
14889
14890        let mut failures = Vec::new();
14891        for codec_case in [
14892            InvalidTaskPayloadCodec::Missing,
14893            InvalidTaskPayloadCodec::Null,
14894            InvalidTaskPayloadCodec::NonString,
14895        ] {
14896            let mut workflow = json!({
14897                "task_id": format!("workflow-{}", codec_case.label()),
14898                "workflow_type": "codec.workflow"
14899            });
14900            codec_case.apply(&mut workflow);
14901            match serde_json::from_value::<WorkflowTask>(workflow) {
14902                Ok(task) => match worker.execute_workflow_task(task) {
14903                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
14904                    outcome => failures.push(format!(
14905                        "workflow {} codec returned {outcome:?}",
14906                        codec_case.label()
14907                    )),
14908                },
14909                Err(error) => failures.push(format!(
14910                    "workflow {} codec failed transport deserialization: {error}",
14911                    codec_case.label()
14912                )),
14913            }
14914
14915            let mut activity = json!({
14916                "task_id": format!("activity-{}", codec_case.label()),
14917                "activity_attempt_id": format!("attempt-{}", codec_case.label()),
14918                "activity_type": "codec.activity",
14919                "attempt_number": 1
14920            });
14921            codec_case.apply(&mut activity);
14922            match serde_json::from_value::<ActivityTask>(activity) {
14923                Ok(task) => match worker.execute_activity_task(task).await {
14924                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
14925                    outcome => failures.push(format!(
14926                        "activity {} codec returned {outcome:?}",
14927                        codec_case.label()
14928                    )),
14929                },
14930                Err(error) => failures.push(format!(
14931                    "activity {} codec failed transport deserialization: {error}",
14932                    codec_case.label()
14933                )),
14934            }
14935
14936            let mut query = json!({
14937                "query_task_id": format!("query-{}", codec_case.label()),
14938                "workflow_type": "codec.workflow",
14939                "query_name": "known"
14940            });
14941            codec_case.apply(&mut query);
14942            match serde_json::from_value::<QueryTask>(query) {
14943                Ok(task) => match worker.execute_query_task(task).await {
14944                    Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
14945                    outcome => failures.push(format!(
14946                        "query {} codec returned {outcome:?}",
14947                        codec_case.label()
14948                    )),
14949                },
14950                Err(error) => failures.push(format!(
14951                    "query {} codec failed transport deserialization: {error}",
14952                    codec_case.label()
14953                )),
14954            }
14955        }
14956
14957        assert!(failures.is_empty(), "{}", failures.join("\n"));
14958        assert_eq!(
14959            handler_calls.load(Ordering::SeqCst),
14960            0,
14961            "invalid task codecs must not invoke a handler"
14962        );
14963    }
14964
14965    #[tokio::test]
14966    async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
14967        for codec_case in [
14968            InvalidTaskPayloadCodec::Missing,
14969            InvalidTaskPayloadCodec::Null,
14970            InvalidTaskPayloadCodec::NonString,
14971        ] {
14972            let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
14973            let client = Client::builder(server.base_url())
14974                .timeout(Duration::from_secs(2))
14975                .build()
14976                .expect("client");
14977            let mut worker = Worker::new(client, "rust-workers")
14978                .worker_id("codec-worker")
14979                .poll_timeout(Duration::from_millis(10));
14980            let handler_calls = Arc::new(AtomicUsize::new(0));
14981
14982            let calls = Arc::clone(&handler_calls);
14983            worker.register_workflow("codec.workflow", move |_ctx, _args| {
14984                calls.fetch_add(1, Ordering::SeqCst);
14985                async move { Ok(Value::Null) }
14986            });
14987            let calls = Arc::clone(&handler_calls);
14988            worker.register_activity("codec.activity", move |_ctx, _args| {
14989                calls.fetch_add(1, Ordering::SeqCst);
14990                async move { Ok(Value::Null) }
14991            });
14992            let calls = Arc::clone(&handler_calls);
14993            worker.register_query("codec.workflow", "known", move |_ctx, _args| {
14994                calls.fetch_add(1, Ordering::SeqCst);
14995                async move { Ok(Value::Null) }
14996            });
14997
14998            assert_eq!(
14999                worker.run_once().await.expect("invalid tasks are settled"),
15000                3,
15001                "all {} codec tasks must be handled",
15002                codec_case.label()
15003            );
15004            assert_eq!(
15005                handler_calls.load(Ordering::SeqCst),
15006                0,
15007                "{} task codecs must fail before every handler",
15008                codec_case.label()
15009            );
15010
15011            for path in [
15012                "/api/worker/workflow-tasks/codec-workflow/fail",
15013                "/api/worker/activity-tasks/codec-activity/fail",
15014                "/api/worker/query-tasks/codec-query/fail",
15015            ] {
15016                let body = server.request_body(path);
15017                assert!(
15018                    body["failure"]["message"]
15019                        .as_str()
15020                        .is_some_and(|message| message.contains("unsupported_payload_codec")),
15021                    "{path} must receive the stable codec diagnostic for the {} case: {body}",
15022                    codec_case.label()
15023                );
15024            }
15025            assert_eq!(
15026                server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
15027                    ["reason"],
15028                "query_payload_decode_failed"
15029            );
15030            for path in [
15031                "/api/worker/workflow-tasks/codec-workflow/complete",
15032                "/api/worker/activity-tasks/codec-activity/complete",
15033                "/api/worker/query-tasks/codec-query/complete",
15034            ] {
15035                assert_eq!(
15036                    server.request_count(path),
15037                    0,
15038                    "invalid {} codec task reached {path}",
15039                    codec_case.label()
15040                );
15041            }
15042        }
15043    }
15044
15045    #[tokio::test]
15046    async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
15047        let client = Client::new("http://127.0.0.1:8080").expect("client");
15048        let mut worker = Worker::new(client, "rust-workers");
15049        let handler_calls = Arc::new(AtomicUsize::new(0));
15050
15051        let calls = Arc::clone(&handler_calls);
15052        worker.register_workflow("codec.workflow", move |_ctx, _args| {
15053            calls.fetch_add(1, Ordering::SeqCst);
15054            async move { Ok(Value::Null) }
15055        });
15056        let calls = Arc::clone(&handler_calls);
15057        worker.register_activity("codec.activity", move |_ctx, _args| {
15058            calls.fetch_add(1, Ordering::SeqCst);
15059            async move { Ok(Value::Null) }
15060        });
15061        let calls = Arc::clone(&handler_calls);
15062        worker.register_update("codec.workflow", "known", move |_ctx, _args| {
15063            calls.fetch_add(1, Ordering::SeqCst);
15064            async move { Ok(Value::Null) }
15065        });
15066        let calls = Arc::clone(&handler_calls);
15067        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
15068            calls.fetch_add(1, Ordering::SeqCst);
15069            async move { Ok(Value::Null) }
15070        });
15071
15072        let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
15073        workflow.payload_codec = "json".to_string();
15074        workflow.arguments = None;
15075        let error = worker
15076            .execute_workflow_task(workflow)
15077            .expect_err("task codec must be checked before workflow invocation");
15078        assert!(error.to_string().contains("unsupported_payload_codec"));
15079
15080        let activity = ActivityTask {
15081            task_id: "activity-invalid-codec".to_string(),
15082            activity_attempt_id: None,
15083            attempt_id: None,
15084            activity_type: "codec.activity".to_string(),
15085            payload_codec: "unknown".to_string(),
15086            arguments: None,
15087            attempt_number: 1,
15088            lease_owner: None,
15089        };
15090        let error = worker
15091            .execute_activity_task(activity)
15092            .await
15093            .expect_err("task codec must be checked before activity invocation");
15094        assert!(error.to_string().contains("unsupported_payload_codec"));
15095
15096        let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
15097        update.workflow_update_id = Some("update-invalid-codec".to_string());
15098        update.update_name = Some("known".to_string());
15099        update.history_events.push(history_event(
15100            "UpdateAccepted",
15101            json!({
15102                "update_id": "update-invalid-codec",
15103                "update_name": "known",
15104                "arguments": {"codec": "json", "blob": null}
15105            }),
15106        ));
15107        let error = worker
15108            .execute_workflow_task(update)
15109            .expect_err("nested update codec must be checked before handler lookup");
15110        assert!(error.to_string().contains("unsupported_payload_codec"));
15111
15112        let query: QueryTask = serde_json::from_value(json!({
15113            "query_task_id": "query-invalid-codec",
15114            "workflow_type": "codec.workflow",
15115            "query_name": "known",
15116            "payload_codec": DEFAULT_CODEC,
15117            "workflow_arguments": null,
15118            "query_arguments": null,
15119            "history_export": {
15120                "payloads": {"codec": DEFAULT_CODEC},
15121                "signals": [{
15122                    "name": "empty",
15123                    "payload_codec": "json",
15124                    "arguments": null
15125                }]
15126            }
15127        }))
15128        .expect("query task");
15129        let failure = worker
15130            .execute_query_task(query)
15131            .await
15132            .expect_err("exported signal codec must be checked before query invocation");
15133        assert_eq!(failure.reason, "query_payload_decode_failed");
15134        assert!(failure.message.contains("unsupported_payload_codec"));
15135
15136        let exported_history: QueryTask = serde_json::from_value(json!({
15137            "query_task_id": "query-invalid-history-codec",
15138            "workflow_type": "codec.workflow",
15139            "query_name": "known",
15140            "payload_codec": DEFAULT_CODEC,
15141            "history_export": {
15142                "payloads": {"codec": DEFAULT_CODEC},
15143                "history_events": [{
15144                    "type": "ActivityCompleted",
15145                    "payload": {"payload_codec": "unknown", "result": null}
15146                }]
15147            }
15148        }))
15149        .expect("query task");
15150        let failure = worker
15151            .execute_query_task(exported_history)
15152            .await
15153            .expect_err("exported history codec must be checked before query invocation");
15154        assert_eq!(failure.reason, "query_payload_decode_failed");
15155        assert!(failure.message.contains("unsupported_payload_codec"));
15156        assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
15157
15158        let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
15159        unknown_workflow.arguments = None;
15160        unknown_workflow.history_events.push(history_event(
15161            "SignalReceived",
15162            json!({
15163                "signal_name": "empty",
15164                "payload_codec": "json",
15165                "arguments": null
15166            }),
15167        ));
15168        let error = worker
15169            .execute_workflow_task(unknown_workflow)
15170            .expect_err("history codec must precede unknown workflow outcome");
15171        assert!(error.to_string().contains("unsupported_payload_codec"));
15172
15173        let unknown_activity = ActivityTask {
15174            task_id: "activity-unknown".to_string(),
15175            activity_attempt_id: None,
15176            attempt_id: None,
15177            activity_type: "missing".to_string(),
15178            payload_codec: "json".to_string(),
15179            arguments: None,
15180            attempt_number: 1,
15181            lease_owner: None,
15182        };
15183        let error = worker
15184            .execute_activity_task(unknown_activity)
15185            .await
15186            .expect_err("codec must precede unknown activity outcome");
15187        assert!(error.to_string().contains("unsupported_payload_codec"));
15188
15189        let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
15190        unknown_update.payload_codec = "json".to_string();
15191        unknown_update.arguments = None;
15192        unknown_update.workflow_update_id = Some("update-unknown".to_string());
15193        unknown_update.update_name = Some("missing".to_string());
15194        let error = worker
15195            .execute_workflow_task(unknown_update)
15196            .expect_err("codec must precede fail_update shortcut");
15197        assert!(error.to_string().contains("unsupported_payload_codec"));
15198
15199        let unknown_query: QueryTask = serde_json::from_value(json!({
15200            "query_task_id": "query-unknown",
15201            "workflow_type": "missing",
15202            "query_name": "missing",
15203            "payload_codec": "json",
15204            "workflow_arguments": null,
15205            "query_arguments": null
15206        }))
15207        .expect("query task");
15208        let failure = worker
15209            .execute_query_task(unknown_query)
15210            .await
15211            .expect_err("codec must precede unknown query outcome");
15212        assert_eq!(failure.reason, "query_payload_decode_failed");
15213        assert!(failure.message.contains("unsupported_payload_codec"));
15214    }
15215
15216    #[tokio::test]
15217    async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
15218        let client = Client::new("http://127.0.0.1:8080").expect("client");
15219        let worker = Worker::new(client, "rust-workers");
15220
15221        for event_type in ["SignalReceived", "SignalApplied"] {
15222            for (payload_field, codec) in [
15223                ("value", "json"),
15224                ("input", "unknown"),
15225                ("arguments", "json"),
15226            ] {
15227                let payload = json!({
15228                    "signal_name": "empty",
15229                    payload_field: {"codec": codec, "blob": null}
15230                });
15231                let workflow = workflow_task(
15232                    "missing",
15233                    vec![history_event(event_type, payload.clone())],
15234                    DEFAULT_CODEC,
15235                );
15236                let error = worker
15237                    .execute_workflow_task(workflow)
15238                    .expect_err("signal payload codec must precede unknown workflow outcome");
15239                assert!(
15240                    error.to_string().contains("unsupported_payload_codec"),
15241                    "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
15242                );
15243
15244                let query: QueryTask = serde_json::from_value(json!({
15245                    "query_task_id": format!("query-{event_type}-{payload_field}"),
15246                    "workflow_type": "missing",
15247                    "query_name": "missing",
15248                    "payload_codec": DEFAULT_CODEC,
15249                    "workflow_arguments": null,
15250                    "query_arguments": null,
15251                    "history_events": [{
15252                        "event_type": event_type,
15253                        "payload": payload
15254                    }]
15255                }))
15256                .expect("query task");
15257                let failure = worker
15258                    .execute_query_task(query)
15259                    .await
15260                    .expect_err("signal payload codec must precede unknown query outcome");
15261                assert_eq!(
15262                    failure.reason, "query_payload_decode_failed",
15263                    "{event_type}.{payload_field} returned an unrelated query outcome"
15264                );
15265                assert!(
15266                    failure.message.contains("unsupported_payload_codec"),
15267                    "{event_type}.{payload_field} returned an unrelated query error: {}",
15268                    failure.message
15269                );
15270            }
15271        }
15272    }
15273
15274    #[test]
15275    fn workflow_context_schedules_activity_until_completion_is_in_history() {
15276        let ctx = WorkflowContext {
15277            state: Arc::new(Mutex::new(
15278                WorkflowState::new_with_identity(
15279                    Vec::new(),
15280                    Some("wf-parent".to_string()),
15281                    Some("run-parent".to_string()),
15282                    "rust-workers".to_string(),
15283                    DEFAULT_CODEC.to_string(),
15284                    None,
15285                )
15286                .expect("workflow state"),
15287            )),
15288        };
15289
15290        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
15291        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15292        assert!(matches!(
15293            call.as_mut().poll(&mut task_context),
15294            Poll::Pending
15295        ));
15296
15297        let commands = ctx.take_commands().expect("commands");
15298        assert_eq!(commands[0]["type"], "schedule_activity");
15299        assert_eq!(commands[0]["activity_type"], "hello.activity");
15300    }
15301
15302    #[test]
15303    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
15304        let ctx = workflow_context(Vec::new());
15305        let options = ActivityOptions::new()
15306            .task_queue("payments")
15307            .retry_policy(
15308                ActivityRetryPolicy::new(4)
15309                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
15310                    .non_retryable_error_type("ValidationError"),
15311            )
15312            .start_to_close_timeout(Duration::from_secs(120))
15313            .schedule_to_start_timeout(Duration::from_secs(10))
15314            .schedule_to_close_timeout(Duration::from_secs(300))
15315            .heartbeat_timeout(Duration::from_secs(15));
15316        let mut call = Box::pin(ctx.activity_with_options(
15317            "charge-card",
15318            options,
15319            json!([{"order_id": "o-1"}]),
15320        ));
15321        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15322
15323        assert!(matches!(
15324            call.as_mut().poll(&mut task_context),
15325            Poll::Pending
15326        ));
15327        assert!(matches!(
15328            call.as_mut().poll(&mut task_context),
15329            Poll::Pending
15330        ));
15331
15332        let commands = ctx.take_commands().expect("activity command");
15333        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
15334        assert_eq!(commands[0]["queue"], "payments");
15335        assert_eq!(
15336            commands[0]["retry_policy"],
15337            json!({
15338                "max_attempts": 4,
15339                "backoff_seconds": [1, 3, 9],
15340                "non_retryable_error_types": ["ValidationError"],
15341            })
15342        );
15343        assert_eq!(commands[0]["start_to_close_timeout"], 120);
15344        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
15345        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
15346        assert_eq!(commands[0]["heartbeat_timeout"], 15);
15347    }
15348
15349    #[test]
15350    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
15351        let ctx = workflow_context(Vec::new());
15352        let options = ActivityOptions::new().retry_policy(
15353            ActivityRetryPolicy::new(3)
15354                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
15355        );
15356        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
15357        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15358
15359        assert!(matches!(
15360            call.as_mut().poll(&mut task_context),
15361            Poll::Pending
15362        ));
15363        assert_eq!(
15364            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
15365            json!([1, 2])
15366        );
15367    }
15368
15369    #[test]
15370    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
15371        let cases = [
15372            (
15373                ActivityOptions::new().task_queue("  "),
15374                ActivityOptionsErrorKind::EmptyTaskQueue,
15375            ),
15376            (
15377                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
15378                ActivityOptionsErrorKind::EmptyRetryPolicy,
15379            ),
15380            (
15381                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
15382                ActivityOptionsErrorKind::InvalidMaxAttempts,
15383            ),
15384            (
15385                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
15386                    max_attempts: None,
15387                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
15388                    non_retryable_error_types: Vec::new(),
15389                }),
15390                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
15391            ),
15392            (
15393                ActivityOptions::new().retry_policy(
15394                    ActivityRetryPolicy::new(2)
15395                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
15396                ),
15397                ActivityOptionsErrorKind::TooManyBackoffIntervals,
15398            ),
15399            (
15400                ActivityOptions::new().retry_policy(
15401                    ActivityRetryPolicy::new(2).exponential_backoff(
15402                        Duration::from_secs(1),
15403                        0,
15404                        None,
15405                    ),
15406                ),
15407                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
15408            ),
15409            (
15410                ActivityOptions::new()
15411                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
15412                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
15413            ),
15414            (
15415                ActivityOptions::new().retry_policy(
15416                    ActivityRetryPolicy::new(10_002).exponential_backoff(
15417                        Duration::from_secs(1),
15418                        1,
15419                        None,
15420                    ),
15421                ),
15422                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
15423            ),
15424            (
15425                ActivityOptions::new().retry_policy(
15426                    ActivityRetryPolicy::new(2)
15427                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
15428                ),
15429                ActivityOptionsErrorKind::BackoffOverflow,
15430            ),
15431        ];
15432
15433        for (options, expected_kind) in cases {
15434            let ctx = workflow_context(Vec::new());
15435            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
15436            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15437            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
15438                call.as_mut().poll(&mut task_context)
15439            else {
15440                panic!("expected typed activity validation error");
15441            };
15442            assert_eq!(error.kind, expected_kind);
15443            assert!(ctx.take_commands().expect("commands").is_empty());
15444        }
15445    }
15446
15447    #[test]
15448    fn activity_options_validate_positive_and_ordered_timeouts() {
15449        let zero_timeout_cases = [
15450            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
15451            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
15452            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
15453            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
15454        ];
15455        for options in zero_timeout_cases {
15456            assert_eq!(
15457                options.validate().expect_err("zero timeout").kind,
15458                ActivityOptionsErrorKind::TimeoutNotPositive
15459            );
15460        }
15461
15462        let ordering_cases = [
15463            ActivityOptions::new()
15464                .heartbeat_timeout(Duration::from_secs(11))
15465                .start_to_close_timeout(Duration::from_secs(10)),
15466            ActivityOptions::new()
15467                .start_to_close_timeout(Duration::from_secs(31))
15468                .schedule_to_close_timeout(Duration::from_secs(30)),
15469            ActivityOptions::new()
15470                .schedule_to_start_timeout(Duration::from_secs(31))
15471                .schedule_to_close_timeout(Duration::from_secs(30)),
15472        ];
15473        for options in ordering_cases {
15474            assert_eq!(
15475                options.validate().expect_err("timeout order").kind,
15476                ActivityOptionsErrorKind::TimeoutOrder
15477            );
15478        }
15479
15480        assert_eq!(
15481            ActivityOptions::new()
15482                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
15483                .validate()
15484                .expect_err("protocol integer overflow")
15485                .kind,
15486            ActivityOptionsErrorKind::TimeoutOverflow
15487        );
15488    }
15489
15490    #[test]
15491    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
15492        let ctx = workflow_context(completed_retry_activity_history());
15493        let mut call =
15494            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
15495        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15496
15497        assert!(matches!(
15498            call.as_mut().poll(&mut task_context),
15499            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
15500        ));
15501        assert!(ctx.take_commands().expect("commands").is_empty());
15502        ctx.ensure_history_consumed().expect("history consumed");
15503    }
15504
15505    #[test]
15506    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
15507        let mut options = retry_activity_options();
15508        options
15509            .retry_policy
15510            .as_mut()
15511            .expect("retry policy")
15512            .non_retryable_error_types
15513            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
15514
15515        let new_ctx = workflow_context(Vec::new());
15516        let mut new_call =
15517            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
15518        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15519        assert!(matches!(
15520            new_call.as_mut().poll(&mut task_context),
15521            Poll::Pending
15522        ));
15523        let commands = new_ctx.take_commands().expect("commands");
15524        assert_eq!(commands.len(), 1);
15525        assert_eq!(
15526            commands[0]["retry_policy"]["non_retryable_error_types"],
15527            json!(["PermanentError"])
15528        );
15529
15530        let replay_ctx = workflow_context(completed_retry_activity_history());
15531        let mut replay_call =
15532            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
15533        assert!(matches!(
15534            replay_call.as_mut().poll(&mut task_context),
15535            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
15536        ));
15537        assert!(replay_ctx.take_commands().expect("commands").is_empty());
15538        replay_ctx
15539            .ensure_history_consumed()
15540            .expect("history consumed");
15541    }
15542
15543    #[test]
15544    fn replayed_intermediate_retry_remains_pending_across_restarts() {
15545        let history = completed_retry_activity_history()
15546            .into_iter()
15547            .take(3)
15548            .collect::<Vec<_>>();
15549
15550        for _restart in 0..2 {
15551            let ctx = workflow_context(history.clone());
15552            let mut call =
15553                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
15554            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15555            assert!(matches!(
15556                call.as_mut().poll(&mut task_context),
15557                Poll::Pending
15558            ));
15559            assert!(ctx.take_commands().expect("commands").is_empty());
15560        }
15561    }
15562
15563    #[test]
15564    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
15565        let mut changed_queue = retry_activity_options();
15566        changed_queue.task_queue = Some("different-queue".to_string());
15567
15568        let mut changed_max_attempts = retry_activity_options();
15569        let retry_policy = changed_max_attempts
15570            .retry_policy
15571            .as_mut()
15572            .expect("retry policy");
15573        retry_policy.max_attempts = Some(4);
15574
15575        let mut changed_backoff = retry_activity_options();
15576        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
15577        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
15578            Duration::from_secs(3),
15579            Duration::from_secs(4),
15580        ]));
15581
15582        let mut changed_non_retryable_types = retry_activity_options();
15583        let retry_policy = changed_non_retryable_types
15584            .retry_policy
15585            .as_mut()
15586            .expect("retry policy");
15587        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
15588
15589        let mut changed_start_to_close = retry_activity_options();
15590        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
15591        let mut changed_schedule_to_start = retry_activity_options();
15592        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
15593        let mut changed_schedule_to_close = retry_activity_options();
15594        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
15595        let mut changed_heartbeat = retry_activity_options();
15596        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
15597
15598        let cases = [
15599            (changed_queue, "activity_task_queue_mismatch"),
15600            (changed_max_attempts, "activity_retry_policy_mismatch"),
15601            (changed_backoff, "activity_retry_policy_mismatch"),
15602            (
15603                changed_non_retryable_types,
15604                "activity_retry_policy_mismatch",
15605            ),
15606            (changed_start_to_close, "activity_retry_policy_mismatch"),
15607            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
15608            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
15609            (changed_heartbeat, "activity_retry_policy_mismatch"),
15610        ];
15611
15612        for (options, expected_reason) in cases {
15613            let ctx = workflow_context(completed_retry_activity_history());
15614            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
15615            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15616            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15617                call.as_mut().poll(&mut task_context)
15618            else {
15619                panic!("changed activity options must fail replay");
15620            };
15621            assert_eq!(failure.reason, expected_reason);
15622            assert_eq!(failure.sequence, Some(1));
15623            assert!(ctx.take_commands().expect("commands").is_empty());
15624        }
15625    }
15626
15627    #[test]
15628    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
15629        let cases = [
15630            (
15631                "execution_mode",
15632                json!("local"),
15633                "activity_execution_mode_mismatch",
15634            ),
15635            (
15636                "snapshot_version",
15637                json!(2),
15638                "activity_retry_policy_mismatch",
15639            ),
15640        ];
15641
15642        for (field, value, expected_reason) in cases {
15643            let mut history = completed_retry_activity_history();
15644            let activity = history[0].payload["activity"]
15645                .as_object_mut()
15646                .expect("activity snapshot");
15647            if field == "execution_mode" {
15648                activity.insert(field.to_string(), value);
15649            } else {
15650                activity["retry_policy"]
15651                    .as_object_mut()
15652                    .expect("retry snapshot")
15653                    .insert(field.to_string(), value);
15654            }
15655
15656            let ctx = workflow_context(history);
15657            let mut call =
15658                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
15659            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15660            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15661                call.as_mut().poll(&mut task_context)
15662            else {
15663                panic!("changed {field} must fail replay");
15664            };
15665            assert_eq!(failure.reason, expected_reason);
15666            assert_eq!(failure.sequence, Some(1));
15667            assert!(ctx.take_commands().expect("commands").is_empty());
15668        }
15669    }
15670
15671    #[test]
15672    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
15673        let mut history = completed_retry_activity_history();
15674        let activity = history[0].payload["activity"]
15675            .as_object_mut()
15676            .expect("activity snapshot");
15677        activity.remove("execution_mode");
15678        activity.remove("retry_policy");
15679
15680        let mut current = retry_activity_options();
15681        current.start_to_close_timeout = Some(Duration::from_secs(45));
15682        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
15683        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
15684        current.heartbeat_timeout = Some(Duration::from_secs(12));
15685
15686        let ctx = workflow_context(history);
15687        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
15688        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15689        assert!(matches!(
15690            call.as_mut().poll(&mut task_context),
15691            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
15692        ));
15693        assert!(ctx.take_commands().expect("commands").is_empty());
15694        ctx.ensure_history_consumed().expect("history consumed");
15695    }
15696
15697    #[test]
15698    fn terminal_activity_failed_after_start_returns_typed_failure() {
15699        let history = vec![
15700            history_event(
15701                "ActivityScheduled",
15702                json!({
15703                    "sequence": 1,
15704                    "activity_type": "flaky",
15705                    "activity_execution_id": "act-terminal",
15706                    "activity": {
15707                        "id": "act-terminal",
15708                        "sequence": 1,
15709                        "type": "flaky",
15710                        "queue": "critical-activities",
15711                        "retry_policy": {
15712                            "snapshot_version": 1,
15713                            "max_attempts": 3,
15714                            "backoff_seconds": [2, 4],
15715                            "non_retryable_error_types": ["PermanentError"]
15716                        }
15717                    }
15718                }),
15719            ),
15720            history_event(
15721                "ActivityStarted",
15722                json!({
15723                    "sequence": 1,
15724                    "activity_type": "flaky",
15725                    "activity_execution_id": "act-terminal",
15726                    "activity_attempt_id": "attempt-1",
15727                    "attempt_number": 1
15728                }),
15729            ),
15730            history_event(
15731                "ActivityFailed",
15732                json!({
15733                    "sequence": 1,
15734                    "activity_type": "flaky",
15735                    "activity_execution_id": "act-terminal",
15736                    "activity_attempt_id": "attempt-1",
15737                    "attempt_number": 1,
15738                    "failure_id": "failure-terminal",
15739                    "failure_category": "activity",
15740                    "exception_type": "PermanentError",
15741                    "message": "cannot retry",
15742                    "non_retryable": true
15743                }),
15744            ),
15745        ];
15746        let ctx = workflow_context(history);
15747        let mut call =
15748            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
15749        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15750
15751        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
15752            call.as_mut().poll(&mut task_context)
15753        else {
15754            panic!("terminal ActivityFailed must settle the activity future");
15755        };
15756        assert_eq!(failure.kind, ActivityFailureKind::Failed);
15757        assert_eq!(
15758            failure.activity_execution_id.as_deref(),
15759            Some("act-terminal")
15760        );
15761        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
15762        assert!(failure.non_retryable);
15763        assert!(ctx.take_commands().expect("commands").is_empty());
15764        ctx.ensure_history_consumed().expect("history consumed");
15765    }
15766
15767    #[test]
15768    fn activity_terminal_events_return_machine_readable_failures() {
15769        let cases = [
15770            (
15771                "ActivityFailed",
15772                json!({
15773                    "sequence": 1,
15774                    "activity_type": "charge-card",
15775                    "activity_execution_id": "act-1",
15776                    "activity_attempt_id": "attempt-2",
15777                    "attempt_number": 2,
15778                    "failure_id": "failure-1",
15779                    "failure_category": "activity",
15780                    "exception_type": "PaymentDeclined",
15781                    "exception_class": "payments.PaymentDeclined",
15782                    "message": "card declined",
15783                    "non_retryable": true
15784                }),
15785                ActivityFailureKind::Failed,
15786                "activity",
15787            ),
15788            (
15789                "ActivityCancelled",
15790                json!({
15791                    "sequence": 1,
15792                    "activity_type": "charge-card",
15793                    "activity_execution_id": "act-1",
15794                    "activity_attempt_id": "attempt-1"
15795                }),
15796                ActivityFailureKind::Cancelled,
15797                "cancelled",
15798            ),
15799        ];
15800
15801        for (event_type, payload, expected_kind, expected_reason) in cases {
15802            let ctx = workflow_context(vec![history_event(event_type, payload)]);
15803            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
15804            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15805            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
15806                call.as_mut().poll(&mut task_context)
15807            else {
15808                panic!("expected terminal activity failure");
15809            };
15810            assert_eq!(failure.kind, expected_kind);
15811            assert_eq!(failure.reason, expected_reason);
15812            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
15813            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
15814        }
15815    }
15816
15817    #[test]
15818    fn every_activity_timeout_class_is_typed() {
15819        for timeout_kind in [
15820            "start_to_close",
15821            "schedule_to_start",
15822            "schedule_to_close",
15823            "heartbeat",
15824        ] {
15825            let ctx = workflow_context(vec![history_event(
15826                "ActivityTimedOut",
15827                json!({
15828                    "sequence": 1,
15829                    "activity_type": "slow",
15830                    "activity_execution_id": "act-timeout",
15831                    "activity_attempt_id": "attempt-timeout",
15832                    "failure_category": "timeout",
15833                    "timeout_kind": timeout_kind,
15834                    "message": "deadline expired"
15835                }),
15836            )]);
15837            let mut call = Box::pin(ctx.activity("slow", json!([])));
15838            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15839            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
15840                call.as_mut().poll(&mut task_context)
15841            else {
15842                panic!("expected timeout failure");
15843            };
15844            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
15845            assert_eq!(failure.reason, timeout_kind);
15846            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
15847            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
15848        }
15849    }
15850
15851    #[test]
15852    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
15853        let ctx = workflow_context(Vec::new());
15854        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
15855        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15856
15857        assert!(matches!(
15858            sleep.as_mut().poll(&mut task_context),
15859            Poll::Pending
15860        ));
15861        assert!(matches!(
15862            sleep.as_mut().poll(&mut task_context),
15863            Poll::Pending
15864        ));
15865
15866        let commands = ctx.take_commands().expect("timer command");
15867        assert_eq!(
15868            commands,
15869            vec![json!({
15870                "type": "start_timer",
15871                "delay_seconds": 2,
15872            })]
15873        );
15874    }
15875
15876    #[test]
15877    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
15878        let history = vec![
15879            history_event(
15880                "TimerScheduled",
15881                json!({
15882                    "sequence": 1,
15883                    "timer_id": "timer-1",
15884                    "delay_seconds": 5,
15885                    "fire_at": "2026-07-11T12:00:05Z",
15886                }),
15887            ),
15888            history_event(
15889                "TimerFired",
15890                json!({
15891                    "sequence": 1,
15892                    "timer_id": "timer-1",
15893                    "delay_seconds": 5,
15894                    "fire_at": "2026-07-11T12:00:05Z",
15895                    "fired_at": "2026-07-11T12:00:05Z",
15896                }),
15897            ),
15898        ];
15899
15900        for _restart in 0..2 {
15901            let ctx = workflow_context(history.clone());
15902            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
15903            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15904            assert!(matches!(
15905                sleep.as_mut().poll(&mut task_context),
15906                Poll::Ready(Ok(()))
15907            ));
15908            assert!(ctx.take_commands().expect("commands").is_empty());
15909            ctx.ensure_history_consumed().expect("history consumed");
15910        }
15911    }
15912
15913    #[test]
15914    fn workflow_sleep_rejects_changed_delay_during_replay() {
15915        let ctx = workflow_context(vec![
15916            history_event(
15917                "TimerScheduled",
15918                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15919            ),
15920            history_event(
15921                "TimerFired",
15922                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15923            ),
15924        ]);
15925        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
15926        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15927
15928        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15929            sleep.as_mut().poll(&mut task_context)
15930        else {
15931            panic!("changed timer delay must be rejected");
15932        };
15933        assert_eq!(failure.reason, "timer_delay_mismatch");
15934        assert_eq!(failure.sequence, Some(1));
15935    }
15936
15937    #[test]
15938    fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
15939        let ctx = workflow_context(Vec::new());
15940        let mut wait = Box::pin(
15941            ctx.wait_condition(
15942                ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
15943                    .timeout(Duration::from_millis(60_001)),
15944                || Ok(false),
15945            ),
15946        );
15947        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15948
15949        assert!(matches!(
15950            wait.as_mut().poll(&mut task_context),
15951            Poll::Pending
15952        ));
15953        assert!(matches!(
15954            wait.as_mut().poll(&mut task_context),
15955            Poll::Pending
15956        ));
15957        assert_eq!(
15958            ctx.take_commands().expect("condition command"),
15959            vec![json!({
15960                "type": "open_condition_wait",
15961                "condition_wait_occurrence_id": "rust:condition-wait:0",
15962                "condition_key": "approval.ready",
15963                "condition_definition_fingerprint": "sha256:approval-v1",
15964                "timeout_seconds": 61,
15965            })]
15966        );
15967    }
15968
15969    #[test]
15970    fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
15971        let ctx = workflow_context(Vec::new());
15972        let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
15973        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15974        assert!(matches!(
15975            satisfied.as_mut().poll(&mut task_context),
15976            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
15977        ));
15978
15979        let mut timed_out = Box::pin(wait_condition!(
15980            ctx,
15981            "no-wait",
15982            timeout: Duration::ZERO,
15983            || Ok(false),
15984        ));
15985        assert!(matches!(
15986            timed_out.as_mut().poll(&mut task_context),
15987            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
15988        ));
15989        assert!(ctx.take_commands().expect("commands").is_empty());
15990    }
15991
15992    #[test]
15993    fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
15994        let signal_history = vec![
15995            history_event(
15996                "ConditionWaitOpened",
15997                json!({
15998                    "sequence": 4,
15999                    "condition_wait_id": "condition:4",
16000                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16001                    "condition_key": "approval",
16002                    "condition_definition_fingerprint": "sha256:approval-v1",
16003                    "timeout_seconds": 30,
16004                }),
16005            ),
16006            history_event(
16007                "SignalReceived",
16008                json!({
16009                    "workflow_sequence": 4,
16010                    "signal_name": "approve",
16011                    "arguments": fixture_envelope(json!(["Ada"])),
16012                }),
16013            ),
16014        ];
16015        for _worker_before_or_after_restart in 0..2 {
16016            let ctx = workflow_context(signal_history.clone());
16017            let predicate_ctx = ctx.clone();
16018            let mut wait = Box::pin(
16019                ctx.wait_condition(
16020                    ConditionWaitOptions::new("approval", "sha256:approval-v1")
16021                        .timeout(Duration::from_secs(30)),
16022                    move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
16023                ),
16024            );
16025            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16026            assert!(matches!(
16027                wait.as_mut().poll(&mut task_context),
16028                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16029            ));
16030            assert!(ctx.take_commands().expect("commands").is_empty());
16031            ctx.ensure_history_consumed().expect("condition consumed");
16032        }
16033
16034        let update_history = vec![
16035            history_event(
16036                "ConditionWaitOpened",
16037                json!({
16038                    "sequence": 7,
16039                    "condition_wait_id": "condition:7",
16040                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16041                    "condition_key": "update-approval",
16042                    "condition_definition_fingerprint": "sha256:update-approval-v1",
16043                }),
16044            ),
16045            history_event(
16046                "UpdateApplied",
16047                json!({
16048                    "sequence": 7,
16049                    "update_id": "update-1",
16050                    "update_name": "approve",
16051                    "arguments": fixture_envelope(json!([true])),
16052                }),
16053            ),
16054        ];
16055        let ctx = workflow_context(update_history);
16056        let predicate_ctx = ctx.clone();
16057        let mut wait = Box::pin(ctx.wait_condition(
16058            ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
16059            move || {
16060                Ok(predicate_ctx
16061                    .updates("approve")?
16062                    .first()
16063                    .and_then(|arguments| arguments.first())
16064                    .and_then(Value::as_bool)
16065                    == Some(true))
16066            },
16067        ));
16068        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16069        assert!(matches!(
16070            wait.as_mut().poll(&mut task_context),
16071            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16072        ));
16073        assert!(ctx.take_commands().expect("commands").is_empty());
16074        ctx.ensure_history_consumed().expect("condition consumed");
16075    }
16076
16077    #[test]
16078    fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
16079        let open_history = vec![
16080            history_event(
16081                "ConditionWaitOpened",
16082                json!({
16083                    "sequence": 3,
16084                    "condition_wait_id": "condition:3",
16085                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16086                    "condition_key": "two-votes",
16087                    "condition_definition_fingerprint": "sha256:two-votes-v1",
16088                    "timeout_seconds": 120,
16089                }),
16090            ),
16091            history_event(
16092                "SignalReceived",
16093                json!({
16094                    "workflow_sequence": 3,
16095                    "signal_name": "vote",
16096                    "arguments": fixture_envelope(json!(["first"])),
16097                }),
16098            ),
16099        ];
16100        for _worker_before_or_after_restart in 0..2 {
16101            let ctx = workflow_context(open_history.clone());
16102            let predicate_ctx = ctx.clone();
16103            let mut wait = Box::pin(
16104                ctx.wait_condition(
16105                    ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
16106                        .timeout(Duration::from_secs(120)),
16107                    move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
16108                ),
16109            );
16110            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16111            assert!(matches!(
16112                wait.as_mut().poll(&mut task_context),
16113                Poll::Pending
16114            ));
16115            assert_eq!(
16116                ctx.take_commands().expect("reopened condition"),
16117                vec![json!({
16118                    "type": "open_condition_wait",
16119                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16120                    "condition_key": "two-votes",
16121                    "condition_definition_fingerprint": "sha256:two-votes-v1",
16122                    "timeout_seconds": 120,
16123                })]
16124            );
16125        }
16126
16127        let satisfied_ctx = workflow_context(vec![
16128            history_event(
16129                "ConditionWaitOpened",
16130                json!({
16131                    "sequence": 5,
16132                    "condition_wait_id": "condition:5",
16133                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16134                    "condition_key": "approval",
16135                    "condition_definition_fingerprint": "sha256:approval-v1",
16136                }),
16137            ),
16138            history_event(
16139                "ConditionWaitSatisfied",
16140                json!({
16141                    "sequence": 5,
16142                    "condition_wait_id": "condition:5",
16143                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16144                    "condition_key": "approval",
16145                    "condition_definition_fingerprint": "sha256:approval-v1",
16146                }),
16147            ),
16148        ]);
16149        let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
16150            ConditionWaitOptions::new("approval", "sha256:approval-v1"),
16151            || Ok(false),
16152        ));
16153        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16154        assert!(matches!(
16155            satisfied.as_mut().poll(&mut task_context),
16156            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16157        ));
16158
16159        let timed_out_ctx = workflow_context(vec![
16160            history_event(
16161                "ConditionWaitOpened",
16162                json!({
16163                    "sequence": 8,
16164                    "condition_wait_id": "condition:8",
16165                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16166                    "condition_key": "approval-timeout",
16167                    "condition_definition_fingerprint": "sha256:approval-timeout-v1",
16168                    "timeout_seconds": 5,
16169                }),
16170            ),
16171            history_event(
16172                "TimerScheduled",
16173                json!({
16174                    "sequence": 9,
16175                    "timer_id": "condition-timer:9",
16176                    "timer_kind": "condition_timeout",
16177                    "condition_wait_id": "condition:8",
16178                    "delay_seconds": 5,
16179                }),
16180            ),
16181            history_event(
16182                "TimerFired",
16183                json!({
16184                    "sequence": 9,
16185                    "timer_id": "condition-timer:9",
16186                    "timer_kind": "condition_timeout",
16187                    "condition_wait_id": "condition:8",
16188                    "delay_seconds": 5,
16189                }),
16190            ),
16191        ]);
16192        let mut timed_out = Box::pin(
16193            timed_out_ctx.wait_condition(
16194                ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
16195                    .timeout(Duration::from_secs(5)),
16196                || Ok(true),
16197            ),
16198        );
16199        assert!(matches!(
16200            timed_out.as_mut().poll(&mut task_context),
16201            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
16202        ));
16203    }
16204
16205    #[test]
16206    fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
16207        let history = vec![
16208            history_event(
16209                "ConditionWaitOpened",
16210                json!({
16211                    "sequence": 3,
16212                    "condition_wait_id": "condition:3",
16213                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16214                    "condition_key": "two-votes",
16215                    "condition_definition_fingerprint": "sha256:two-votes-v1",
16216                }),
16217            ),
16218            history_event(
16219                "SignalReceived",
16220                json!({
16221                    "workflow_sequence": 3,
16222                    "signal_name": "vote",
16223                    "arguments": fixture_envelope(json!(["first"])),
16224                }),
16225            ),
16226            history_event(
16227                "ConditionWaitSatisfied",
16228                json!({
16229                    "sequence": 3,
16230                    "condition_wait_id": "condition:3",
16231                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16232                    "condition_key": "two-votes",
16233                    "condition_definition_fingerprint": "sha256:two-votes-v1",
16234                }),
16235            ),
16236            history_event(
16237                "ConditionWaitOpened",
16238                json!({
16239                    "sequence": 5,
16240                    "condition_wait_id": "condition:5",
16241                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16242                    "condition_key": "two-votes",
16243                    "condition_definition_fingerprint": "sha256:two-votes-v1",
16244                }),
16245            ),
16246            history_event(
16247                "SignalReceived",
16248                json!({
16249                    "workflow_sequence": 5,
16250                    "signal_name": "vote",
16251                    "arguments": fixture_envelope(json!(["second"])),
16252                }),
16253            ),
16254            history_event(
16255                "ConditionWaitSatisfied",
16256                json!({
16257                    "sequence": 5,
16258                    "condition_wait_id": "condition:5",
16259                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16260                    "condition_key": "two-votes",
16261                    "condition_definition_fingerprint": "sha256:two-votes-v1",
16262                }),
16263            ),
16264        ];
16265        for _cold_worker_or_restart in 0..2 {
16266            let ctx = workflow_context(history.clone());
16267            let predicate_ctx = ctx.clone();
16268            let mut wait = Box::pin(ctx.wait_condition(
16269                ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
16270                move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
16271            ));
16272            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16273
16274            assert!(matches!(
16275                wait.as_mut().poll(&mut task_context),
16276                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16277            ));
16278            assert!(ctx.take_commands().expect("commands").is_empty());
16279            ctx.ensure_history_consumed()
16280                .expect("every physical wait-open is consumed");
16281        }
16282    }
16283
16284    #[test]
16285    fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
16286        let history = vec![
16287            history_event(
16288                "ConditionWaitOpened",
16289                json!({
16290                    "sequence": 3,
16291                    "condition_wait_id": "condition:3",
16292                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16293                    "condition_key": "approved",
16294                    "condition_definition_fingerprint": "sha256:approved-v1",
16295                }),
16296            ),
16297            history_event(
16298                "UpdateApplied",
16299                json!({
16300                    "sequence": 3,
16301                    "update_id": "update-1",
16302                    "update_name": "approve",
16303                    "arguments": fixture_envelope(json!([false])),
16304                }),
16305            ),
16306            history_event(
16307                "ConditionWaitOpened",
16308                json!({
16309                    "sequence": 5,
16310                    "condition_wait_id": "condition:5",
16311                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16312                    "condition_key": "approved",
16313                    "condition_definition_fingerprint": "sha256:approved-v1",
16314                }),
16315            ),
16316            history_event(
16317                "UpdateApplied",
16318                json!({
16319                    "sequence": 5,
16320                    "update_id": "update-2",
16321                    "update_name": "approve",
16322                    "arguments": fixture_envelope(json!([true])),
16323                }),
16324            ),
16325        ];
16326
16327        for _cold_worker_or_restart in 0..2 {
16328            let ctx = workflow_context(history.clone());
16329            let predicate_ctx = ctx.clone();
16330            let mut wait = Box::pin(ctx.wait_condition(
16331                ConditionWaitOptions::new("approved", "sha256:approved-v1"),
16332                move || {
16333                    Ok(predicate_ctx
16334                        .updates("approve")?
16335                        .last()
16336                        .and_then(|arguments| arguments.first())
16337                        .and_then(Value::as_bool)
16338                        == Some(true))
16339                },
16340            ));
16341            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16342
16343            assert!(matches!(
16344                wait.as_mut().poll(&mut task_context),
16345                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16346            ));
16347            assert!(ctx.take_commands().expect("commands").is_empty());
16348            ctx.ensure_history_consumed()
16349                .expect("every update-driven reopen is consumed");
16350        }
16351    }
16352
16353    #[test]
16354    fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
16355        for (first_key, first_fingerprint, second_key, second_fingerprint) in [
16356            ("shared", "sha256:first", "shared", "sha256:second"),
16357            ("first", "sha256:shared", "second", "sha256:shared"),
16358            ("shared", "sha256:shared", "shared", "sha256:shared"),
16359            ("first", "sha256:first", "second", "sha256:second"),
16360        ] {
16361            let history = vec![
16362                history_event(
16363                    "ConditionWaitOpened",
16364                    json!({
16365                        "sequence": 3,
16366                        "condition_wait_id": "condition:3",
16367                        "condition_wait_occurrence_id": "rust:condition-wait:0",
16368                        "condition_key": first_key,
16369                        "condition_definition_fingerprint": first_fingerprint,
16370                    }),
16371                ),
16372                history_event(
16373                    "ConditionWaitSatisfied",
16374                    json!({
16375                        "sequence": 3,
16376                        "condition_wait_id": "condition:3",
16377                        "condition_wait_occurrence_id": "rust:condition-wait:0",
16378                        "condition_key": first_key,
16379                        "condition_definition_fingerprint": first_fingerprint,
16380                    }),
16381                ),
16382                history_event(
16383                    "ConditionWaitOpened",
16384                    json!({
16385                        "sequence": 4,
16386                        "condition_wait_id": "condition:4",
16387                        "condition_wait_occurrence_id": "rust:condition-wait:1",
16388                        "condition_key": second_key,
16389                        "condition_definition_fingerprint": second_fingerprint,
16390                    }),
16391                ),
16392                history_event(
16393                    "ConditionWaitSatisfied",
16394                    json!({
16395                        "sequence": 4,
16396                        "condition_wait_id": "condition:4",
16397                        "condition_wait_occurrence_id": "rust:condition-wait:1",
16398                        "condition_key": second_key,
16399                        "condition_definition_fingerprint": second_fingerprint,
16400                    }),
16401                ),
16402            ];
16403            for _cold_worker_or_restart in 0..2 {
16404                let ctx = workflow_context(history.clone());
16405                let mut task_context = TaskContext::from_waker(noop_waker_ref());
16406                let mut first = Box::pin(ctx.wait_condition(
16407                    ConditionWaitOptions::new(first_key, first_fingerprint),
16408                    || Ok(false),
16409                ));
16410                assert!(matches!(
16411                    first.as_mut().poll(&mut task_context),
16412                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16413                ));
16414
16415                let mut second = Box::pin(ctx.wait_condition(
16416                    ConditionWaitOptions::new(second_key, second_fingerprint),
16417                    || Ok(false),
16418                ));
16419                assert!(matches!(
16420                    second.as_mut().poll(&mut task_context),
16421                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16422                ));
16423                assert!(ctx.take_commands().expect("commands").is_empty());
16424                ctx.ensure_history_consumed()
16425                    .expect("each authored wait consumes one occurrence");
16426            }
16427        }
16428    }
16429
16430    #[test]
16431    fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
16432        fn worker() -> Worker {
16433            let client = Client::new("http://127.0.0.1:8080").expect("client");
16434            let mut worker = Worker::new(client, "rust-workers");
16435            worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
16436                let mut outcomes = Vec::new();
16437                for _ in 0..2 {
16438                    outcomes.push(
16439                        ctx.wait_condition(
16440                            ConditionWaitOptions::new("shared", "sha256:shared"),
16441                            || Ok(false),
16442                        )
16443                        .await?,
16444                    );
16445                }
16446                Ok(json!(outcomes))
16447            });
16448            worker
16449        }
16450
16451        let task = workflow_task(
16452            "rust.condition-loop",
16453            vec![
16454                history_event(
16455                    "ConditionWaitOpened",
16456                    json!({
16457                        "sequence": 1,
16458                        "condition_wait_id": "condition:1",
16459                        "condition_wait_occurrence_id": "rust:condition-wait:0",
16460                        "condition_key": "shared",
16461                        "condition_definition_fingerprint": "sha256:shared",
16462                    }),
16463                ),
16464                history_event(
16465                    "ConditionWaitSatisfied",
16466                    json!({
16467                        "sequence": 1,
16468                        "condition_wait_id": "condition:1",
16469                        "condition_wait_occurrence_id": "rust:condition-wait:0",
16470                        "condition_key": "shared",
16471                        "condition_definition_fingerprint": "sha256:shared",
16472                    }),
16473                ),
16474                history_event(
16475                    "ConditionWaitOpened",
16476                    json!({
16477                        "sequence": 2,
16478                        "condition_wait_id": "condition:2",
16479                        "condition_wait_occurrence_id": "rust:condition-wait:1",
16480                        "condition_key": "shared",
16481                        "condition_definition_fingerprint": "sha256:shared",
16482                    }),
16483                ),
16484                history_event(
16485                    "ConditionWaitSatisfied",
16486                    json!({
16487                        "sequence": 2,
16488                        "condition_wait_id": "condition:2",
16489                        "condition_wait_occurrence_id": "rust:condition-wait:1",
16490                        "condition_key": "shared",
16491                        "condition_definition_fingerprint": "sha256:shared",
16492                    }),
16493                ),
16494            ],
16495            DEFAULT_CODEC,
16496        );
16497
16498        for _cold_worker_or_restart in 0..2 {
16499            let commands = worker()
16500                .execute_workflow_task(task.clone())
16501                .expect("adjacent loop waits replay deterministically");
16502            assert_eq!(commands.len(), 1);
16503            assert_eq!(commands[0]["type"], "complete_workflow");
16504            assert_eq!(
16505                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
16506                json!(["satisfied", "satisfied"])
16507            );
16508        }
16509    }
16510
16511    #[test]
16512    fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
16513        let history = vec![history_event(
16514            "ConditionWaitOpened",
16515            json!({
16516                "sequence": 12,
16517                "condition_wait_id": "condition:12",
16518                "condition_wait_occurrence_id": "rust:condition-wait:0",
16519                "condition_key": "approval",
16520                "condition_definition_fingerprint": "sha256:approval-v1",
16521                "timeout_seconds": 30,
16522            }),
16523        )];
16524        for (options, expected_reason) in [
16525            (
16526                ConditionWaitOptions::new("changed", "sha256:approval-v1")
16527                    .timeout(Duration::from_secs(30)),
16528                "condition_wait_key_mismatch",
16529            ),
16530            (
16531                ConditionWaitOptions::new("approval", "sha256:approval-v2")
16532                    .timeout(Duration::from_secs(30)),
16533                "condition_wait_predicate_mismatch",
16534            ),
16535            (
16536                ConditionWaitOptions::new("approval", "sha256:approval-v1")
16537                    .timeout(Duration::from_secs(29)),
16538                "condition_wait_timeout_mismatch",
16539            ),
16540        ] {
16541            let ctx = workflow_context(history.clone());
16542            let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
16543            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16544            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
16545                wait.as_mut().poll(&mut task_context)
16546            else {
16547                panic!("changed condition definition must fail replay");
16548            };
16549            assert_eq!(failure.reason, expected_reason);
16550            assert_eq!(failure.sequence, Some(12));
16551        }
16552    }
16553
16554    #[test]
16555    fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
16556        let error = WorkflowState::new(
16557            vec![history_event(
16558                "ConditionWaitOpened",
16559                json!({
16560                    "sequence": 12,
16561                    "condition_wait_id": "condition:12",
16562                    "condition_wait_occurrence_id": "rust:condition-wait:0",
16563                    "condition_key": "approval",
16564                }),
16565            )],
16566            "rust-workers".to_string(),
16567            DEFAULT_CODEC.to_string(),
16568            None,
16569        )
16570        .expect_err("condition history without a predicate fingerprint must fail");
16571
16572        assert!(matches!(
16573            error,
16574            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16575                if reason == "condition_wait_predicate_fingerprint_missing"
16576        ));
16577    }
16578
16579    #[test]
16580    fn condition_wait_history_requires_authored_occurrence_identity() {
16581        let error = WorkflowState::new(
16582            vec![history_event(
16583                "ConditionWaitOpened",
16584                json!({
16585                    "sequence": 12,
16586                    "condition_wait_id": "condition:12",
16587                    "condition_key": "approval",
16588                    "condition_definition_fingerprint": "sha256:approval-v1",
16589                }),
16590            )],
16591            "rust-workers".to_string(),
16592            DEFAULT_CODEC.to_string(),
16593            None,
16594        )
16595        .expect_err("condition history without occurrence identity must fail");
16596
16597        assert!(matches!(
16598            error,
16599            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16600                if reason == "condition_wait_occurrence_id_missing"
16601        ));
16602    }
16603
16604    #[test]
16605    fn typed_search_attribute_updates_validate_emit_and_replay() {
16606        let update = SearchAttributeUpdate::new()
16607            .keyword("OrderStatus", " waiting ")
16608            .expect("keyword")
16609            .int("Attempt", 3)
16610            .expect("int")
16611            .bool("Escalated", false)
16612            .expect("bool")
16613            .keyword_list("Regions", ["us-east", "eu-west"])
16614            .expect("list")
16615            .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
16616            .expect("datetime")
16617            .delete("LegacyStatus")
16618            .expect("delete");
16619        let ctx = workflow_context(Vec::new());
16620        ctx.upsert_search_attributes(update.clone())
16621            .expect("typed update");
16622        assert_eq!(
16623            ctx.take_commands().expect("search-attribute command"),
16624            vec![json!({
16625                "type": "upsert_search_attributes",
16626                "attributes": {
16627                    "Attempt": 3,
16628                    "Escalated": false,
16629                    "LegacyStatus": null,
16630                    "OrderStatus": "waiting",
16631                    "Regions": ["us-east", "eu-west"],
16632                    "UpdatedAt": "2026-08-22T04:00:00Z",
16633                },
16634                "attribute_types": {
16635                    "Attempt": "int",
16636                    "Escalated": "bool",
16637                    "OrderStatus": "keyword",
16638                    "Regions": "keyword_list",
16639                    "UpdatedAt": "datetime",
16640                },
16641            })]
16642        );
16643
16644        let replay = workflow_context(vec![history_event(
16645            "SearchAttributesUpserted",
16646            json!({
16647                "sequence": 6,
16648                "attributes": {
16649                    "Attempt": 3,
16650                    "Escalated": false,
16651                    "LegacyStatus": null,
16652                    "OrderStatus": "waiting",
16653                    "Regions": ["us-east", "eu-west"],
16654                    "UpdatedAt": "2026-08-22T04:00:00Z",
16655                },
16656                "attribute_types": {
16657                    "Attempt": "int",
16658                    "Escalated": "bool",
16659                    "OrderStatus": "keyword",
16660                    "Regions": "keyword_list",
16661                    "UpdatedAt": "datetime",
16662                },
16663                "merged": {},
16664            }),
16665        )]);
16666        replay
16667            .upsert_search_attributes(update)
16668            .expect("matching update replays");
16669        assert!(replay.take_commands().expect("commands").is_empty());
16670        replay.ensure_history_consumed().expect("history consumed");
16671
16672        let type_drift = workflow_context(vec![history_event(
16673            "SearchAttributesUpserted",
16674            json!({
16675                "sequence": 7,
16676                "attributes": {"OrderStatus": "waiting"},
16677                "attribute_types": {"OrderStatus": "keyword"},
16678                "merged": {"OrderStatus": "waiting"},
16679            }),
16680        )]);
16681        let error = type_drift
16682            .upsert_search_attributes(
16683                SearchAttributeUpdate::new()
16684                    .string("OrderStatus", "waiting")
16685                    .expect("string update"),
16686            )
16687            .expect_err("same JSON value with a changed type must fail replay");
16688        assert!(matches!(
16689            error,
16690            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16691                if reason == "search_attribute_type_mismatch"
16692        ));
16693
16694        let malformed_types = WorkflowState::new(
16695            vec![history_event(
16696                "SearchAttributesUpserted",
16697                json!({
16698                    "sequence": 8,
16699                    "attributes": {"OrderStatus": "waiting"},
16700                    "attribute_types": {"OrderStatus": "unsupported"},
16701                    "merged": {"OrderStatus": "waiting"},
16702                }),
16703            )],
16704            "rust-workers".to_string(),
16705            DEFAULT_CODEC.to_string(),
16706            None,
16707        )
16708        .expect_err("unsupported search-attribute type metadata must fail");
16709        assert!(matches!(
16710            malformed_types,
16711            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16712                if reason == "search_attribute_types_malformed"
16713        ));
16714
16715        assert!(matches!(
16716            SearchAttributeUpdate::new().keyword("bad key", "value"),
16717            Err(SearchAttributeUpdateError::InvalidKey(_))
16718        ));
16719        assert!(matches!(
16720            SearchAttributeUpdate::new().float("Ratio", f64::NAN),
16721            Err(SearchAttributeUpdateError::NonFiniteFloat(_))
16722        ));
16723        assert!(matches!(
16724            SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
16725            Err(SearchAttributeUpdateError::ValueTooLong { .. })
16726        ));
16727        assert!(matches!(
16728            SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
16729            Err(SearchAttributeUpdateError::InvalidDateTime(_))
16730        ));
16731        assert!(matches!(
16732            workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
16733            Err(Error::InvalidSearchAttributeUpdate(
16734                SearchAttributeUpdateError::Empty
16735            ))
16736        ));
16737    }
16738
16739    #[test]
16740    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
16741        let lone_fire = WorkflowState::new(
16742            vec![history_event(
16743                "TimerFired",
16744                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16745            )],
16746            "rust-workers".to_string(),
16747            DEFAULT_CODEC.to_string(),
16748            None,
16749        )
16750        .expect_err("TimerFired requires TimerScheduled");
16751        assert!(matches!(
16752            lone_fire,
16753            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16754                if reason == "timer_schedule_missing_or_duplicate"
16755        ));
16756
16757        let wrong_identity = WorkflowState::new(
16758            vec![
16759                history_event(
16760                    "TimerScheduled",
16761                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16762                ),
16763                history_event(
16764                    "TimerFired",
16765                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
16766                ),
16767            ],
16768            "rust-workers".to_string(),
16769            DEFAULT_CODEC.to_string(),
16770            None,
16771        )
16772        .expect_err("fire must match scheduled timer identity");
16773        assert!(matches!(
16774            wrong_identity,
16775            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16776                if reason == "timer_identity_mismatch"
16777        ));
16778
16779        let duplicate_fire = WorkflowState::new(
16780            vec![
16781                history_event(
16782                    "TimerScheduled",
16783                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16784                ),
16785                history_event(
16786                    "TimerFired",
16787                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16788                ),
16789                history_event(
16790                    "TimerFired",
16791                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16792                ),
16793            ],
16794            "rust-workers".to_string(),
16795            DEFAULT_CODEC.to_string(),
16796            None,
16797        )
16798        .expect_err("a durable timer cannot fire twice");
16799        assert!(matches!(
16800            duplicate_fire,
16801            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16802                if reason == "duplicate_timer_fire"
16803        ));
16804
16805        let wrong_fired_delay = WorkflowState::new(
16806            vec![
16807                history_event(
16808                    "TimerScheduled",
16809                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16810                ),
16811                history_event(
16812                    "TimerFired",
16813                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
16814                ),
16815            ],
16816            "rust-workers".to_string(),
16817            DEFAULT_CODEC.to_string(),
16818            None,
16819        )
16820        .expect_err("timer schedule and fire delays must agree");
16821        assert!(matches!(
16822            wrong_fired_delay,
16823            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16824                if reason == "timer_history_delay_mismatch"
16825        ));
16826    }
16827
16828    #[test]
16829    fn replay_rejects_activity_moved_before_recorded_timer() {
16830        let ctx = workflow_context(vec![
16831            history_event(
16832                "TimerScheduled",
16833                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16834            ),
16835            history_event(
16836                "TimerFired",
16837                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16838            ),
16839            history_event(
16840                "ActivityCompleted",
16841                json!({
16842                    "sequence": 2,
16843                    "activity_type": "after-timer",
16844                    "payload_codec": DEFAULT_CODEC,
16845                    "result": fixture_envelope(json!("done")),
16846                }),
16847            ),
16848        ]);
16849        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
16850        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16851
16852        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
16853            activity.as_mut().poll(&mut task_context)
16854        else {
16855            panic!("reordered durable command must be rejected");
16856        };
16857        assert_eq!(failure.reason, "recorded_command_mismatch");
16858        assert_eq!(failure.sequence, Some(1));
16859        assert_eq!(failure.expected.as_deref(), Some("timer"));
16860        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
16861    }
16862
16863    #[test]
16864    fn workflow_context_emits_a_typed_named_signal_wait() {
16865        let ctx = workflow_context(Vec::new());
16866        let mut signal = Box::pin(ctx.wait_signal("finish"));
16867        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16868
16869        assert!(matches!(
16870            signal.as_mut().poll(&mut task_context),
16871            Poll::Pending
16872        ));
16873        assert_eq!(
16874            ctx.take_commands().expect("signal-wait command"),
16875            vec![json!({
16876                "type": "open_signal_wait",
16877                "signal_name": "finish",
16878            })]
16879        );
16880    }
16881
16882    #[test]
16883    fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
16884        let ctx = workflow_context(Vec::new());
16885        let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
16886        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16887
16888        let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
16889        else {
16890            panic!("runtime-reserved signal should be rejected");
16891        };
16892        assert!(message.contains("reserved by the workflow runtime"));
16893        assert!(ctx.take_commands().expect("commands").is_empty());
16894    }
16895
16896    #[tokio::test]
16897    async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
16898        let client = Client::builder("http://127.0.0.1:9")
16899            .build()
16900            .expect("client");
16901        let error = client
16902            .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
16903            .await
16904            .expect_err("runtime-reserved signal should be rejected before transport");
16905
16906        assert!(
16907            matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
16908        );
16909    }
16910
16911    #[test]
16912    fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
16913        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
16914            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
16915                value.to_string(),
16916            )]))
16917            .expect("message payload");
16918            json!({
16919                "schema": MESSAGE_STREAM_SCHEMA,
16920                "stream_name": "orders",
16921                "message_id": message_id,
16922                "position": position,
16923                "payload_envelope": payload,
16924            })
16925        }
16926
16927        fn opened(sequence: u64) -> HistoryEvent {
16928            history_event(
16929                "SignalWaitOpened",
16930                json!({
16931                    "sequence": sequence,
16932                    "signal_name": MESSAGE_STREAM_SIGNAL,
16933                }),
16934            )
16935        }
16936
16937        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
16938            history_event(
16939                "SignalApplied",
16940                json!({
16941                    "sequence": sequence,
16942                    "signal_name": MESSAGE_STREAM_SIGNAL,
16943                    "value": fixture_envelope(json!([delivery])),
16944                }),
16945            )
16946        }
16947
16948        fn received(delivery: Value) -> HistoryEvent {
16949            history_event(
16950                "SignalReceived",
16951                json!({
16952                    "signal_name": MESSAGE_STREAM_SIGNAL,
16953                    "arguments": fixture_envelope(json!([delivery])),
16954                    "payload_codec": DEFAULT_CODEC,
16955                }),
16956            )
16957        }
16958
16959        let client = Client::new("http://127.0.0.1:8080").expect("client");
16960        let mut worker = Worker::new(client, "rust-workers");
16961        worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
16962            let messages = ctx.message_stream("orders")?.receive(2).await?;
16963            Ok(json!(messages
16964                .into_iter()
16965                .map(|message| message.message_id)
16966                .collect::<Vec<_>>()))
16967        });
16968
16969        let first = delivery("message-1", 1, "one");
16970        let second = delivery("message-2", 2, "two");
16971        let batch = worker
16972            .execute_workflow_task_decision(workflow_task(
16973                "rust.message-stream-batch",
16974                vec![
16975                    opened(1),
16976                    received(first.clone()),
16977                    applied(1, first.clone()),
16978                    received(first.clone()),
16979                    received(second),
16980                ],
16981                DEFAULT_CODEC,
16982            ))
16983            .expect("worker task consumes the available batch");
16984
16985        assert_eq!(batch.commands.len(), 1);
16986        assert_eq!(batch.commands[0]["type"], "complete_workflow");
16987        assert_eq!(
16988            decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
16989                .expect("workflow result"),
16990            json!(["message-1", "message-2"])
16991        );
16992        assert_eq!(
16993            batch.message_stream_cursors,
16994            vec![json!({"stream_name": "orders", "through_position": 2})]
16995        );
16996        assert!(batch.message_stream_waits.is_empty());
16997
16998        let partial = worker
16999            .execute_workflow_task_decision(workflow_task(
17000                "rust.message-stream-batch",
17001                vec![opened(1), received(first.clone()), applied(1, first)],
17002                DEFAULT_CODEC,
17003            ))
17004            .expect("worker task returns without waiting for a missing second item");
17005        assert_eq!(partial.commands.len(), 1);
17006        assert_eq!(partial.commands[0]["type"], "complete_workflow");
17007        assert_eq!(
17008            decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
17009                .expect("workflow result"),
17010            json!(["message-1"])
17011        );
17012        assert_eq!(
17013            partial.message_stream_cursors,
17014            vec![json!({"stream_name": "orders", "through_position": 1})]
17015        );
17016        assert!(partial.message_stream_waits.is_empty());
17017    }
17018
17019    #[test]
17020    fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
17021        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
17022            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
17023                value.to_string(),
17024            )]))
17025            .expect("message payload");
17026            json!({
17027                "schema": MESSAGE_STREAM_SCHEMA,
17028                "stream_name": "orders",
17029                "message_id": message_id,
17030                "position": position,
17031                "payload_envelope": payload,
17032            })
17033        }
17034
17035        fn opened(sequence: u64) -> HistoryEvent {
17036            history_event(
17037                "SignalWaitOpened",
17038                json!({
17039                    "sequence": sequence,
17040                    "signal_name": MESSAGE_STREAM_SIGNAL,
17041                }),
17042            )
17043        }
17044
17045        fn received(delivery: Value) -> HistoryEvent {
17046            history_event(
17047                "SignalReceived",
17048                json!({
17049                    "signal_name": MESSAGE_STREAM_SIGNAL,
17050                    "arguments": fixture_envelope(json!([delivery])),
17051                    "payload_codec": DEFAULT_CODEC,
17052                }),
17053            )
17054        }
17055
17056        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
17057            history_event(
17058                "SignalApplied",
17059                json!({
17060                    "sequence": sequence,
17061                    "signal_name": MESSAGE_STREAM_SIGNAL,
17062                    "value": fixture_envelope(json!([delivery])),
17063                }),
17064            )
17065        }
17066
17067        let client = Client::new("http://127.0.0.1:8080").expect("client");
17068        let mut worker = Worker::new(client, "rust-workers");
17069        worker.register_workflow(
17070            "rust.message-stream-partial-batches",
17071            |ctx, _input| async move {
17072                let stream = ctx.message_stream("orders")?;
17073                let first = stream.receive(10).await?;
17074                let second = stream.receive(10).await?;
17075                Ok(json!([
17076                    first
17077                        .into_iter()
17078                        .map(|message| message.message_id)
17079                        .collect::<Vec<_>>(),
17080                    second
17081                        .into_iter()
17082                        .map(|message| message.message_id)
17083                        .collect::<Vec<_>>(),
17084                ]))
17085            },
17086        );
17087
17088        let first = delivery("message-1", 1, "one");
17089        let second = delivery("message-2", 2, "two");
17090        let decision = worker
17091            .execute_workflow_task_decision(workflow_task(
17092                "rust.message-stream-partial-batches",
17093                vec![
17094                    opened(1),
17095                    received(first.clone()),
17096                    applied(1, first),
17097                    opened(2),
17098                    received(second.clone()),
17099                    applied(2, second),
17100                ],
17101                DEFAULT_CODEC,
17102            ))
17103            .expect("cold replay preserves both authored receive boundaries");
17104
17105        assert_eq!(decision.commands.len(), 1);
17106        assert_eq!(decision.commands[0]["type"], "complete_workflow");
17107        assert_eq!(
17108            decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
17109                .expect("workflow result"),
17110            json!([["message-1"], ["message-2"]])
17111        );
17112        assert_eq!(
17113            decision.message_stream_cursors,
17114            vec![json!({"stream_name": "orders", "through_position": 2})]
17115        );
17116        assert!(decision.message_stream_waits.is_empty());
17117    }
17118
17119    #[test]
17120    fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
17121        let ctx = workflow_context(Vec::new());
17122        let stream = ctx.message_stream("orders").expect("message stream");
17123        let mut receive = Box::pin(stream.receive(10));
17124        let mut task_context = TaskContext::from_waker(noop_waker_ref());
17125
17126        assert!(matches!(
17127            receive.as_mut().poll(&mut task_context),
17128            Poll::Pending
17129        ));
17130        assert_eq!(
17131            ctx.take_commands().expect("message-stream wait command"),
17132            vec![json!({
17133                "type": "open_signal_wait",
17134                "signal_name": MESSAGE_STREAM_SIGNAL,
17135            })]
17136        );
17137        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
17138        assert!(cursors.is_empty());
17139        assert_eq!(
17140            waits,
17141            vec![json!({"stream_name": "orders", "after_position": 0})]
17142        );
17143    }
17144
17145    #[test]
17146    fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
17147        let ctx = workflow_context(vec![history_event(
17148            "SignalReceived",
17149            json!({
17150                "signal_name": MESSAGE_STREAM_SIGNAL,
17151                "arguments": fixture_envelope(json!([{
17152                    "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
17153                    "stream_name": "orders",
17154                    "through_position": 2,
17155                }])),
17156                "payload_codec": DEFAULT_CODEC,
17157            }),
17158        )]);
17159        let stream = ctx.message_stream("orders").expect("message stream");
17160        let mut receive = Box::pin(stream.receive(10));
17161        let mut task_context = TaskContext::from_waker(noop_waker_ref());
17162
17163        assert!(matches!(
17164            receive.as_mut().poll(&mut task_context),
17165            Poll::Pending
17166        ));
17167        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
17168        assert_eq!(
17169            cursors,
17170            vec![json!({"stream_name": "orders", "through_position": 2})]
17171        );
17172        assert_eq!(
17173            waits,
17174            vec![json!({"stream_name": "orders", "after_position": 2})]
17175        );
17176    }
17177
17178    #[test]
17179    fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
17180        let mut empty_map = BTreeMap::new();
17181        let mut nested = BTreeMap::new();
17182        nested.insert(
17183            "value".to_string(),
17184            AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
17185        );
17186        let values = vec![
17187            AvroValue::Bytes(vec![0, 255]),
17188            AvroValue::Long(1),
17189            AvroValue::Double(1.0),
17190            AvroValue::Array(Vec::new()),
17191            AvroValue::Map(std::mem::take(&mut empty_map)),
17192            AvroValue::Map(nested),
17193        ];
17194        let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
17195        let transport = vec![json!({
17196            "schema": MESSAGE_STREAM_SCHEMA,
17197            "stream_name": "orders",
17198            "message_id": "message-1",
17199            "position": 1,
17200            "payload_envelope": payload,
17201        })];
17202
17203        for _ in 0..2 {
17204            let Some(MessageStreamDelivery::Message(message)) =
17205                decode_message_stream_delivery(transport.clone()).expect("delivery")
17206            else {
17207                panic!("message delivery expected");
17208            };
17209            assert_eq!(message.arguments, values);
17210            assert!(matches!(message.arguments[1], AvroValue::Long(1)));
17211            assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
17212        }
17213    }
17214
17215    #[test]
17216    fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
17217        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
17218            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
17219                value.to_string(),
17220            )]))
17221            .expect("message payload");
17222            json!({
17223                "schema": MESSAGE_STREAM_SCHEMA,
17224                "stream_name": "orders",
17225                "message_id": message_id,
17226                "position": position,
17227                "payload_envelope": payload,
17228            })
17229        }
17230
17231        fn opened(sequence: u64) -> HistoryEvent {
17232            history_event(
17233                "SignalWaitOpened",
17234                json!({
17235                    "sequence": sequence,
17236                    "signal_name": MESSAGE_STREAM_SIGNAL,
17237                }),
17238            )
17239        }
17240
17241        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
17242            history_event(
17243                "SignalApplied",
17244                json!({
17245                    "sequence": sequence,
17246                    "signal_name": MESSAGE_STREAM_SIGNAL,
17247                    "value": fixture_envelope(json!([delivery])),
17248                }),
17249            )
17250        }
17251
17252        fn worker() -> Worker {
17253            let client = Client::new("http://127.0.0.1:8080").expect("client");
17254            let mut worker = Worker::new(client, "rust-workers");
17255            worker.register_workflow("rust.message-stream", |ctx, _input| async move {
17256                let stream = ctx.message_stream("orders")?;
17257                let first = stream.receive_one().await?;
17258                let second = stream.receive_one().await?;
17259                Ok(json!([first.message_id, second.message_id]))
17260            });
17261            worker
17262        }
17263
17264        fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
17265            let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
17266            task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
17267            task.signal_arguments = Some(fixture_envelope(json!([delivery])));
17268            task
17269        }
17270
17271        let waiting = worker()
17272            .execute_workflow_task_decision(workflow_task(
17273                "rust.message-stream",
17274                Vec::new(),
17275                DEFAULT_CODEC,
17276            ))
17277            .expect("first worker opens the stream wait");
17278        assert_eq!(
17279            waiting.commands,
17280            vec![json!({
17281                "type": "open_signal_wait",
17282                "signal_name": MESSAGE_STREAM_SIGNAL,
17283            })]
17284        );
17285        assert!(waiting.message_stream_cursors.is_empty());
17286        assert_eq!(
17287            waiting.message_stream_waits,
17288            vec![json!({"stream_name": "orders", "after_position": 0})]
17289        );
17290
17291        let first_delivery = delivery("message-1", 1, "one");
17292        let first_arrival = worker()
17293            .execute_workflow_task_decision(task_with_resume(
17294                vec![opened(1)],
17295                first_delivery.clone(),
17296            ))
17297            .expect("replacement worker consumes the first arrival");
17298        assert_eq!(
17299            first_arrival.commands,
17300            vec![json!({
17301                "type": "open_signal_wait",
17302                "signal_name": MESSAGE_STREAM_SIGNAL,
17303            })]
17304        );
17305        assert_eq!(
17306            first_arrival.message_stream_cursors,
17307            vec![json!({"stream_name": "orders", "through_position": 1})]
17308        );
17309        assert_eq!(
17310            first_arrival.message_stream_waits,
17311            vec![json!({"stream_name": "orders", "after_position": 1})]
17312        );
17313
17314        let second_delivery = delivery("message-2", 2, "two");
17315        let first_applied = applied(1, first_delivery);
17316        let completed = worker()
17317            .execute_workflow_task_decision(task_with_resume(
17318                vec![opened(1), first_applied.clone(), opened(2)],
17319                second_delivery.clone(),
17320            ))
17321            .expect("next replacement worker consumes the second arrival");
17322        assert_eq!(completed.commands.len(), 1);
17323        assert_eq!(completed.commands[0]["type"], "complete_workflow");
17324        assert_eq!(
17325            decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
17326                .expect("workflow result"),
17327            json!(["message-1", "message-2"])
17328        );
17329        assert_eq!(
17330            completed.message_stream_cursors,
17331            vec![json!({"stream_name": "orders", "through_position": 2})]
17332        );
17333        assert!(completed.message_stream_waits.is_empty());
17334
17335        let replay_history = vec![
17336            opened(1),
17337            first_applied,
17338            opened(2),
17339            applied(2, second_delivery),
17340        ];
17341        for _cold_worker_or_restart in 0..2 {
17342            let replayed = worker()
17343                .execute_workflow_task_decision(workflow_task(
17344                    "rust.message-stream",
17345                    replay_history.clone(),
17346                    DEFAULT_CODEC,
17347                ))
17348                .expect("cold worker replays each logical message exactly once");
17349            assert_eq!(replayed.commands.len(), 1);
17350            assert_eq!(
17351                decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
17352                    .expect("replayed workflow result"),
17353                json!(["message-1", "message-2"])
17354            );
17355            assert_eq!(
17356                replayed.message_stream_cursors,
17357                vec![json!({"stream_name": "orders", "through_position": 2})]
17358            );
17359            assert!(replayed.message_stream_waits.is_empty());
17360        }
17361    }
17362
17363    #[test]
17364    fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
17365        assert!(!worker_protocol_supports_message_streams("1.14"));
17366        assert!(worker_protocol_supports_message_streams("1.15"));
17367        assert!(worker_protocol_supports_message_streams("1.16"));
17368        assert!(worker_protocol_supports_message_streams(
17369            WORKER_PROTOCOL_VERSION
17370        ));
17371        assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
17372    }
17373
17374    #[test]
17375    fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
17376        let ctx = workflow_context(vec![
17377            history_event(
17378                "ConditionWaitOpened",
17379                json!({
17380                    "sequence": 1,
17381                    "condition_wait_id": "condition:1",
17382                    "condition_wait_occurrence_id": "rust:condition-wait:0",
17383                    "condition_key": "signal:finish",
17384                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
17385                }),
17386            ),
17387            history_event(
17388                "ConditionWaitSatisfied",
17389                json!({
17390                    "sequence": 1,
17391                    "condition_wait_id": "condition:1",
17392                    "condition_wait_occurrence_id": "rust:condition-wait:0",
17393                    "condition_key": "signal:finish",
17394                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
17395                }),
17396            ),
17397            history_event(
17398                "SignalReceived",
17399                json!({"signal_name": "finish", "arguments": []}),
17400            ),
17401        ]);
17402        let mut signal = Box::pin(ctx.wait_signal("finish"));
17403        let mut task_context = TaskContext::from_waker(noop_waker_ref());
17404
17405        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
17406            signal.as_mut().poll(&mut task_context)
17407        else {
17408            panic!("condition history must not resolve as a typed signal wait");
17409        };
17410        assert_eq!(failure.reason, "recorded_command_mismatch");
17411        assert_eq!(failure.expected.as_deref(), Some("condition wait"));
17412    }
17413
17414    #[test]
17415    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
17416        let signal_then_timer = vec![
17417            history_event(
17418                "SignalWaitOpened",
17419                json!({"sequence": 1, "signal_name": "go"}),
17420            ),
17421            history_event(
17422                "SignalApplied",
17423                json!({
17424                    "sequence": 1,
17425                    "signal_name": "go",
17426                    "value": fixture_envelope(json!(["now"])),
17427                }),
17428            ),
17429            history_event(
17430                "TimerScheduled",
17431                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
17432            ),
17433            history_event(
17434                "TimerFired",
17435                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
17436            ),
17437        ];
17438
17439        let ctx = workflow_context(signal_then_timer.clone());
17440        let mut signal = Box::pin(ctx.wait_signal("go"));
17441        let mut task_context = TaskContext::from_waker(noop_waker_ref());
17442        assert!(matches!(
17443            signal.as_mut().poll(&mut task_context),
17444            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
17445        ));
17446        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
17447        assert!(matches!(
17448            timer.as_mut().poll(&mut task_context),
17449            Poll::Ready(Ok(()))
17450        ));
17451        ctx.ensure_history_consumed()
17452            .expect("signal and timer history consumed in order");
17453
17454        let reordered = workflow_context(signal_then_timer);
17455        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
17456        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
17457            timer_first.as_mut().poll(&mut task_context)
17458        else {
17459            panic!("timer cannot consume signal-wait-first history");
17460        };
17461        assert_eq!(failure.reason, "recorded_command_mismatch");
17462        assert_eq!(failure.sequence, Some(1));
17463        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
17464
17465        let timer_then_signal = vec![
17466            history_event(
17467                "TimerScheduled",
17468                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
17469            ),
17470            history_event(
17471                "TimerFired",
17472                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
17473            ),
17474            history_event(
17475                "SignalWaitOpened",
17476                json!({"sequence": 2, "signal_name": "go"}),
17477            ),
17478            history_event(
17479                "SignalApplied",
17480                json!({
17481                    "sequence": 2,
17482                    "signal_name": "go",
17483                    "value": fixture_envelope(json!([])),
17484                }),
17485            ),
17486        ];
17487        let reordered = workflow_context(timer_then_signal);
17488        let mut signal_first = Box::pin(reordered.wait_signal("go"));
17489        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
17490            signal_first.as_mut().poll(&mut task_context)
17491        else {
17492            panic!("signal wait cannot consume timer-first history");
17493        };
17494        assert_eq!(failure.reason, "recorded_command_mismatch");
17495        assert_eq!(failure.sequence, Some(1));
17496        assert_eq!(failure.expected.as_deref(), Some("timer"));
17497    }
17498
17499    #[test]
17500    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
17501        let duplicate_timer = WorkflowState::new(
17502            vec![
17503                history_event(
17504                    "TimerScheduled",
17505                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
17506                ),
17507                history_event(
17508                    "TimerScheduled",
17509                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
17510                ),
17511            ],
17512            "rust-workers".to_string(),
17513            DEFAULT_CODEC.to_string(),
17514            None,
17515        )
17516        .expect_err("one workflow sequence cannot schedule two timers");
17517        assert!(matches!(
17518            duplicate_timer,
17519            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17520                if reason == "timer_schedule_missing_or_duplicate"
17521        ));
17522
17523        let colliding_kinds = WorkflowState::new(
17524            vec![
17525                history_event(
17526                    "TimerScheduled",
17527                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
17528                ),
17529                history_event(
17530                    "ActivityCompleted",
17531                    json!({"sequence": 1, "activity_type": "same-sequence"}),
17532                ),
17533            ],
17534            "rust-workers".to_string(),
17535            DEFAULT_CODEC.to_string(),
17536            None,
17537        )
17538        .expect_err("one workflow sequence cannot identify two command kinds");
17539        assert!(matches!(
17540            colliding_kinds,
17541            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17542                if reason == "durable_command_sequence_collision"
17543        ));
17544
17545        let duplicate_signal_wait = WorkflowState::new(
17546            vec![
17547                history_event(
17548                    "SignalWaitOpened",
17549                    json!({"sequence": 1, "signal_name": "go"}),
17550                ),
17551                history_event(
17552                    "SignalWaitOpened",
17553                    json!({"sequence": 1, "signal_name": "go"}),
17554                ),
17555            ],
17556            "rust-workers".to_string(),
17557            DEFAULT_CODEC.to_string(),
17558            None,
17559        )
17560        .expect_err("one workflow sequence cannot open two signal waits");
17561        assert!(matches!(
17562            duplicate_signal_wait,
17563            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17564                if reason == "signal_wait_open_missing_or_duplicate"
17565        ));
17566    }
17567
17568    #[test]
17569    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
17570        let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
17571            .expect("side-effect result");
17572        let ctx = workflow_context(vec![history_event(
17573            "SideEffectRecorded",
17574            json!({"sequence": 99, "result": result}),
17575        )]);
17576
17577        let replayed: Value = ctx
17578            .side_effect(|| panic!("recorded side effect must not run"))
17579            .expect("positive global workflow sequence is valid");
17580        assert_eq!(replayed, json!({"captured": true}));
17581        ctx.ensure_history_consumed().expect("history consumed");
17582    }
17583
17584    #[test]
17585    fn workflow_history_rejects_zero_and_descending_command_sequences() {
17586        let result =
17587            encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
17588        let zero = WorkflowState::new(
17589            vec![history_event(
17590                "SideEffectRecorded",
17591                json!({"sequence": 0, "result": result.clone()}),
17592            )],
17593            "rust-workers".to_string(),
17594            DEFAULT_CODEC.to_string(),
17595            None,
17596        )
17597        .expect_err("durable command sequences must be positive");
17598        assert!(matches!(
17599            zero,
17600            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17601                if reason == "durable_command_sequence_invalid"
17602        ));
17603
17604        let descending = WorkflowState::new(
17605            vec![
17606                history_event(
17607                    "SideEffectRecorded",
17608                    json!({"sequence": 3, "result": result}),
17609                ),
17610                history_event(
17611                    "VersionMarkerRecorded",
17612                    json!({
17613                        "sequence": 2,
17614                        "change_id": "descending-marker",
17615                        "version": 1,
17616                        "min_supported": 1,
17617                        "max_supported": 1,
17618                    }),
17619                ),
17620            ],
17621            "rust-workers".to_string(),
17622            DEFAULT_CODEC.to_string(),
17623            None,
17624        )
17625        .expect_err("new durable commands must remain strictly ordered");
17626        let Error::NonDeterministicReplay(failure) = descending else {
17627            panic!("expected typed replay failure");
17628        };
17629        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
17630        assert_eq!(failure.sequence, Some(2));
17631        assert_eq!(
17632            failure.expected.as_deref(),
17633            Some("workflow sequence greater than 3")
17634        );
17635        assert_eq!(failure.actual.as_deref(), Some("2"));
17636    }
17637
17638    #[test]
17639    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
17640        fn worker() -> Worker {
17641            let client = Client::new("http://127.0.0.1:8080").expect("client");
17642            let mut worker = Worker::new(client, "rust-workers");
17643            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
17644                ctx.wait_signal("finish").await?;
17645                let marker: String =
17646                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
17647                assert_eq!(marker, "after-finish");
17648                Ok(json!("finished"))
17649            });
17650            worker
17651        }
17652
17653        let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
17654            .expect("side-effect result");
17655        let task = workflow_task(
17656            "rust.finish-after-gaps",
17657            vec![
17658                history_event(
17659                    "SignalWaitOpened",
17660                    json!({"sequence": 1, "signal_name": "finish"}),
17661                ),
17662                history_event(
17663                    "SignalReceived",
17664                    json!({
17665                        "signal_id": "increment-3",
17666                        "signal_name": "increment",
17667                        "workflow_sequence": 2,
17668                        "payload_codec": DEFAULT_CODEC,
17669                        "arguments": fixture_envelope(json!([3])),
17670                    }),
17671                ),
17672                history_event(
17673                    "SignalReceived",
17674                    json!({
17675                        "signal_id": "increment-5",
17676                        "signal_name": "increment",
17677                        "workflow_sequence": 3,
17678                        "payload_codec": DEFAULT_CODEC,
17679                        "arguments": fixture_envelope(json!([5])),
17680                    }),
17681                ),
17682                history_event(
17683                    "SignalReceived",
17684                    json!({
17685                        "signal_id": "finish",
17686                        "signal_name": "finish",
17687                        "workflow_sequence": 4,
17688                        "payload_codec": DEFAULT_CODEC,
17689                        "arguments": fixture_envelope(json!([])),
17690                    }),
17691                ),
17692                history_event(
17693                    "SignalApplied",
17694                    json!({
17695                        "sequence": 1,
17696                        "signal_id": "finish",
17697                        "signal_name": "finish",
17698                        "payload_codec": DEFAULT_CODEC,
17699                        "value": fixture_envelope(json!([])),
17700                    }),
17701                ),
17702                history_event(
17703                    "SideEffectRecorded",
17704                    json!({"sequence": 5, "result": marker}),
17705                ),
17706            ],
17707            DEFAULT_CODEC,
17708        );
17709
17710        for _original_or_cold_worker in 0..2 {
17711            let commands = worker()
17712                .execute_workflow_task(task.clone())
17713                .expect("signal gaps preserve deterministic replay");
17714            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
17715            assert_eq!(commands[0]["type"], "complete_workflow");
17716            assert_eq!(
17717                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
17718                json!("finished")
17719            );
17720        }
17721    }
17722
17723    #[test]
17724    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
17725        let ctx = workflow_context(Vec::new());
17726        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
17727        let mut task_context = TaskContext::from_waker(noop_waker_ref());
17728        assert!(matches!(
17729            sleep.as_mut().poll(&mut task_context),
17730            Poll::Ready(Err(Error::TimerDurationOverflow))
17731        ));
17732        assert!(ctx.take_commands().expect("commands").is_empty());
17733    }
17734
17735    #[test]
17736    fn workflow_memo_update_emits_canonical_command_and_replays_once() {
17737        let entries = AvroValue::Map(BTreeMap::from([
17738            ("text".to_string(), AvroValue::String("same".to_string())),
17739            (
17740                "nested".to_string(),
17741                AvroValue::Map(BTreeMap::from([
17742                    ("beta".to_string(), AvroValue::Long(2)),
17743                    ("alpha".to_string(), AvroValue::Long(1)),
17744                ])),
17745            ),
17746            ("long".to_string(), AvroValue::Long(7)),
17747            ("double".to_string(), AvroValue::Double(7.0)),
17748            ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
17749        ]));
17750        let ctx = workflow_context(Vec::new());
17751        ctx.upsert_memo(entries.clone()).expect("valid memo update");
17752        let commands = ctx.take_commands().expect("commands");
17753
17754        assert_eq!(commands.len(), 1);
17755        assert_eq!(commands[0]["type"], "upsert_memo");
17756        let server_entries = json!({
17757            "codec": "avro",
17758            "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
17759        });
17760        assert_eq!(
17761            commands[0]["entries"]
17762                .as_object()
17763                .expect("entries envelope")
17764                .keys()
17765                .collect::<Vec<_>>(),
17766            vec!["blob", "codec"]
17767        );
17768        assert_eq!(commands[0]["entries"], server_entries);
17769        let wire_entries =
17770            decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
17771        assert_eq!(wire_entries, entries);
17772
17773        let history = vec![history_event(
17774            "MemoUpserted",
17775            json!({
17776                "sequence": 1,
17777                "entries": server_entries.clone(),
17778                "merged": server_entries,
17779            }),
17780        )];
17781        let replay = workflow_context(history.clone());
17782        replay
17783            .upsert_memo(entries.clone())
17784            .expect("matching replay identity");
17785        assert!(replay.take_commands().expect("replay commands").is_empty());
17786
17787        let changed_types = AvroValue::Map(BTreeMap::from([
17788            ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
17789            (
17790                "nested".to_string(),
17791                AvroValue::Map(BTreeMap::from([
17792                    ("alpha".to_string(), AvroValue::Long(1)),
17793                    ("beta".to_string(), AvroValue::Long(2)),
17794                ])),
17795            ),
17796            ("long".to_string(), AvroValue::Double(7.0)),
17797            ("double".to_string(), AvroValue::Long(7)),
17798            ("binary".to_string(), AvroValue::String("same".to_string())),
17799        ]));
17800        let error = workflow_context(history)
17801            .upsert_memo(changed_types)
17802            .expect_err("memo replay identity must preserve Avro value types");
17803        assert!(matches!(
17804            error,
17805            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
17806        ));
17807    }
17808
17809    #[test]
17810    fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
17811        let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
17812            .expect("memo envelope");
17813        let replay = workflow_context(vec![history_event(
17814            "MemoUpserted",
17815            json!({
17816                "sequence": 1,
17817                "entries": original.clone(),
17818                "merged": original
17819            }),
17820        )]);
17821        let error = replay
17822            .upsert_memo(json!({"stage": "changed"}))
17823            .expect_err("changed memo update must fail replay");
17824        assert!(matches!(
17825            error,
17826            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
17827        ));
17828
17829        let invalid = workflow_context(Vec::new())
17830            .upsert_memo(
17831                json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
17832            )
17833            .expect_err("oversized key");
17834        assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
17835    }
17836
17837    #[test]
17838    fn workflow_memo_replay_distinguishes_signed_zero_identity() {
17839        let negative_zero = AvroValue::Map(BTreeMap::from([(
17840            "reading".to_string(),
17841            AvroValue::Double(-0.0),
17842        )]));
17843        let negative_zero_envelope =
17844            encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
17845        let history = vec![history_event(
17846            "MemoUpserted",
17847            json!({
17848                "sequence": 1,
17849                "entries": negative_zero_envelope.clone(),
17850                "merged": negative_zero_envelope,
17851            }),
17852        )];
17853
17854        workflow_context(history.clone())
17855            .upsert_memo(negative_zero)
17856            .expect("matching negative-zero history identity");
17857
17858        let error = workflow_context(history)
17859            .upsert_memo(AvroValue::Map(BTreeMap::from([(
17860                "reading".to_string(),
17861                AvroValue::Double(0.0),
17862            )])))
17863            .expect_err("positive zero must not consume negative-zero memo history");
17864        assert!(matches!(
17865            error,
17866            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
17867        ));
17868    }
17869
17870    #[test]
17871    fn workflow_memo_capability_requires_flag_and_command_advertisement() {
17872        let supported = json!({
17873            "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
17874            "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
17875        });
17876        assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
17877        assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
17878            "workflow_memo_updates": {"supported": false},
17879            "supported_workflow_task_commands": ["upsert_memo"]
17880        }))));
17881        assert!(commands_use_workflow_memo_updates(&[json!({
17882            "type": "upsert_memo",
17883            "entries": {"stage": "processing"}
17884        })]));
17885    }
17886
17887    #[test]
17888    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
17889        let client = Client::new("http://127.0.0.1:8080").expect("client");
17890        let mut worker = Worker::new(client, "rust-workers");
17891        worker.register_workflow("rust.timer", |ctx, _input| async move {
17892            ctx.sleep(Duration::from_secs(5)).await?;
17893            ctx.activity("after-timer", json!([])).await
17894        });
17895
17896        let task = |history_events| WorkflowTask {
17897            task_id: "wft-rust-timer-1".to_string(),
17898            workflow_command_id: None,
17899            workflow_id: Some("wf-rust-timer".to_string()),
17900            run_id: Some("run-rust-timer".to_string()),
17901            workflow_type: "rust.timer".to_string(),
17902            cancel_requested: false,
17903            payload_codec: DEFAULT_CODEC.to_string(),
17904            arguments: Some(
17905                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
17906            ),
17907            history_events,
17908            total_history_events: None,
17909            history_size_bytes: None,
17910            continue_as_new_recommended: None,
17911            history_budget_pressure: None,
17912            next_history_page_token: None,
17913            workflow_task_attempt: 1,
17914            workflow_signal_id: None,
17915            signal_name: None,
17916            signal_arguments: None,
17917            workflow_update_id: None,
17918            update_name: None,
17919            lease_owner: Some("rust-worker".to_string()),
17920        };
17921
17922        let initial = worker
17923            .execute_workflow_task(task(Vec::new()))
17924            .expect("initial timer task");
17925        assert_eq!(
17926            initial,
17927            vec![json!({"type": "start_timer", "delay_seconds": 5})]
17928        );
17929
17930        let activity_result =
17931            encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
17932        let replayed = worker
17933            .execute_workflow_task(task(vec![
17934                history_event(
17935                    "TimerScheduled",
17936                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
17937                ),
17938                history_event(
17939                    "TimerFired",
17940                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
17941                ),
17942                history_event(
17943                    "ActivityCompleted",
17944                    json!({
17945                        "sequence": 2,
17946                        "activity_type": "after-timer",
17947                        "payload_codec": DEFAULT_CODEC,
17948                        "result": activity_result,
17949                    }),
17950                ),
17951            ]))
17952            .expect("replayed workflow task");
17953        assert_eq!(replayed.len(), 1);
17954        assert_eq!(replayed[0]["type"], "complete_workflow");
17955        assert_eq!(
17956            decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
17957            json!("done")
17958        );
17959    }
17960
17961    #[test]
17962    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
17963        let client = Client::new("http://127.0.0.1:8080").expect("client");
17964        let mut worker = Worker::new(client, "rust-workers");
17965        worker.register_workflow("rust.continue", |ctx, _input| async move {
17966            ctx.continue_as_new_with_options(
17967                ContinueAsNewOptions::new()
17968                    .workflow_type("rust.next")
17969                    .task_queue("next-workers"),
17970                json!([2, {"cursor": "next"}]),
17971            )
17972        });
17973
17974        let commands = worker
17975            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
17976            .expect("continue-as-new command");
17977
17978        assert_eq!(commands.len(), 1);
17979        assert_eq!(commands[0]["type"], "continue_as_new");
17980        assert_eq!(commands[0]["workflow_type"], "rust.next");
17981        assert_eq!(commands[0]["queue"], "next-workers");
17982        assert_eq!(
17983            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
17984                .expect("continue-as-new arguments"),
17985            json!([2, {"cursor": "next"}])
17986        );
17987    }
17988
17989    #[test]
17990    fn continue_as_new_preserves_typed_arguments() {
17991        let client = Client::new("http://127.0.0.1:8080").expect("client");
17992        let mut worker = Worker::new(client, "rust-workers");
17993        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
17994            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
17995            unreachable!("continue-as-new returns a control-flow error")
17996        });
17997
17998        let commands = worker
17999            .execute_workflow_task(workflow_task(
18000                "rust.typed-continue",
18001                Vec::new(),
18002                DEFAULT_CODEC,
18003            ))
18004            .expect("typed continue-as-new command");
18005
18006        assert_eq!(commands[0]["type"], "continue_as_new");
18007        assert_eq!(
18008            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
18009                .expect("typed continue arguments"),
18010            AvroValue::Array(vec![typed_fidelity_probe()])
18011        );
18012    }
18013
18014    #[test]
18015    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
18016        let client = Client::new("http://127.0.0.1:8080").expect("client");
18017        let mut worker = Worker::new(client, "rust-workers");
18018        worker.register_workflow("rust.continue", |ctx, _input| async move {
18019            ctx.continue_as_new(json!([2]))
18020        });
18021        let task = workflow_task(
18022            "rust.continue",
18023            vec![history_event(
18024                "WorkflowContinuedAsNew",
18025                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
18026            )],
18027            DEFAULT_CODEC,
18028        );
18029
18030        for _worker_restart_or_redelivery in 0..2 {
18031            let commands = worker
18032                .execute_workflow_task(task.clone())
18033                .expect("recorded transition replays");
18034            assert!(
18035                commands.is_empty(),
18036                "replay must not emit another successor"
18037            );
18038        }
18039    }
18040
18041    #[test]
18042    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
18043        let ctx = workflow_context(Vec::new());
18044        let error = ctx
18045            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
18046            .expect_err("blank queue must be rejected");
18047
18048        let Error::InvalidContinueAsNewOptions(error) = error else {
18049            panic!("expected typed continue-as-new validation error");
18050        };
18051        assert_eq!(error.field, "task_queue");
18052        assert!(ctx.take_commands().expect("commands").is_empty());
18053    }
18054
18055    #[test]
18056    fn workflow_context_exposes_server_history_budget() {
18057        let client = Client::new("http://127.0.0.1:8080").expect("client");
18058        let mut worker = Worker::new(client, "rust-workers");
18059        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
18060            let budget = ctx.history_budget()?;
18061            Ok(json!({
18062                "events": budget.event_count,
18063                "bytes": budget.size_bytes,
18064                "recommended": budget.continue_as_new_recommended,
18065                "pressure": budget.pressure,
18066            }))
18067        });
18068        let task: WorkflowTask = serde_json::from_value(json!({
18069            "task_id": "task-history-budget",
18070            "workflow_type": "rust.history-budget",
18071            "payload_codec": DEFAULT_CODEC,
18072            "history_events": [],
18073            "total_history_events": 480,
18074            "history_size_bytes": 1_048_576,
18075            "continue_as_new_recommended": true,
18076            "history_budget_pressure": "continue_as_new_recommended",
18077        }))
18078        .expect("published workflow task");
18079
18080        let commands = worker
18081            .execute_workflow_task(task)
18082            .expect("history-budget workflow");
18083        let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
18084        assert_eq!(result["events"], 480);
18085        assert_eq!(result["bytes"], 1_048_576);
18086        assert_eq!(result["recommended"], true);
18087        assert_eq!(result["pressure"], "continue_as_new_recommended");
18088    }
18089
18090    #[test]
18091    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
18092        let client = Client::new("http://127.0.0.1:8080").expect("client");
18093        let mut worker = Worker::new(client, "rust-workers");
18094        worker.register_workflow("rust.failing", |_ctx, _input| async move {
18095            Err(Error::Codec("rust_conformance_failure".to_string()))
18096        });
18097        let task = WorkflowTask {
18098            task_id: "wft-rust-failing-1".to_string(),
18099            workflow_command_id: None,
18100            workflow_id: Some("wf-rust-failing".to_string()),
18101            run_id: Some("run-rust-failing".to_string()),
18102            workflow_type: "rust.failing".to_string(),
18103            cancel_requested: false,
18104            payload_codec: DEFAULT_CODEC.to_string(),
18105            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
18106            history_events: Vec::new(),
18107            total_history_events: Some(0),
18108            history_size_bytes: None,
18109            continue_as_new_recommended: None,
18110            history_budget_pressure: None,
18111            next_history_page_token: None,
18112            workflow_task_attempt: 1,
18113            workflow_signal_id: None,
18114            signal_name: None,
18115            signal_arguments: None,
18116            workflow_update_id: None,
18117            update_name: None,
18118            lease_owner: Some("rust-worker".to_string()),
18119        };
18120
18121        let commands = worker
18122            .execute_workflow_task(task)
18123            .expect("handler failure becomes a workflow command");
18124
18125        assert_eq!(commands.len(), 1);
18126        assert_eq!(commands[0]["type"], "fail_workflow");
18127        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
18128        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
18129        assert_eq!(commands[0]["non_retryable"], false);
18130        assert_eq!(
18131            commands[0]["message"],
18132            "codec error: rust_conformance_failure"
18133        );
18134        assert_eq!(
18135            commands[0]["exception"]["message"],
18136            "codec error: rust_conformance_failure"
18137        );
18138    }
18139
18140    #[test]
18141    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
18142        let client = Client::new("http://127.0.0.1:8080").expect("client");
18143        let mut worker = Worker::new(client, "rust-workers");
18144        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
18145            let _: String = ctx.side_effect(|| "captured".to_string())?;
18146            Err(Error::WorkerLoop("application failure".to_string()))
18147        });
18148
18149        let commands = worker
18150            .execute_workflow_task(workflow_task(
18151                "rust.failing-after-side-effect",
18152                Vec::new(),
18153                DEFAULT_CODEC,
18154            ))
18155            .expect("ordinary failure remains a workflow decision");
18156
18157        assert_eq!(commands.len(), 2);
18158        assert_eq!(commands[0]["type"], "record_side_effect");
18159        assert_eq!(commands[1]["type"], "fail_workflow");
18160    }
18161
18162    #[test]
18163    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
18164        let client = Client::new("http://127.0.0.1:8080").expect("client");
18165        let mut worker = Worker::new(client, "rust-workers");
18166        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
18167            Err(Error::WorkerLoop("application failure".to_string()))
18168        });
18169        let result =
18170            encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
18171
18172        let error = worker
18173            .execute_workflow_task(workflow_task(
18174                "rust.removed-side-effect",
18175                vec![history_event(
18176                    "SideEffectRecorded",
18177                    json!({"sequence": 1, "result": result}),
18178                )],
18179                DEFAULT_CODEC,
18180            ))
18181            .expect_err("removed committed history must not become fail_workflow");
18182
18183        let Error::NonDeterministicReplay(failure) = error else {
18184            panic!("expected typed replay failure");
18185        };
18186        assert_eq!(failure.reason, "recorded_commands_unconsumed");
18187        assert_eq!(failure.sequence, Some(1));
18188        assert_eq!(failure.expected.as_deref(), Some("side effect"));
18189    }
18190
18191    #[test]
18192    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
18193        let client = Client::new("http://127.0.0.1:8080").expect("client");
18194        let mut worker = Worker::new(client, "rust-workers");
18195        worker.register_workflow(
18196            "rust.side-effect-before-marker-error",
18197            |ctx, _input| async move {
18198                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
18199                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
18200                ctx.get_version("restart-safe", 2, 2)?;
18201                Ok(Value::Null)
18202            },
18203        );
18204
18205        let error = worker
18206            .execute_workflow_task(workflow_task(
18207                "rust.side-effect-before-marker-error",
18208                vec![history_event(
18209                    "VersionMarkerRecorded",
18210                    json!({
18211                        "sequence": 1,
18212                        "change_id": "restart-safe",
18213                        "version": 1,
18214                        "min_supported": 1,
18215                        "max_supported": 1,
18216                    }),
18217                )],
18218                DEFAULT_CODEC,
18219            ))
18220            .expect_err("replay error must return no queued workflow commands");
18221
18222        let Error::NonDeterministicReplay(failure) = error else {
18223            panic!("expected typed replay failure");
18224        };
18225        assert_eq!(failure.reason, "version_marker_incompatible_range");
18226        assert_eq!(failure.sequence, Some(1));
18227    }
18228
18229    #[test]
18230    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
18231        let client = Client::new("http://127.0.0.1:8080").expect("client");
18232        let mut worker = Worker::new(client, "rust-workers");
18233        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
18234            ctx.sleep(Duration::from_secs(5)).await?;
18235            Ok(json!({"status": "timer fired"}))
18236        });
18237
18238        let task = WorkflowTask {
18239            task_id: "wft-rust-timer-pending".to_string(),
18240            workflow_command_id: None,
18241            workflow_id: Some("wf-rust-timer".to_string()),
18242            run_id: Some("run-rust-timer".to_string()),
18243            workflow_type: "rust.timer.pending".to_string(),
18244            cancel_requested: false,
18245            payload_codec: DEFAULT_CODEC.to_string(),
18246            arguments: Some(
18247                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
18248            ),
18249            history_events: vec![history_event(
18250                "TimerScheduled",
18251                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18252            )],
18253            total_history_events: Some(1),
18254            history_size_bytes: None,
18255            continue_as_new_recommended: None,
18256            history_budget_pressure: None,
18257            next_history_page_token: None,
18258            workflow_task_attempt: 1,
18259            workflow_signal_id: None,
18260            signal_name: None,
18261            signal_arguments: None,
18262            workflow_update_id: None,
18263            update_name: None,
18264            lease_owner: Some("rust-worker".to_string()),
18265        };
18266
18267        for _redelivery_or_restart in 0..2 {
18268            let commands = worker
18269                .execute_workflow_task(task.clone())
18270                .expect("recorded timer remains pending");
18271            assert!(
18272                commands.is_empty(),
18273                "recorded timer must not be rescheduled"
18274            );
18275        }
18276    }
18277
18278    #[test]
18279    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
18280        let client = Client::new("http://127.0.0.1:8080").expect("client");
18281        let mut worker = Worker::new(client, "rust-workers");
18282        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
18283            Ok(json!({"status": "completed"}))
18284        });
18285        let task = WorkflowTask {
18286            task_id: "wft-rust-timer-removed".to_string(),
18287            workflow_command_id: None,
18288            workflow_id: Some("wf-rust-timer".to_string()),
18289            run_id: Some("run-rust-timer".to_string()),
18290            workflow_type: "rust.timer.removed".to_string(),
18291            cancel_requested: false,
18292            payload_codec: DEFAULT_CODEC.to_string(),
18293            arguments: Some(
18294                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
18295            ),
18296            history_events: vec![
18297                history_event(
18298                    "TimerScheduled",
18299                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18300                ),
18301                history_event(
18302                    "TimerFired",
18303                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18304                ),
18305            ],
18306            total_history_events: Some(2),
18307            history_size_bytes: None,
18308            continue_as_new_recommended: None,
18309            history_budget_pressure: None,
18310            next_history_page_token: None,
18311            workflow_task_attempt: 1,
18312            workflow_signal_id: None,
18313            signal_name: None,
18314            signal_arguments: None,
18315            workflow_update_id: None,
18316            update_name: None,
18317            lease_owner: Some("rust-worker".to_string()),
18318        };
18319
18320        let Error::NonDeterministicReplay(failure) = worker
18321            .execute_workflow_task(task)
18322            .expect_err("removed timer must fail replay")
18323        else {
18324            panic!("expected typed replay failure");
18325        };
18326        assert_eq!(failure.reason, "recorded_commands_unconsumed");
18327        assert_eq!(failure.sequence, Some(1));
18328    }
18329
18330    #[test]
18331    fn workflow_context_emits_explicit_child_workflow_contract() {
18332        let ctx = WorkflowContext {
18333            state: Arc::new(Mutex::new(
18334                WorkflowState::new_with_identity(
18335                    Vec::new(),
18336                    Some("wf-parent".to_string()),
18337                    Some("run-parent".to_string()),
18338                    "parent-workers".to_string(),
18339                    DEFAULT_CODEC.to_string(),
18340                    None,
18341                )
18342                .expect("workflow state"),
18343            )),
18344        };
18345        let options = ChildWorkflowOptions::new("python-workers")
18346            .parent_close_policy(ParentClosePolicy::RequestCancel)
18347            .retry_policy(ChildWorkflowRetryPolicy {
18348                max_attempts: Some(3),
18349                backoff_seconds: vec![1, 5],
18350                non_retryable_error_types: vec!["ValidationError".to_string()],
18351            })
18352            .execution_timeout_seconds(600)
18353            .run_timeout_seconds(120);
18354        let mut call = Box::pin(ctx.start_child_workflow(
18355            "python.fulfil-order",
18356            options,
18357            json!([{"order_id": "order-42"}]),
18358        ));
18359        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18360
18361        assert!(matches!(
18362            call.as_mut().poll(&mut task_context),
18363            Poll::Pending
18364        ));
18365        let commands = ctx.take_commands().expect("commands");
18366        assert_eq!(commands.len(), 1);
18367        let command = &commands[0];
18368        assert_eq!(command["type"], "start_child_workflow");
18369        assert_eq!(command["workflow_type"], "python.fulfil-order");
18370        assert_eq!(command["queue"], "python-workers");
18371        assert_eq!(command["parent_close_policy"], "request_cancel");
18372        assert_eq!(command["retry_policy"]["max_attempts"], 3);
18373        assert_eq!(command["execution_timeout_seconds"], 600);
18374        assert_eq!(command["run_timeout_seconds"], 120);
18375        assert_eq!(
18376            decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
18377            json!([{"order_id": "order-42"}])
18378        );
18379    }
18380
18381    fn child_parent_worker() -> Worker {
18382        let client = Client::new("http://127.0.0.1:8080").expect("client");
18383        let mut worker = Worker::new(client, "rust-parent-workers");
18384        worker.register_workflow("rust.parent", |ctx, _input| async move {
18385            let child = ctx
18386                .start_child_workflow(
18387                    "python.child",
18388                    ChildWorkflowOptions::new("python-child-workers")
18389                        .parent_close_policy(ParentClosePolicy::Terminate),
18390                    json!([{"codec_probe": [1, true, "rust"]}]),
18391                )
18392                .await?;
18393            Ok(json!({
18394                "parent_workflow_id": child.parent.workflow_id,
18395                "parent_run_id": child.parent.run_id,
18396                "child_workflow_id": child.child.workflow_id,
18397                "child_run_id": child.child.run_id,
18398                "child_workflow_type": child.child_workflow_type,
18399                "result": child.result,
18400            }))
18401        });
18402        worker
18403    }
18404
18405    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
18406        WorkflowTask {
18407            task_id: "wft-child-parent".to_string(),
18408            workflow_command_id: None,
18409            workflow_id: Some("wf-parent".to_string()),
18410            run_id: Some("run-parent".to_string()),
18411            workflow_type: "rust.parent".to_string(),
18412            cancel_requested: false,
18413            payload_codec: DEFAULT_CODEC.to_string(),
18414            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
18415            history_events: vec![
18416                HistoryEvent {
18417                    event_type: "ChildWorkflowScheduled".to_string(),
18418                    payload: json!({
18419                        "sequence": 1,
18420                        "child_call_id": "call-child",
18421                        "child_workflow_instance_id": "wf-child",
18422                        "child_workflow_run_id": "run-child",
18423                        "child_workflow_type": "python.child",
18424                    }),
18425                    raw: HashMap::new(),
18426                },
18427                HistoryEvent {
18428                    event_type: event_type.to_string(),
18429                    payload,
18430                    raw: HashMap::new(),
18431                },
18432            ],
18433            total_history_events: Some(2),
18434            history_size_bytes: None,
18435            continue_as_new_recommended: None,
18436            history_budget_pressure: None,
18437            next_history_page_token: None,
18438            workflow_task_attempt: 1,
18439            workflow_signal_id: None,
18440            signal_name: None,
18441            signal_arguments: None,
18442            workflow_update_id: None,
18443            update_name: None,
18444            lease_owner: Some("rust-worker".to_string()),
18445        }
18446    }
18447
18448    #[test]
18449    fn committed_child_result_replays_without_starting_a_duplicate() {
18450        let worker = child_parent_worker();
18451        let task = child_parent_task(
18452            "ChildRunCompleted",
18453            json!({
18454                "sequence": 1,
18455                "child_call_id": "call-child",
18456                "child_workflow_instance_id": "wf-child",
18457                "child_workflow_run_id": "run-child",
18458                "child_workflow_type": "python.child",
18459                "payload_codec": DEFAULT_CODEC,
18460                "result": fixture_envelope(json!({"from":"python","ok":true})),
18461            }),
18462        );
18463
18464        for _restart in 0..2 {
18465            let commands = worker
18466                .execute_workflow_task(task.clone())
18467                .expect("replayed parent task");
18468            assert_eq!(commands.len(), 1);
18469            assert_eq!(commands[0]["type"], "complete_workflow");
18470            assert!(!commands
18471                .iter()
18472                .any(|command| command["type"] == "start_child_workflow"));
18473            let output =
18474                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
18475            assert_eq!(output["parent_workflow_id"], "wf-parent");
18476            assert_eq!(output["parent_run_id"], "run-parent");
18477            assert_eq!(output["child_workflow_id"], "wf-child");
18478            assert_eq!(output["child_run_id"], "run-child");
18479            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
18480        }
18481    }
18482
18483    #[test]
18484    fn typed_child_arguments_and_results_survive_replay() {
18485        let client = Client::new("http://127.0.0.1:8080").expect("client");
18486        let mut worker = Worker::new(client, "rust-parent-workers");
18487        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
18488            let child = ctx
18489                .start_child_workflow_avro_value(
18490                    "python.typed-child",
18491                    ChildWorkflowOptions::new("python-workers"),
18492                    AvroValue::Array(vec![typed_fidelity_probe()]),
18493                )
18494                .await?;
18495            Ok(child.result)
18496        });
18497
18498        let initial = worker
18499            .execute_workflow_task(workflow_task(
18500                "rust.typed-parent",
18501                Vec::new(),
18502                DEFAULT_CODEC,
18503            ))
18504            .expect("typed child start");
18505        assert_eq!(initial[0]["type"], "start_child_workflow");
18506        assert_eq!(
18507            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
18508                .expect("typed child arguments"),
18509            AvroValue::Array(vec![typed_fidelity_probe()])
18510        );
18511
18512        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
18513            .expect("typed child result");
18514        let task = workflow_task(
18515            "rust.typed-parent",
18516            vec![
18517                history_event(
18518                    "ChildWorkflowScheduled",
18519                    json!({
18520                        "sequence": 1,
18521                        "child_call_id": "call-typed",
18522                        "child_workflow_instance_id": "wf-child",
18523                        "child_workflow_run_id": "run-child",
18524                        "child_workflow_type": "python.typed-child",
18525                    }),
18526                ),
18527                history_event(
18528                    "ChildRunCompleted",
18529                    json!({
18530                        "sequence": 1,
18531                        "child_call_id": "call-typed",
18532                        "child_workflow_instance_id": "wf-child",
18533                        "child_workflow_run_id": "run-child",
18534                        "child_workflow_type": "python.typed-child",
18535                        "payload_codec": DEFAULT_CODEC,
18536                        "result": result,
18537                    }),
18538                ),
18539            ],
18540            DEFAULT_CODEC,
18541        );
18542
18543        let commands = worker
18544            .execute_workflow_task(task)
18545            .expect("typed child replay");
18546        assert_eq!(commands[0]["type"], "complete_workflow");
18547        assert_eq!(
18548            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
18549                .expect("typed parent result"),
18550            typed_fidelity_probe()
18551        );
18552    }
18553
18554    #[test]
18555    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
18556        let worker = child_parent_worker();
18557        let mut task = child_parent_task("unused", Value::Null);
18558        task.history_events.truncate(1);
18559        task.total_history_events = Some(1);
18560
18561        for _redelivery_or_restart in 0..2 {
18562            let commands = worker
18563                .execute_workflow_task(task.clone())
18564                .expect("recorded child remains pending");
18565            assert!(
18566                commands.is_empty(),
18567                "recorded pending child must not be started again"
18568            );
18569        }
18570    }
18571
18572    #[test]
18573    fn child_cancellation_becomes_stable_parent_failure_command() {
18574        let worker = child_parent_worker();
18575        let task = child_parent_task(
18576            "ChildRunCancelled",
18577            json!({
18578                "sequence": 1,
18579                "child_workflow_instance_id": "wf-child",
18580                "child_workflow_run_id": "run-child",
18581                "child_workflow_type": "python.child",
18582                "failure_id": "failure-child",
18583                "failure_category": "cancelled",
18584                "message": "cancelled by parent-close policy",
18585            }),
18586        );
18587
18588        let commands = worker
18589            .execute_workflow_task(task)
18590            .expect("parent settlement");
18591        assert_eq!(commands.len(), 1);
18592        assert_eq!(commands[0]["type"], "fail_workflow");
18593        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
18594        assert_eq!(
18595            commands[0]["exception"]["properties"]["reason"],
18596            "cancelled"
18597        );
18598        assert_eq!(
18599            commands[0]["exception"]["properties"]["child_workflow_run_id"],
18600            "run-child"
18601        );
18602    }
18603
18604    #[test]
18605    fn workflow_can_handle_typed_child_failure() {
18606        let client = Client::new("http://127.0.0.1:8080").expect("client");
18607        let mut worker = Worker::new(client, "rust-parent-workers");
18608        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
18609            match ctx
18610                .start_child_workflow(
18611                    "python.child",
18612                    ChildWorkflowOptions::new("python-child-workers"),
18613                    json!([]),
18614                )
18615                .await
18616            {
18617                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
18618                    "reason": failure.reason,
18619                    "failure_id": failure.failure_id,
18620                    "exception_class": failure.exception_class,
18621                    "child_run_id": failure.child_workflow_run_id,
18622                })),
18623                Err(error) => Err(error),
18624                Ok(_) => Err(Error::WorkerLoop(
18625                    "child unexpectedly succeeded".to_string(),
18626                )),
18627            }
18628        });
18629        let mut task = child_parent_task(
18630            "ChildRunFailed",
18631            json!({
18632                "sequence": 1,
18633                "child_workflow_instance_id": "wf-child",
18634                "child_workflow_run_id": "run-child",
18635                "child_workflow_type": "python.child",
18636                "failure_id": "failure-child",
18637                "failure_category": "child_workflow",
18638                "message": "payment rejected",
18639                "exception": {
18640                    "type": "PaymentRejected",
18641                    "class": "payments.PaymentRejected",
18642                    "message": "payment rejected"
18643                }
18644            }),
18645        );
18646        task.workflow_type = "rust.handled-parent".to_string();
18647
18648        let commands = worker.execute_workflow_task(task).expect("handled failure");
18649        assert_eq!(commands[0]["type"], "complete_workflow");
18650        let output =
18651            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
18652        assert_eq!(output["reason"], "child_workflow");
18653        assert_eq!(output["failure_id"], "failure-child");
18654        assert_eq!(output["exception_class"], "payments.PaymentRejected");
18655        assert_eq!(output["child_run_id"], "run-child");
18656    }
18657
18658    #[test]
18659    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
18660        let client = Client::new("http://127.0.0.1:8080").expect("client");
18661        let mut worker = Worker::new(client, "rust-workers");
18662
18663        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
18664            let signal = ctx.wait_signal("start").await?;
18665            let name = signal
18666                .first()
18667                .and_then(|value| value.as_str())
18668                .unwrap_or("world");
18669            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
18670            Ok(json!({
18671                "greeting": greeting,
18672                "language": "rust"
18673            }))
18674        });
18675
18676        let signal_arguments =
18677            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
18678        let task = WorkflowTask {
18679            task_id: "wft-rust-signal-1".to_string(),
18680            workflow_command_id: None,
18681            workflow_id: Some("wf-rust-hello".to_string()),
18682            run_id: Some("run-rust-hello".to_string()),
18683            workflow_type: "rust.hello_workflow".to_string(),
18684            cancel_requested: false,
18685            payload_codec: DEFAULT_CODEC.to_string(),
18686            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
18687            history_events: vec![HistoryEvent {
18688                event_type: "SignalReceived".to_string(),
18689                payload: json!({
18690                    "signal_id": "sig-rust-1",
18691                    "signal_name": "start"
18692                }),
18693                raw: HashMap::new(),
18694            }],
18695            total_history_events: Some(1),
18696            history_size_bytes: None,
18697            continue_as_new_recommended: None,
18698            history_budget_pressure: None,
18699            next_history_page_token: None,
18700            workflow_task_attempt: 1,
18701            workflow_signal_id: Some("sig-rust-1".to_string()),
18702            signal_name: Some("start".to_string()),
18703            signal_arguments: Some(signal_arguments),
18704            workflow_update_id: None,
18705            update_name: None,
18706            lease_owner: Some("rust-worker".to_string()),
18707        };
18708
18709        let commands = worker.execute_workflow_task(task).expect("workflow task");
18710
18711        assert_eq!(commands.len(), 1);
18712        assert_eq!(commands[0]["type"], "schedule_activity");
18713        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
18714        assert_eq!(
18715            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
18716            json!(["Rust"])
18717        );
18718    }
18719
18720    #[test]
18721    fn workflow_task_appends_paginated_history_events() {
18722        let mut task = WorkflowTask {
18723            task_id: "wft-rust-pages-1".to_string(),
18724            workflow_command_id: None,
18725            workflow_id: Some("wf-rust-pages".to_string()),
18726            run_id: Some("run-rust-pages".to_string()),
18727            workflow_type: "rust.hello_workflow".to_string(),
18728            cancel_requested: false,
18729            payload_codec: DEFAULT_CODEC.to_string(),
18730            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
18731            history_events: vec![HistoryEvent {
18732                event_type: "WorkflowStarted".to_string(),
18733                payload: json!({}),
18734                raw: HashMap::new(),
18735            }],
18736            total_history_events: Some(3),
18737            history_size_bytes: None,
18738            continue_as_new_recommended: None,
18739            history_budget_pressure: None,
18740            next_history_page_token: Some("MQ==".to_string()),
18741            workflow_task_attempt: 1,
18742            workflow_signal_id: None,
18743            signal_name: None,
18744            signal_arguments: None,
18745            workflow_update_id: None,
18746            update_name: None,
18747            lease_owner: Some("rust-worker".to_string()),
18748        };
18749
18750        task.append_history_page(WorkflowTaskHistoryPage {
18751            history_events: vec![
18752                HistoryEvent {
18753                    event_type: "SignalReceived".to_string(),
18754                    payload: json!({
18755                        "signal_id": "sig-rust-1",
18756                        "signal_name": "start",
18757                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
18758                            .expect("signal arguments")
18759                    }),
18760                    raw: HashMap::new(),
18761                },
18762                HistoryEvent {
18763                    event_type: "MarkerRecorded".to_string(),
18764                    payload: json!({"sequence": 3}),
18765                    raw: HashMap::new(),
18766                },
18767            ],
18768            total_history_events: Some(3),
18769            next_history_page_token: None,
18770        });
18771
18772        assert_eq!(task.history_events.len(), 3);
18773        assert_eq!(task.total_history_events, Some(3));
18774        assert_eq!(task.next_history_page_token, None);
18775
18776        let signal = task
18777            .history_events
18778            .iter()
18779            .find(|event| event.event_type == "SignalReceived")
18780            .expect("signal event");
18781        assert_eq!(
18782            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
18783            vec![AvroValue::String("Rust".to_string())]
18784        );
18785    }
18786
18787    #[tokio::test]
18788    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
18789        let client = Client::new("http://127.0.0.1:8080").expect("client");
18790        let mut worker = Worker::new(client, "rust-workers");
18791        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
18792        worker.register_query("counter", "current", |ctx, _args| async move {
18793            let mut count = 0_i64;
18794            for signal in ctx.signal_events() {
18795                let value = signal
18796                    .arguments
18797                    .first()
18798                    .and_then(Value::as_i64)
18799                    .unwrap_or_default();
18800                match signal.name.as_str() {
18801                    "increment" => count += value,
18802                    "set" => count = value,
18803                    _ => {}
18804                }
18805            }
18806            Ok(json!(count))
18807        });
18808
18809        let task = QueryTask {
18810            query_task_id: "query-rust-counter".to_string(),
18811            query_task_attempt: 1,
18812            lease_owner: Some("rust-worker".to_string()),
18813            workflow_id: Some("counter-1".to_string()),
18814            run_id: Some("run-counter-1".to_string()),
18815            workflow_type: "counter".to_string(),
18816            query_name: "current".to_string(),
18817            payload_codec: DEFAULT_CODEC.to_string(),
18818            workflow_arguments: Some(
18819                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
18820            ),
18821            query_arguments: Some(
18822                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
18823            ),
18824            history_events: vec![
18825                HistoryEvent {
18826                    event_type: "SignalReceived".to_string(),
18827                    payload: json!({
18828                        "signal_id": "php-signal-1",
18829                        "signal_name": "increment",
18830                        "workflow_sequence": 1,
18831                        "payload_codec": DEFAULT_CODEC,
18832                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
18833                    }),
18834                    raw: HashMap::new(),
18835                },
18836                HistoryEvent {
18837                    event_type: "SignalReceived".to_string(),
18838                    payload: json!({
18839                        "signal_id": "python-signal-2",
18840                        "signal_name": "increment",
18841                        "workflow_sequence": 2,
18842                        "payload_codec": DEFAULT_CODEC,
18843                        "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
18844                    }),
18845                    raw: HashMap::new(),
18846                },
18847                HistoryEvent {
18848                    event_type: "SignalReceived".to_string(),
18849                    payload: json!({
18850                        "signal_id": "rust-signal-3",
18851                        "signal_name": "set",
18852                        "workflow_sequence": 3,
18853                        "payload_codec": DEFAULT_CODEC,
18854                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
18855                    }),
18856                    raw: HashMap::new(),
18857                },
18858            ],
18859            history_export: None,
18860            run_status: Some("completed".to_string()),
18861        };
18862
18863        let result = worker.execute_query_task(task).await.expect("query result");
18864        assert_eq!(result.into_json().expect("query projection"), json!(0));
18865    }
18866
18867    #[tokio::test]
18868    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
18869        let worker = replay_counter_worker();
18870        let running_history = json!([
18871            {
18872                "type": "ActivityCompleted",
18873                "payload": {
18874                    "sequence": 1,
18875                    "activity_type": "load-counter",
18876                    "payload_codec": DEFAULT_CODEC,
18877                    "result": fixture_envelope(json!("loaded"))
18878                }
18879            },
18880            {
18881                "type": "SignalWaitOpened",
18882                "payload": {
18883                    "sequence": 3,
18884                    "signal_name": "increment"
18885                }
18886            },
18887            {
18888                "type": "SignalReceived",
18889                "payload": {
18890                    "signal_id": "signal-3",
18891                    "signal_name": "increment",
18892                    "workflow_sequence": 2,
18893                    "payload_codec": DEFAULT_CODEC,
18894                    "arguments": fixture_envelope(json!([3]))
18895                }
18896            },
18897            {
18898                "type": "SignalApplied",
18899                "payload": {
18900                    "sequence": 3,
18901                    "signal_id": "signal-3",
18902                    "signal_name": "increment",
18903                    "payload_codec": DEFAULT_CODEC,
18904                    "value": fixture_envelope(json!([3]))
18905                }
18906            }
18907        ]);
18908
18909        let running = worker
18910            .execute_query_task(replay_counter_query(
18911                "current",
18912                running_history.clone(),
18913                "running",
18914            ))
18915            .await
18916            .expect("running replay query");
18917        assert_eq!(
18918            running.clone().into_json().expect("query projection"),
18919            json!({"loaded": "loaded", "count": 3, "finished": false})
18920        );
18921
18922        let detached = worker
18923            .execute_query_task(replay_counter_query(
18924                "detached-mutation",
18925                running_history.clone(),
18926                "running",
18927            ))
18928            .await
18929            .expect("query mutates only its detached state clone");
18930        assert_eq!(detached.into_json().expect("query projection"), json!(999));
18931        let failed = worker
18932            .execute_query_task(replay_counter_query(
18933                "failed-mutation",
18934                running_history.clone(),
18935                "running",
18936            ))
18937            .await
18938            .expect_err("failed query");
18939        assert_eq!(failed.reason, "query_rejected");
18940        let unchanged = worker
18941            .execute_query_task(replay_counter_query("current", running_history, "running"))
18942            .await
18943            .expect("later query reconstructs unchanged state");
18944        assert_eq!(unchanged, running);
18945
18946        let restarted_worker = replay_counter_worker();
18947        let empty_arguments = fixture_envelope(json!([]));
18948        let loaded_result = fixture_envelope(json!("loaded"));
18949        let signal_three = fixture_blob(json!([3]));
18950        let signal_five = fixture_blob(json!([5]));
18951        let restarted_task: QueryTask = serde_json::from_value(json!({
18952            "query_task_id": "query-after-restart",
18953            "workflow_id": "counter-1",
18954            "run_id": "run-counter-1",
18955            "workflow_type": "replay-counter",
18956            "query_name": "current",
18957            "payload_codec": DEFAULT_CODEC,
18958            "workflow_arguments": empty_arguments.clone(),
18959            "query_arguments": empty_arguments,
18960            "history_events": [],
18961            "history_export": {
18962                "payloads": {"codec": DEFAULT_CODEC},
18963                "history_events": [
18964                    {
18965                        "type": "ActivityCompleted",
18966                        "payload": {
18967                            "sequence": 1,
18968                            "activity_type": "load-counter",
18969                            "payload_codec": DEFAULT_CODEC,
18970                            "result": null
18971                        }
18972                    },
18973                    {
18974                        "type": "SignalWaitOpened",
18975                        "payload": {
18976                            "sequence": 3,
18977                            "signal_name": "increment"
18978                        }
18979                    },
18980                    {
18981                        "type": "SignalReceived",
18982                        "payload": {
18983                            "signal_id": "signal-3",
18984                            "signal_name": "increment",
18985                            "workflow_sequence": 2
18986                        }
18987                    },
18988                    {
18989                        "type": "SignalApplied",
18990                        "payload": {
18991                            "sequence": 3,
18992                            "signal_id": "signal-3",
18993                            "signal_name": "increment"
18994                        }
18995                    },
18996                    {
18997                        "type": "SignalWaitOpened",
18998                        "payload": {
18999                            "sequence": 5,
19000                            "signal_name": "increment"
19001                        }
19002                    },
19003                    {
19004                        "type": "SignalReceived",
19005                        "payload": {
19006                            "signal_id": "signal-5",
19007                            "signal_name": "increment",
19008                            "workflow_sequence": 4
19009                        }
19010                    },
19011                    {
19012                        "type": "SignalApplied",
19013                        "payload": {
19014                            "sequence": 5,
19015                            "signal_id": "signal-5",
19016                            "signal_name": "increment"
19017                        }
19018                    }
19019                ],
19020                "activities": [{
19021                    "sequence": 1,
19022                    "activity_type": "load-counter",
19023                    "payload_codec": DEFAULT_CODEC,
19024                    "result": loaded_result
19025                }],
19026                "signals": [
19027                    {
19028                        "id": "signal-3",
19029                        "name": "increment",
19030                        "workflow_sequence": 2,
19031                        "payload_codec": DEFAULT_CODEC,
19032                        "arguments": signal_three
19033                    },
19034                    {
19035                        "id": "signal-5",
19036                        "name": "increment",
19037                        "workflow_sequence": 4,
19038                        "payload_codec": DEFAULT_CODEC,
19039                        "arguments": signal_five
19040                    }
19041                ]
19042            },
19043            "run_status": "completed"
19044        }))
19045        .expect("cold replay query task");
19046        let completed = restarted_worker
19047            .execute_query_task(restarted_task)
19048            .await
19049            .expect("completed cold replay query");
19050        assert_eq!(
19051            completed.into_json().expect("query projection"),
19052            json!({"loaded": "loaded", "count": 8, "finished": true})
19053        );
19054    }
19055
19056    #[tokio::test]
19057    async fn replayed_query_replay_failures_are_machine_readable() {
19058        let worker = replay_counter_worker();
19059        let task = replay_counter_query(
19060            "current",
19061            json!([{
19062                "type": "ActivityCompleted",
19063                "payload": {
19064                    "sequence": 1,
19065                    "payload_codec": DEFAULT_CODEC,
19066                    "result": {"codec": DEFAULT_CODEC, "blob": "{"}
19067                }
19068            }]),
19069            "running",
19070        );
19071        let failure = worker
19072            .execute_query_task(task)
19073            .await
19074            .expect_err("invalid replay history payload");
19075        assert_eq!(failure.reason, "query_payload_decode_failed");
19076        assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
19077        assert!(failure.message.contains("invalid_payload_framing"));
19078    }
19079
19080    #[tokio::test]
19081    async fn query_task_restores_compact_history_from_export() {
19082        let client = Client::new("http://127.0.0.1:8080").expect("client");
19083        let mut worker = Worker::new(client, "rust-workers");
19084        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19085        worker.register_query("counter", "current", |ctx, _args| async move {
19086            Ok(json!(ctx.signals("increment")[0][0]))
19087        });
19088        let empty_arguments = fixture_envelope(json!([]));
19089        let exported_signal = fixture_blob(json!([9]));
19090        let task: QueryTask = serde_json::from_value(json!({
19091            "query_task_id": "query-export",
19092            "workflow_type": "counter",
19093            "query_name": "current",
19094            "payload_codec": DEFAULT_CODEC,
19095            "workflow_arguments": empty_arguments.clone(),
19096            "query_arguments": empty_arguments,
19097            "history_events": [],
19098            "history_export": {
19099                "payloads": {"codec": DEFAULT_CODEC},
19100                "history_events": [{
19101                    "type": "SignalReceived",
19102                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
19103                }],
19104                "signals": [{
19105                    "id": "signal-export",
19106                    "name": "increment",
19107                    "status": "applied",
19108                    "workflow_sequence": 1,
19109                    "payload_codec": DEFAULT_CODEC,
19110                    "arguments": exported_signal
19111                }]
19112            }
19113        }))
19114        .expect("query task");
19115
19116        let result = worker.execute_query_task(task).await.expect("query result");
19117        assert_eq!(result.into_json().expect("query projection"), json!(9));
19118    }
19119
19120    #[tokio::test]
19121    async fn query_task_failures_have_stable_reasons() {
19122        let client = Client::new("http://127.0.0.1:8080").expect("client");
19123        let mut worker = Worker::new(client, "rust-workers");
19124        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19125        worker.register_query(
19126            "counter",
19127            "current",
19128            |_ctx, _args| async move { Ok(json!(0)) },
19129        );
19130
19131        let base_task = QueryTask {
19132            query_task_id: "query-errors".to_string(),
19133            query_task_attempt: 1,
19134            lease_owner: None,
19135            workflow_id: Some("counter-errors".to_string()),
19136            run_id: Some("run-errors".to_string()),
19137            workflow_type: "counter".to_string(),
19138            query_name: "missing".to_string(),
19139            payload_codec: DEFAULT_CODEC.to_string(),
19140            workflow_arguments: Some(fixture_envelope(json!([]))),
19141            query_arguments: Some(fixture_envelope(json!([]))),
19142            history_events: Vec::new(),
19143            history_export: None,
19144            run_status: Some("running".to_string()),
19145        };
19146
19147        let unknown = worker
19148            .execute_query_task(base_task.clone())
19149            .await
19150            .expect_err("unknown query");
19151        assert_eq!(unknown.reason, "rejected_unknown_query");
19152
19153        let mut malformed = base_task;
19154        malformed.query_name = "current".to_string();
19155        malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
19156        let malformed = worker
19157            .execute_query_task(malformed)
19158            .await
19159            .expect_err("malformed payload");
19160        assert_eq!(malformed.reason, "query_payload_decode_failed");
19161
19162        let client = Client::new("http://127.0.0.1:8080").expect("client");
19163        let mut unavailable_worker = Worker::new(client, "rust-workers");
19164        unavailable_worker
19165            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19166        let empty_arguments = fixture_envelope(json!([]));
19167        let unavailable_task: QueryTask = serde_json::from_value(json!({
19168            "query_task_id": "query-unavailable",
19169            "workflow_type": "counter",
19170            "query_name": "current",
19171            "payload_codec": DEFAULT_CODEC,
19172            "workflow_arguments": empty_arguments.clone(),
19173            "query_arguments": empty_arguments
19174        }))
19175        .expect("query task");
19176        let unavailable = unavailable_worker
19177            .execute_query_task(unavailable_task)
19178            .await
19179            .expect_err("query handler unavailable");
19180        assert_eq!(unavailable.reason, "query_handler_unavailable");
19181    }
19182
19183    #[tokio::test]
19184    async fn client_query_decodes_result_and_typed_failure() {
19185        let server = MockWorkerServer::start();
19186        let client = Client::builder(server.base_url())
19187            .timeout(Duration::from_secs(2))
19188            .build()
19189            .expect("client");
19190
19191        let result = client
19192            .query_workflow("counter-1", "current", json!([]))
19193            .await
19194            .expect("query result");
19195        assert_eq!(result, json!({"count": 8}));
19196
19197        let error = client
19198            .query_workflow("counter-1", "missing", json!([]))
19199            .await
19200            .expect_err("unknown query");
19201        let Error::QueryFailed(failure) = error else {
19202            panic!("expected typed query failure");
19203        };
19204        assert_eq!(failure.status, 404);
19205        assert_eq!(failure.reason, "rejected_unknown_query");
19206    }
19207
19208    #[tokio::test]
19209    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
19210        let server = MockWorkerServer::start();
19211        let client = Client::builder(server.base_url())
19212            .timeout(Duration::from_secs(2))
19213            .build()
19214            .expect("client");
19215        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
19216
19217        client
19218            .start_workflow(
19219                "typed.echo",
19220                "rust-workers",
19221                "typed-start",
19222                arguments.clone(),
19223            )
19224            .await
19225            .expect("typed workflow start");
19226        assert_eq!(
19227            decode_wire_avro_value(
19228                &server.request_body("/api/workflows")["input"],
19229                DEFAULT_CODEC,
19230            )
19231            .expect("typed start input"),
19232            arguments
19233        );
19234
19235        client
19236            .signal_workflow("typed-1", "changed", arguments.clone())
19237            .await
19238            .expect("typed signal");
19239        assert_eq!(
19240            decode_wire_avro_value(
19241                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
19242                DEFAULT_CODEC,
19243            )
19244            .expect("typed signal input"),
19245            arguments
19246        );
19247
19248        assert_eq!(
19249            client
19250                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
19251                .await
19252                .expect("typed query"),
19253            typed_fidelity_probe()
19254        );
19255        assert_eq!(
19256            decode_wire_avro_value(
19257                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
19258                DEFAULT_CODEC,
19259            )
19260            .expect("typed query input"),
19261            arguments
19262        );
19263
19264        assert_eq!(
19265            client
19266                .update_workflow_avro_value(
19267                    "typed-1",
19268                    "replace",
19269                    arguments.clone(),
19270                    Some("typed-request"),
19271                )
19272                .await
19273                .expect("typed update"),
19274            typed_fidelity_probe()
19275        );
19276        let update = server.request_body("/api/workflows/typed-1/update/replace");
19277        assert_eq!(update["request_id"], "typed-request");
19278        assert_eq!(
19279            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
19280            arguments
19281        );
19282
19283        let handle = WorkflowHandle {
19284            client: client.clone(),
19285            workflow_id: "typed-1".to_string(),
19286            run_id: Some("run-typed-1".to_string()),
19287            workflow_type: "typed.echo".to_string(),
19288        };
19289        assert_eq!(
19290            handle
19291                .result_avro_value(WorkflowResultOptions::default())
19292                .await
19293                .expect("typed workflow result"),
19294            typed_fidelity_probe()
19295        );
19296
19297        client
19298            .complete_activity_task(
19299                "activity-typed",
19300                "attempt-typed",
19301                "rust-worker",
19302                typed_fidelity_probe(),
19303                DEFAULT_CODEC,
19304            )
19305            .await
19306            .expect("typed activity completion");
19307        assert_eq!(
19308            decode_wire_avro_value(
19309                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
19310                    ["result"],
19311                DEFAULT_CODEC,
19312            )
19313            .expect("typed activity result"),
19314            typed_fidelity_probe()
19315        );
19316        client
19317            .fail_activity_task(
19318                "activity-typed",
19319                "attempt-typed",
19320                "rust-worker",
19321                "typed failure",
19322                true,
19323            )
19324            .await
19325            .expect("activity failure");
19326    }
19327
19328    #[tokio::test]
19329    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
19330        let server = MockWorkerServer::start();
19331        let client = Client::builder(server.base_url())
19332            .timeout(Duration::from_secs(2))
19333            .build()
19334            .expect("client");
19335
19336        let options = WorkflowCommandOptions::new()
19337            .reason("cleanup requested")
19338            .request_id("cancel-17");
19339        let cancelled = client
19340            .cancel_workflow("wf-lifecycle", options)
19341            .await
19342            .expect("instance cancellation");
19343        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
19344        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
19345        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
19346        assert_eq!(
19347            server.request_body("/api/workflows/wf-lifecycle/cancel"),
19348            json!({"reason":"cleanup requested","request_id":"cancel-17"})
19349        );
19350
19351        let terminated = client
19352            .terminate_workflow(
19353                "wf-lifecycle",
19354                WorkflowCommandOptions::new().reason("forced stop"),
19355            )
19356            .await
19357            .expect("instance termination");
19358        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
19359        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
19360
19361        client
19362            .cancel_workflow_run(
19363                "wf-lifecycle",
19364                "run-current",
19365                WorkflowCommandOptions::default(),
19366            )
19367            .await
19368            .expect("selected run cancellation");
19369        client
19370            .terminate_workflow_run(
19371                "wf-lifecycle",
19372                "run-current",
19373                WorkflowCommandOptions::default(),
19374            )
19375            .await
19376            .expect("selected run termination");
19377
19378        for (command, error) in [
19379            (
19380                WorkflowCommandKind::Cancel,
19381                client
19382                    .cancel_workflow_run(
19383                        "wf-lifecycle",
19384                        "run-stale",
19385                        WorkflowCommandOptions::default(),
19386                    )
19387                    .await
19388                    .expect_err("stale cancellation must be rejected"),
19389            ),
19390            (
19391                WorkflowCommandKind::Terminate,
19392                client
19393                    .terminate_workflow_run(
19394                        "wf-lifecycle",
19395                        "run-stale",
19396                        WorkflowCommandOptions::default(),
19397                    )
19398                    .await
19399                    .expect_err("stale termination must be rejected"),
19400            ),
19401        ] {
19402            let Error::WorkflowCommandRejected(rejection) = error else {
19403                panic!("expected typed command rejection");
19404            };
19405            assert_eq!(rejection.command, command);
19406            assert_eq!(rejection.status, 409);
19407            assert_eq!(rejection.reason, "historical_run_command_rejected");
19408            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
19409            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
19410        }
19411    }
19412
19413    #[tokio::test]
19414    async fn workflow_start_options_send_server_enforced_deadlines() {
19415        let server = MockWorkerServer::start();
19416        let client = Client::builder(server.base_url())
19417            .timeout(Duration::from_secs(2))
19418            .build()
19419            .expect("client");
19420
19421        let handle = client
19422            .start_workflow_with_options(
19423                "rust.timeout",
19424                "rust-timeouts",
19425                "wf-start-options",
19426                WorkflowStartOptions::new()
19427                    .execution_timeout_seconds(30)
19428                    .run_timeout_seconds(1),
19429                json!([]),
19430            )
19431            .await
19432            .expect("workflow start");
19433
19434        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
19435        let body = server.request_body("/api/workflows");
19436        assert_eq!(body["execution_timeout_seconds"], 30);
19437        assert_eq!(body["run_timeout_seconds"], 1);
19438
19439        let invalid = client
19440            .start_workflow_with_options(
19441                "rust.timeout",
19442                "rust-timeouts",
19443                "wf-invalid-options",
19444                WorkflowStartOptions::new()
19445                    .execution_timeout_seconds(1)
19446                    .run_timeout_seconds(2),
19447                json!([]),
19448            )
19449            .await
19450            .expect_err("invalid deadline ordering");
19451        assert!(invalid
19452            .to_string()
19453            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
19454    }
19455
19456    #[tokio::test]
19457    async fn workflow_result_returns_each_typed_terminal_outcome() {
19458        let server = MockWorkerServer::start();
19459        let client = Client::builder(server.base_url())
19460            .timeout(Duration::from_secs(2))
19461            .build()
19462            .expect("client");
19463        let options = WorkflowResultOptions {
19464            poll_interval: Duration::ZERO,
19465            timeout: Duration::from_secs(1),
19466        };
19467
19468        let failed = WorkflowHandle {
19469            client: client.clone(),
19470            workflow_id: "wf-failed".to_string(),
19471            run_id: Some("run-failed".to_string()),
19472            workflow_type: "failure".to_string(),
19473        }
19474        .result(options)
19475        .await
19476        .expect_err("failed outcome");
19477        let Error::WorkflowFailed(failure) = failed else {
19478            panic!("expected WorkflowFailed");
19479        };
19480        assert_eq!(failure.workflow_id, "wf-failed");
19481        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
19482        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
19483        assert_eq!(failure.failure_category.as_deref(), Some("application"));
19484        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
19485        assert_eq!(
19486            failure.exception_class.as_deref(),
19487            Some("billing::PaymentError")
19488        );
19489        assert_eq!(failure.non_retryable, Some(true));
19490
19491        for (workflow_id, expected_kind, expected_reason) in [
19492            (
19493                "wf-cancelled",
19494                WorkflowTerminalKind::Cancelled,
19495                "cleanup requested",
19496            ),
19497            (
19498                "wf-terminated",
19499                WorkflowTerminalKind::Terminated,
19500                "forced stop",
19501            ),
19502            (
19503                "wf-timed-out",
19504                WorkflowTerminalKind::TimedOut,
19505                "run_timeout",
19506            ),
19507        ] {
19508            let error = WorkflowHandle {
19509                client: client.clone(),
19510                workflow_id: workflow_id.to_string(),
19511                run_id: None,
19512                workflow_type: "terminal".to_string(),
19513            }
19514            .result(options)
19515            .await
19516            .expect_err("typed terminal outcome");
19517            let outcome = match error {
19518                Error::WorkflowCancelled(outcome) => outcome,
19519                Error::WorkflowTerminated(outcome) => outcome,
19520                Error::WorkflowTimedOut(outcome) => outcome,
19521                other => panic!("unexpected terminal error: {other}"),
19522            };
19523            assert_eq!(outcome.kind, expected_kind);
19524            assert_eq!(outcome.workflow_id, workflow_id);
19525            assert_eq!(outcome.reason, expected_reason);
19526        }
19527
19528        let wait_timeout = WorkflowHandle {
19529            client,
19530            workflow_id: "wf-waiting".to_string(),
19531            run_id: Some("run-waiting".to_string()),
19532            workflow_type: "waiting".to_string(),
19533        }
19534        .result(WorkflowResultOptions {
19535            poll_interval: Duration::ZERO,
19536            timeout: Duration::ZERO,
19537        })
19538        .await
19539        .expect_err("client wait timeout");
19540        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
19541            panic!("expected typed client timeout");
19542        };
19543        assert_eq!(timeout.reason, "result_wait_timeout");
19544        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
19545        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
19546    }
19547
19548    #[tokio::test]
19549    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
19550        let server = MockWorkerServer::start();
19551        let client = Client::builder(server.base_url())
19552            .timeout(Duration::from_secs(2))
19553            .build()
19554            .expect("client");
19555
19556        let handle = WorkflowHandle {
19557            client,
19558            workflow_id: "wf-selected".to_string(),
19559            run_id: Some("run-selected".to_string()),
19560            workflow_type: "selected".to_string(),
19561        };
19562        let options = WorkflowResultOptions {
19563            poll_interval: Duration::ZERO,
19564            timeout: Duration::from_secs(1),
19565        };
19566
19567        let current = handle
19568            .result(options)
19569            .await
19570            .expect("instance result follows the current run");
19571        assert_eq!(current, json!("current run output"));
19572
19573        let error = handle
19574            .result_selected_run(options)
19575            .await
19576            .expect_err("the selected run is cancelled even though the current run completed");
19577
19578        let Error::WorkflowCancelled(outcome) = error else {
19579            panic!("expected selected run cancellation");
19580        };
19581        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
19582        assert_eq!(outcome.reason, "selected run cancelled");
19583        assert_eq!(
19584            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
19585            1
19586        );
19587        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
19588    }
19589
19590    #[tokio::test]
19591    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
19592        let server = MockWorkerServer::draining_polls();
19593        let client = Client::builder(server.base_url())
19594            .timeout(Duration::from_secs(2))
19595            .build()
19596            .expect("client");
19597
19598        let workflow = client
19599            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
19600            .await
19601            .expect("workflow drain response");
19602        let activity = client
19603            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
19604            .await
19605            .expect("activity drain response");
19606        let query = client
19607            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
19608            .await
19609            .expect("query drain response");
19610
19611        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
19612            assert_eq!(
19613                outcome,
19614                WorkerPollOutcome::Stop {
19615                    poll_status: Some("draining".to_string()),
19616                    reason: Some("worker_draining".to_string()),
19617                }
19618            );
19619        }
19620
19621        assert!(client
19622            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
19623            .await
19624            .expect("compatibility poll")
19625            .is_none());
19626    }
19627
19628    #[tokio::test]
19629    async fn managed_worker_honors_drain_stop_for_every_task_family() {
19630        let server = MockWorkerServer::draining_polls();
19631        let client = Client::builder(server.base_url())
19632            .timeout(Duration::from_secs(2))
19633            .build()
19634            .expect("client");
19635
19636        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
19637            .worker_id("draining-workflow-worker")
19638            .poll_timeout(Duration::ZERO);
19639        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
19640        workflow_worker
19641            .run()
19642            .await
19643            .expect("workflow drain is a clean stop");
19644
19645        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
19646            .worker_id("draining-activity-worker")
19647            .poll_timeout(Duration::ZERO);
19648        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
19649        activity_worker
19650            .run()
19651            .await
19652            .expect("activity drain is a clean stop");
19653
19654        let mut query_worker = Worker::new(client, "rust-workers")
19655            .worker_id("draining-query-worker")
19656            .poll_timeout(Duration::ZERO);
19657        query_worker.register_query("counter", "current", |_ctx, _args| async {
19658            Ok(Value::Null)
19659        });
19660        query_worker
19661            .run()
19662            .await
19663            .expect("query drain is a clean stop");
19664    }
19665
19666    #[tokio::test]
19667    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
19668        let server = MockWorkerServer::start();
19669        let client = Client::builder(server.base_url())
19670            .timeout(Duration::from_secs(2))
19671            .build()
19672            .expect("client");
19673
19674        let heartbeat = client
19675            .heartbeat_activity_task(
19676                "activity-cancel",
19677                "attempt-cancel",
19678                "rust-worker",
19679                typed_fidelity_probe(),
19680            )
19681            .await
19682            .expect("cancellation heartbeat");
19683        assert!(heartbeat.cancel_requested);
19684        assert!(heartbeat.should_stop());
19685        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
19686        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
19687        let heartbeat_body =
19688            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
19689        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
19690        assert_eq!(
19691            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
19692                .expect("typed heartbeat details"),
19693            typed_fidelity_probe()
19694        );
19695
19696        let error = client
19697            .complete_activity_task(
19698                "activity-cancel",
19699                "attempt-cancel",
19700                "rust-worker",
19701                json!({"late":true}),
19702                DEFAULT_CODEC,
19703            )
19704            .await
19705            .expect_err("late completion must be refused");
19706        assert!(activity_task_rejection_is_final(&error));
19707        let Error::ActivityTaskRejected(rejection) = error else {
19708            panic!("expected typed activity rejection");
19709        };
19710        assert_eq!(rejection.status, 409);
19711        assert_eq!(rejection.reason, "run_cancelled");
19712        assert!(rejection.cancel_requested);
19713        assert_eq!(rejection.can_continue, Some(false));
19714    }
19715
19716    #[tokio::test]
19717    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
19718        let server = MockWorkerServer::cancelled_activity();
19719        let client = Client::builder(server.base_url())
19720            .timeout(Duration::from_secs(2))
19721            .build()
19722            .expect("client");
19723        let cancellation_observed = Arc::new(AtomicBool::new(false));
19724        let observed = Arc::clone(&cancellation_observed);
19725        let mut worker = Worker::new(client.clone(), "rust-workers")
19726            .worker_id("rust-cancel-worker")
19727            .poll_timeout(Duration::from_millis(10));
19728        worker.register_activity("cancel-aware", move |ctx, _args| {
19729            let observed = Arc::clone(&observed);
19730            async move {
19731                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
19732                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
19733                Ok(json!({"late":"completion"}))
19734            }
19735        });
19736
19737        assert_eq!(
19738            worker.run_once().await.expect("cancelled attempt handled"),
19739            1
19740        );
19741        assert!(cancellation_observed.load(Ordering::SeqCst));
19742        assert_eq!(
19743            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
19744            1
19745        );
19746
19747        let mut restarted = Worker::new(client, "rust-workers")
19748            .worker_id("rust-cancel-worker-restarted")
19749            .poll_timeout(Duration::from_millis(10));
19750        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
19751        assert_eq!(
19752            restarted
19753                .run_once()
19754                .await
19755                .expect("replacement worker continues polling"),
19756            0
19757        );
19758    }
19759
19760    #[tokio::test]
19761    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
19762        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
19763        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
19764        let client = Client::builder(server.base_url())
19765            .timeout(Duration::from_secs(2))
19766            .build()
19767            .expect("client");
19768
19769        let direct_error = client
19770            .complete_workflow_task(
19771                "workflow-timeout-task",
19772                "timeout-worker",
19773                3,
19774                vec![json!({
19775                    "type": "complete_workflow",
19776                    "result": fixture_envelope(Value::Null)
19777                })],
19778            )
19779            .await
19780            .expect_err("the low-level client preserves the completion rejection");
19781        let Error::Http { status, body } = direct_error else {
19782            panic!("expected the original HTTP completion rejection");
19783        };
19784        assert_eq!(status, reqwest::StatusCode::CONFLICT);
19785        assert_eq!(
19786            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
19787            "run_timed_out"
19788        );
19789
19790        let mut worker = Worker::new(client, "rust-workers")
19791            .worker_id("timeout-worker")
19792            .poll_timeout(Duration::from_millis(10));
19793        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
19794            Ok(json!({"late": "result"}))
19795        });
19796
19797        assert_eq!(
19798            worker
19799                .run_once()
19800                .await
19801                .expect("authoritative selected-run timeout settles the tick"),
19802            1
19803        );
19804        assert_eq!(
19805            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
19806            2,
19807            "both the direct client proof and managed worker must see the rejection"
19808        );
19809    }
19810
19811    #[tokio::test]
19812    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
19813        for (name, status, response) in [
19814            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
19815            (
19816                "command was recorded",
19817                "409 Conflict",
19818                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
19819            ),
19820            (
19821                "lease conflict",
19822                "409 Conflict",
19823                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
19824            ),
19825            (
19826                "nonterminal run",
19827                "409 Conflict",
19828                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
19829            ),
19830            (
19831                "different selected run",
19832                "409 Conflict",
19833                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
19834            ),
19835            (
19836                "different task attempt",
19837                "409 Conflict",
19838                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
19839            ),
19840            (
19841                "authentication failure",
19842                "401 Unauthorized",
19843                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
19844            ),
19845            (
19846                "authorization failure",
19847                "403 Forbidden",
19848                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
19849            ),
19850            (
19851                "protocol failure",
19852                "400 Bad Request",
19853                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
19854            ),
19855            (
19856                "malformed command",
19857                "422 Unprocessable Entity",
19858                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
19859            ),
19860            (
19861                "transient server failure",
19862                "503 Service Unavailable",
19863                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
19864            ),
19865        ] {
19866            let server = MockWorkerServer::workflow_completion(status, response);
19867            let client = Client::builder(server.base_url())
19868                .timeout(Duration::from_secs(2))
19869                .build()
19870                .expect("client");
19871            let mut worker = Worker::new(client, "rust-workers")
19872                .worker_id("timeout-worker")
19873                .poll_timeout(Duration::from_millis(10));
19874            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
19875                Ok(json!({"late": "result"}))
19876            });
19877
19878            let error = worker
19879                .run_once()
19880                .await
19881                .expect_err(&format!("{name} must remain an error"));
19882            assert!(
19883                matches!(error, Error::Http { .. } | Error::Protocol(_)),
19884                "{name} returned an unexpected error variant: {error}"
19885            );
19886        }
19887    }
19888
19889    #[tokio::test]
19890    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
19891        let server = MockWorkerServer::start();
19892        let client = Client::builder(server.base_url())
19893            .worker_token(Some("worker-secret".to_string()))
19894            .namespace("orders")
19895            .timeout(Duration::from_secs(2))
19896            .build()
19897            .expect("client");
19898        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
19899
19900        let result = client
19901            .deregister_worker_registration("worker/α space")
19902            .await
19903            .expect("deregister worker registration");
19904
19905        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
19906        assert_eq!(
19907            server.worker_protocol_for(path).as_deref(),
19908            Some(WORKER_PROTOCOL_VERSION)
19909        );
19910        assert_eq!(server.control_protocol_for(path), None);
19911        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
19912        assert_eq!(
19913            server.authorization_for(path).as_deref(),
19914            Some("Bearer worker-secret")
19915        );
19916        assert_eq!(
19917            result,
19918            WorkerDeregistrationEnvelope {
19919                worker_id: "deregistered-worker".to_string(),
19920                outcome: "deregistered".to_string(),
19921                recovered_workflow_task_count: 2,
19922            }
19923        );
19924    }
19925
19926    #[tokio::test]
19927    async fn low_level_registration_rejects_update_validators_before_transport() {
19928        let server = MockWorkerServer::start();
19929        let client = Client::builder(server.base_url())
19930            .timeout(Duration::from_secs(2))
19931            .build()
19932            .expect("client");
19933
19934        for update_validators in [json!(["approve"]), json!("approve")] {
19935            let error = client
19936                .register_worker_with_command_contracts(
19937                    "validator-claiming-worker",
19938                    "rust-workers",
19939                    vec!["orders".to_string()],
19940                    vec![],
19941                    1,
19942                    1,
19943                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
19944                    json!({
19945                        "orders": {
19946                            "queries": ["current"],
19947                            "updates": ["approve"],
19948                            "update_validators": update_validators,
19949                        },
19950                    }),
19951                )
19952                .await
19953                .expect_err("unsupported validator claims must fail before registration");
19954
19955            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
19956                panic!("expected typed unsupported-validator failure");
19957            };
19958            assert_eq!(workflow_type, "orders");
19959        }
19960        assert_eq!(server.request_count("/api/worker/register"), 0);
19961    }
19962
19963    #[tokio::test]
19964    async fn low_level_registration_preserves_query_and_update_contracts() {
19965        let server = MockWorkerServer::start();
19966        let client = Client::builder(server.base_url())
19967            .timeout(Duration::from_secs(2))
19968            .build()
19969            .expect("client");
19970        let contracts = json!({
19971            "orders": {
19972                "queries": ["current"],
19973                "updates": ["approve"],
19974                "update_validators": [],
19975            },
19976            "payments": {
19977                "queries": ["status"],
19978                "updates": ["capture"],
19979            },
19980        });
19981
19982        client
19983            .register_worker_with_command_contracts(
19984                "command-worker",
19985                "rust-workers",
19986                vec!["orders".to_string(), "payments".to_string()],
19987                vec![],
19988                1,
19989                1,
19990                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
19991                contracts.clone(),
19992            )
19993            .await
19994            .expect("query and update contracts must remain supported");
19995
19996        assert_eq!(
19997            server.request_body("/api/worker/register")["workflow_command_contracts"],
19998            contracts
19999        );
20000    }
20001
20002    #[tokio::test]
20003    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
20004        let server = MockWorkerServer::start();
20005        let control_only = Client::builder(server.base_url())
20006            .control_token(Some("control-secret".to_string()))
20007            .build()
20008            .expect("control client");
20009
20010        let error = control_only
20011            .register_worker("worker", "queue", vec![], vec![], 1, 1)
20012            .await
20013            .expect_err("control token must not authorize a worker request");
20014        assert!(matches!(
20015            error,
20016            Error::MissingRoleCredentials { role: "worker", .. }
20017        ));
20018        assert_eq!(server.request_count("/api/worker/register"), 0);
20019
20020        let worker_only = Client::builder(server.base_url())
20021            .worker_token(Some("worker-secret".to_string()))
20022            .build()
20023            .expect("worker client");
20024        let error = worker_only
20025            .health()
20026            .await
20027            .expect_err("worker token must not authorize a control request");
20028        assert!(matches!(
20029            error,
20030            Error::MissingRoleCredentials {
20031                role: "control",
20032                ..
20033            }
20034        ));
20035        assert_eq!(server.request_count("/api/health"), 0);
20036    }
20037
20038    #[tokio::test]
20039    async fn shared_token_supports_worker_and_control_planes() {
20040        let server = MockWorkerServer::start();
20041        let client = Client::builder(server.base_url())
20042            .token(Some("shared-secret".to_string()))
20043            .build()
20044            .expect("client");
20045
20046        client.health().await.expect("control request");
20047        client
20048            .register_worker("worker", "queue", vec![], vec![], 1, 1)
20049            .await
20050            .expect("worker request");
20051
20052        assert_eq!(
20053            server.authorization_for("/api/health").as_deref(),
20054            Some("Bearer shared-secret")
20055        );
20056        assert_eq!(
20057            server.control_protocol_for("/api/health").as_deref(),
20058            Some(CONTROL_PLANE_VERSION)
20059        );
20060        assert_eq!(
20061            server.authorization_for("/api/worker/register").as_deref(),
20062            Some("Bearer shared-secret")
20063        );
20064        assert_eq!(
20065            server
20066                .worker_protocol_for("/api/worker/register")
20067                .as_deref(),
20068            Some(WORKER_PROTOCOL_VERSION)
20069        );
20070    }
20071
20072    #[tokio::test]
20073    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
20074        let server = MockWorkerServer::start();
20075        let client = Client::builder(server.base_url())
20076            .timeout(Duration::from_secs(2))
20077            .build()
20078            .expect("client");
20079
20080        client
20081            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
20082            .await
20083            .expect("register");
20084        client
20085            .heartbeat_worker("capture-worker", 1, 1)
20086            .await
20087            .expect("heartbeat");
20088        client
20089            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
20090            .await
20091            .expect("workflow poll");
20092        client
20093            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
20094            .await
20095            .expect("activity poll");
20096
20097        for path in [
20098            "/api/worker/register",
20099            "/api/worker/heartbeat",
20100            "/api/worker/workflow-tasks/poll",
20101            "/api/worker/activity-tasks/poll",
20102        ] {
20103            assert_eq!(
20104                server.worker_protocol_for(path).as_deref(),
20105                Some(WORKER_PROTOCOL_VERSION),
20106                "unexpected protocol for {path}"
20107            );
20108        }
20109
20110        assert_eq!(
20111            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
20112            1
20113        );
20114        assert_eq!(
20115            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
20116            1
20117        );
20118        assert!(
20119            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
20120                .as_str()
20121                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
20122        );
20123        assert!(
20124            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
20125                .as_str()
20126                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
20127        );
20128    }
20129
20130    #[tokio::test]
20131    async fn query_task_endpoints_send_the_query_feature_protocol() {
20132        let server = MockWorkerServer::start();
20133        let client = Client::builder(server.base_url())
20134            .timeout(Duration::from_secs(2))
20135            .build()
20136            .expect("client");
20137
20138        client
20139            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
20140            .await
20141            .expect("query poll");
20142        client
20143            .complete_query_task(
20144                "query-capture",
20145                "capture-worker",
20146                1,
20147                json!(8),
20148                DEFAULT_CODEC,
20149            )
20150            .await
20151            .expect("query complete");
20152        client
20153            .fail_query_task(
20154                "query-capture",
20155                "capture-worker",
20156                1,
20157                "failed",
20158                "query_rejected",
20159                "QueryFailed",
20160            )
20161            .await
20162            .expect("query fail");
20163
20164        for path in [
20165            "/api/worker/query-tasks/poll",
20166            "/api/worker/query-tasks/query-capture/complete",
20167            "/api/worker/query-tasks/query-capture/fail",
20168        ] {
20169            assert_eq!(
20170                server.worker_protocol_for(path).as_deref(),
20171                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
20172                "unexpected protocol for {path}"
20173            );
20174        }
20175
20176        assert_eq!(
20177            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
20178            1
20179        );
20180        assert!(
20181            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
20182                .as_str()
20183                .is_some_and(|id| id.starts_with("rust-query-poll-"))
20184        );
20185    }
20186
20187    #[tokio::test]
20188    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
20189        let server = MockWorkerServer::transient_worker_failures();
20190        let client = Client::builder(server.base_url())
20191            .timeout(Duration::from_secs(2))
20192            .build()
20193            .expect("client");
20194
20195        client
20196            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
20197            .await
20198            .expect("workflow poll retry");
20199        client
20200            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
20201            .await
20202            .expect("activity poll retry");
20203        client
20204            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
20205            .await
20206            .expect("query poll retry");
20207
20208        for path in [
20209            "/api/worker/workflow-tasks/poll",
20210            "/api/worker/activity-tasks/poll",
20211            "/api/worker/query-tasks/poll",
20212        ] {
20213            let bodies = server.request_bodies(path);
20214            assert_eq!(bodies.len(), 2, "{path} must be retried once");
20215            assert_eq!(
20216                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
20217                "{path} must preserve the request binding across retry"
20218            );
20219        }
20220    }
20221
20222    #[tokio::test]
20223    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
20224        let server = MockWorkerServer::consecutive_poll_failures(2);
20225        let client = Client::builder(server.base_url())
20226            .timeout(Duration::from_secs(2))
20227            .build()
20228            .expect("client");
20229        let mut worker = Worker::new(client, "capture")
20230            .worker_id("capture-worker")
20231            .poll_timeout(Duration::from_millis(10))
20232            .retry_policy(WorkerRetryPolicy {
20233                max_retries: 2,
20234                initial_backoff: Duration::from_millis(1),
20235                max_backoff: Duration::from_millis(1),
20236            });
20237        worker.register_workflow(
20238            "capture.workflow",
20239            |_ctx, _input| async move { Ok(Value::Null) },
20240        );
20241        worker.register_activity(
20242            "capture.activity",
20243            |_ctx, _input| async move { Ok(Value::Null) },
20244        );
20245        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
20246            Ok(Value::Null)
20247        });
20248
20249        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
20250
20251        for path in [
20252            "/api/worker/workflow-tasks/poll",
20253            "/api/worker/activity-tasks/poll",
20254            "/api/worker/query-tasks/poll",
20255        ] {
20256            let bodies = server.request_bodies(path);
20257            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
20258            assert!(
20259                bodies
20260                    .iter()
20261                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
20262                "{path} must preserve one request binding across every retry"
20263            );
20264        }
20265    }
20266
20267    #[tokio::test]
20268    async fn query_protocol_rejection_from_older_server_is_typed() {
20269        let server = MockWorkerServer::reject_query_protocol();
20270        let client = Client::builder(server.base_url())
20271            .timeout(Duration::from_secs(2))
20272            .build()
20273            .expect("client");
20274
20275        let error = client
20276            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
20277            .await
20278            .expect_err("server below query protocol floor must reject");
20279        let Error::Protocol(failure) = error else {
20280            panic!("expected typed protocol failure");
20281        };
20282
20283        assert_eq!(failure.status, 400);
20284        assert_eq!(failure.reason, "unsupported_protocol_version");
20285        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
20286        assert_eq!(
20287            failure.requested_version.as_deref(),
20288            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
20289        );
20290        assert_eq!(
20291            server
20292                .worker_protocol_for("/api/worker/query-tasks/poll")
20293                .as_deref(),
20294            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
20295        );
20296    }
20297
20298    #[tokio::test]
20299    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
20300        let server = MockWorkerServer::reject_query_protocol();
20301        let client = Client::builder(server.base_url())
20302            .timeout(Duration::from_secs(2))
20303            .build()
20304            .expect("client");
20305        let mut worker = Worker::new(client, "rust-workers")
20306            .worker_id("baseline-worker")
20307            .poll_timeout(Duration::from_millis(10));
20308
20309        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
20310            Ok(Value::Null)
20311        });
20312
20313        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
20314        assert_eq!(
20315            server
20316                .worker_protocol_for("/api/worker/workflow-tasks/poll")
20317                .as_deref(),
20318            Some(WORKER_PROTOCOL_VERSION)
20319        );
20320        assert_eq!(
20321            server.worker_protocol_for("/api/worker/query-tasks/poll"),
20322            None,
20323            "a worker without query handlers must not use the query-task endpoint"
20324        );
20325    }
20326
20327    #[tokio::test]
20328    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
20329        let server = MockWorkerServer::reject_query_completion();
20330        let client = Client::builder(server.base_url())
20331            .timeout(Duration::from_secs(2))
20332            .build()
20333            .expect("client");
20334
20335        let error = client
20336            .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
20337            .await
20338            .expect_err("expired completion must be rejected");
20339        let Error::QueryFailed(failure) = error else {
20340            panic!("expected typed query failure");
20341        };
20342        assert_eq!(failure.status, 409);
20343        assert_eq!(failure.reason, "query_task_timed_out");
20344
20345        let mut worker = Worker::new(client, "rust-workers")
20346            .worker_id("late-worker")
20347            .poll_timeout(Duration::from_millis(10));
20348        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
20349        worker.register_query(
20350            "counter",
20351            "current",
20352            |_ctx, _args| async move { Ok(json!(8)) },
20353        );
20354
20355        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
20356        assert_eq!(
20357            worker
20358                .run_once()
20359                .await
20360                .expect("worker continues after late completion"),
20361            0
20362        );
20363        assert_eq!(
20364            server.request_count("/api/worker/query-tasks/query-late/complete"),
20365            2
20366        );
20367        assert_eq!(
20368            server.request_count("/api/worker/query-tasks/query-late/fail"),
20369            0,
20370            "a server completion rejection must not be reported as an encoding failure"
20371        );
20372    }
20373
20374    #[tokio::test]
20375    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
20376        let server = MockWorkerServer::start();
20377        let client = Client::builder(server.base_url())
20378            .timeout(Duration::from_secs(2))
20379            .build()
20380            .expect("client");
20381        let mut worker = Worker::new(client, "rust-workers")
20382            .worker_id("joined-worker")
20383            .poll_timeout(Duration::from_millis(10));
20384        worker.register_workflow(
20385            "joined.workflow",
20386            |_ctx, _input| async move { Ok(Value::Null) },
20387        );
20388        worker.register_activity(
20389            "joined.activity",
20390            |_ctx, _input| async move { Ok(Value::Null) },
20391        );
20392        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
20393            Ok(Value::Null)
20394        });
20395
20396        worker
20397            .run_until(tokio::time::sleep(Duration::from_millis(20)))
20398            .await
20399            .expect("normal shutdown");
20400
20401        let deregistration_path = "/api/worker/registrations/mock-worker";
20402        assert_eq!(server.request_count(deregistration_path), 1);
20403        for poll_path in [
20404            "/api/worker/workflow-tasks/poll",
20405            "/api/worker/activity-tasks/poll",
20406            "/api/worker/query-tasks/poll",
20407        ] {
20408            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
20409        }
20410        assert_eq!(
20411            server.captured_paths().last().map(String::as_str),
20412            Some(deregistration_path),
20413            "deregistration must start only after every poller has joined"
20414        );
20415    }
20416
20417    #[tokio::test]
20418    async fn registration_failure_does_not_deregister() {
20419        let server = MockWorkerServer::rejected_registration();
20420        let client = Client::builder(server.base_url())
20421            .timeout(Duration::from_secs(2))
20422            .build()
20423            .expect("client");
20424        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
20425
20426        let error = worker
20427            .run_until(async {})
20428            .await
20429            .expect_err("registration must fail");
20430        assert!(matches!(
20431            error,
20432            Error::Http {
20433                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
20434                ..
20435            }
20436        ));
20437        assert!(server
20438            .captured_paths()
20439            .iter()
20440            .all(|path| !path.starts_with("/api/worker/registrations/")));
20441    }
20442
20443    #[tokio::test]
20444    async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
20445        let server = MockWorkerServer::rejected_registration_protocol();
20446        let client = Client::builder(server.base_url())
20447            .timeout(Duration::from_secs(2))
20448            .build()
20449            .expect("client");
20450        let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
20451
20452        let error = worker
20453            .run_until(async {})
20454            .await
20455            .expect_err("a protocol 1.16 server must reject this worker");
20456        let Error::Protocol(failure) = error else {
20457            panic!("expected typed protocol rejection");
20458        };
20459        assert_eq!(failure.reason, "unsupported_protocol_version");
20460        assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
20461        assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
20462        assert_eq!(
20463            server
20464                .worker_protocol_for("/api/worker/register")
20465                .as_deref(),
20466            Some(WORKER_PROTOCOL_VERSION)
20467        );
20468    }
20469
20470    #[tokio::test]
20471    async fn declined_registration_does_not_deregister() {
20472        let server = MockWorkerServer::declined_registration();
20473        let client = Client::builder(server.base_url())
20474            .timeout(Duration::from_secs(2))
20475            .build()
20476            .expect("client");
20477        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
20478
20479        let error = worker
20480            .run_until(async {})
20481            .await
20482            .expect_err("declined registration must fail");
20483        assert!(matches!(error, Error::WorkerLoop(_)));
20484        assert!(error.to_string().contains("was not accepted"));
20485        assert!(server
20486            .captured_paths()
20487            .iter()
20488            .all(|path| !path.starts_with("/api/worker/registrations/")));
20489    }
20490
20491    #[tokio::test]
20492    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
20493        let server = MockWorkerServer::rejected_deregistration();
20494        let client = Client::builder(server.base_url())
20495            .timeout(Duration::from_secs(2))
20496            .build()
20497            .expect("client");
20498        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
20499
20500        let error = worker
20501            .run_until(async {})
20502            .await
20503            .expect_err("deregistration must fail");
20504        assert!(matches!(
20505            error,
20506            Error::Http {
20507                status: reqwest::StatusCode::FORBIDDEN,
20508                ..
20509            }
20510        ));
20511        assert_eq!(
20512            server.request_count("/api/worker/registrations/mock-worker"),
20513            1
20514        );
20515    }
20516
20517    #[tokio::test]
20518    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
20519        let server = MockWorkerServer::rejected_deregistration_protocol();
20520        let client = Client::builder(server.base_url())
20521            .timeout(Duration::from_secs(2))
20522            .build()
20523            .expect("client");
20524        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
20525
20526        let error = worker
20527            .run_until(async {})
20528            .await
20529            .expect_err("protocol rejection must fail shutdown");
20530        let Error::Protocol(failure) = error else {
20531            panic!("expected typed protocol failure");
20532        };
20533        assert_eq!(failure.reason, "unsupported_protocol_version");
20534        assert_eq!(
20535            failure.requested_version.as_deref(),
20536            Some(WORKER_PROTOCOL_VERSION)
20537        );
20538        assert_eq!(
20539            server.request_count("/api/worker/registrations/mock-worker"),
20540            1
20541        );
20542    }
20543
20544    #[tokio::test]
20545    async fn primary_poller_error_retains_deregistration_failure_context() {
20546        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
20547        let client = Client::builder(server.base_url())
20548            .timeout(Duration::from_secs(2))
20549            .build()
20550            .expect("client");
20551        let mut worker = Worker::new(client, "rust-workers")
20552            .worker_id("combined-failure")
20553            .poll_timeout(Duration::from_millis(10));
20554        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
20555            Ok(Value::Null)
20556        });
20557
20558        let error = worker
20559            .run()
20560            .await
20561            .expect_err("worker and cleanup must fail");
20562        let summary = error.to_string();
20563        assert!(summary.contains("authentication_failed"));
20564        assert!(summary.contains("worker cannot deregister"));
20565        let Error::WorkerShutdown {
20566            primary,
20567            deregistration,
20568        } = error
20569        else {
20570            panic!("expected combined worker shutdown error");
20571        };
20572        assert!(matches!(
20573            *primary,
20574            Error::Http {
20575                status: reqwest::StatusCode::UNAUTHORIZED,
20576                ..
20577            }
20578        ));
20579        assert!(matches!(
20580            *deregistration,
20581            Error::Http {
20582                status: reqwest::StatusCode::FORBIDDEN,
20583                ..
20584            }
20585        ));
20586        assert_eq!(
20587            server.request_count("/api/worker/registrations/mock-worker"),
20588            1
20589        );
20590    }
20591
20592    #[tokio::test]
20593    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
20594        let server = MockWorkerServer::start();
20595        let client = Client::builder(server.base_url())
20596            .timeout(Duration::from_secs(2))
20597            .build()
20598            .expect("client");
20599        let mut worker = Worker::new(client, "rust-workers")
20600            .worker_id("activity-only-worker")
20601            .poll_timeout(Duration::from_millis(10));
20602
20603        worker.register_activity(
20604            "activity.only",
20605            |_ctx, _args| async move { Ok(Value::Null) },
20606        );
20607
20608        worker.run_until(async {}).await.expect("run worker");
20609    }
20610
20611    #[tokio::test]
20612    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
20613        let server = MockWorkerServer::start();
20614        let client = Client::builder(server.base_url())
20615            .timeout(Duration::from_secs(2))
20616            .build()
20617            .expect("client");
20618        let mut worker = Worker::new(client, "rust-workers")
20619            .worker_id("workflow-only-worker")
20620            .poll_timeout(Duration::from_millis(10));
20621
20622        worker.register_workflow(
20623            "workflow.only",
20624            |_ctx, _input| async move { Ok(Value::Null) },
20625        );
20626
20627        worker.run_until(async {}).await.expect("run worker");
20628    }
20629
20630    #[tokio::test]
20631    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
20632        let server = MockWorkerServer::start();
20633        let client = Client::builder(server.base_url())
20634            .timeout(Duration::from_secs(2))
20635            .build()
20636            .expect("client");
20637        let observations = Arc::new(Mutex::new(Vec::new()));
20638        let observed = Arc::clone(&observations);
20639        let mut worker = Worker::new(client, "rust-workers")
20640            .worker_id("observed-heartbeat-worker")
20641            .poll_timeout(Duration::from_millis(10))
20642            .on_worker_heartbeat(move |observation| {
20643                observed
20644                    .lock()
20645                    .expect("heartbeat observations")
20646                    .push(observation.clone());
20647            });
20648
20649        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
20650            Ok(Value::Null)
20651        });
20652        let acknowledged = Arc::clone(&observations);
20653        worker
20654            .run_until(async move {
20655                tokio::time::timeout(Duration::from_secs(2), async move {
20656                    loop {
20657                        if !acknowledged
20658                            .lock()
20659                            .expect("heartbeat observations")
20660                            .is_empty()
20661                        {
20662                            break;
20663                        }
20664                        tokio::time::sleep(Duration::from_millis(1)).await;
20665                    }
20666                })
20667                .await
20668                .expect("heartbeat acknowledgement within timeout");
20669            })
20670            .await
20671            .expect("run worker");
20672
20673        let observations = observations.lock().expect("heartbeat observations");
20674        let first = observations.first().expect("heartbeat acknowledgement");
20675        assert_eq!(first.worker_id, "observed-heartbeat-worker");
20676        assert_eq!(first.task_queue, "rust-workers");
20677        assert!(first.acknowledged_at_unix_millis > 0);
20678        assert_eq!(first.acknowledgement, json!({}));
20679    }
20680
20681    #[tokio::test]
20682    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
20683        let server = MockWorkerServer::delayed_heartbeat_worker();
20684        let client = Client::builder(server.base_url())
20685            .timeout(Duration::from_secs(3))
20686            .build()
20687            .expect("client");
20688        let observations = Arc::new(Mutex::new(Vec::new()));
20689        let observed = Arc::clone(&observations);
20690        let mut worker = Worker::new(client, "rust-snapshot-workers")
20691            .worker_id("rust-snapshot-worker")
20692            .poll_timeout(Duration::from_millis(10))
20693            .on_worker_heartbeat(move |observation| {
20694                observed
20695                    .lock()
20696                    .expect("heartbeat observations")
20697                    .push(observation.clone());
20698            });
20699
20700        worker.register_workflow("snapshot", |ctx, _input| async move {
20701            ctx.wait_signal("finish").await?;
20702            Ok(json!({"status": "finished"}))
20703        });
20704        worker.register_query("snapshot", "current", |ctx, _args| async move {
20705            Ok(json!(ctx
20706                .signals("increment")
20707                .iter()
20708                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
20709                .sum::<i64>()))
20710        });
20711        worker.register_activity("cancel-aware", |_ctx, _args| async move {
20712            Ok(json!({"late": "completion"}))
20713        });
20714
20715        worker
20716            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
20717            .await
20718            .expect("delayed heartbeat must allow a clean worker shutdown");
20719
20720        let observations = observations.lock().expect("heartbeat observations");
20721        assert!(
20722            observations.len() >= 3,
20723            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
20724        );
20725        assert!(
20726            observations.windows(2).all(|pair| {
20727                pair[1].acknowledged_at_unix_millis
20728                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
20729                    >= 850
20730            }),
20731            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
20732        );
20733        drop(observations);
20734
20735        let heartbeat_times = server.request_times("/api/worker/heartbeat");
20736        let delayed_request_at = *heartbeat_times
20737            .get(1)
20738            .expect("intentionally delayed heartbeat request");
20739        let delay_window_start = delayed_request_at + Duration::from_millis(100);
20740        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
20741        for path in [
20742            "/api/worker/workflow-tasks/poll",
20743            "/api/worker/activity-tasks/poll",
20744            "/api/worker/query-tasks/poll",
20745        ] {
20746            assert!(
20747                server
20748                    .request_times(path)
20749                    .iter()
20750                    .any(|received_at| *received_at >= delay_window_start
20751                        && *received_at <= delay_window_end),
20752                "{path} must keep polling while a heartbeat acknowledgement is delayed"
20753            );
20754        }
20755        assert!(
20756            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
20757            "workflow work must be settled"
20758        );
20759        assert!(
20760            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
20761            "activity work must be settled"
20762        );
20763        assert!(
20764            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
20765            "query work must be settled"
20766        );
20767    }
20768
20769    #[tokio::test]
20770    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
20771        let server = MockWorkerServer::heartbeat_retry_worker();
20772        let client = Client::builder(server.base_url())
20773            .timeout(Duration::from_secs(2))
20774            .build()
20775            .expect("client");
20776        let observations = Arc::new(Mutex::new(Vec::new()));
20777        let observed = Arc::clone(&observations);
20778        let worker = Worker::new(client, "rust-workers")
20779            .worker_id("heartbeat-retry-worker")
20780            .retry_policy(WorkerRetryPolicy {
20781                max_retries: 1,
20782                initial_backoff: Duration::from_millis(300),
20783                max_backoff: Duration::from_millis(300),
20784            })
20785            .on_worker_heartbeat(move |observation| {
20786                observed
20787                    .lock()
20788                    .expect("heartbeat observations")
20789                    .push(observation.clone());
20790            });
20791
20792        worker
20793            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
20794            .await
20795            .expect("retryable heartbeat failure must remain bounded and recover");
20796
20797        let observations = observations.lock().expect("heartbeat observations");
20798        assert!(observations.len() >= 3, "heartbeat retry must recover");
20799        assert!(
20800            observations.windows(2).all(|pair| {
20801                pair[1]
20802                    .acknowledged_at_unix_millis
20803                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
20804                    >= 850
20805            }),
20806            "a successful retry must start a fresh advertised cadence: {observations:?}"
20807        );
20808        assert_eq!(
20809            server.request_count("/api/worker/heartbeat"),
20810            observations.len() + 1,
20811            "one retryable failure must add exactly one bounded request"
20812        );
20813    }
20814
20815    #[tokio::test]
20816    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
20817        let server = MockWorkerServer::waiting_query_worker();
20818        let client = Client::builder(server.base_url())
20819            .timeout(Duration::from_secs(2))
20820            .build()
20821            .expect("client");
20822        let observations = Arc::new(Mutex::new(Vec::new()));
20823        let observed = Arc::clone(&observations);
20824        let mut worker = Worker::new(client, "rust-snapshot-workers")
20825            .worker_id("rust-snapshot-worker")
20826            .poll_timeout(Duration::from_millis(10))
20827            .on_worker_heartbeat(move |observation| {
20828                observed
20829                    .lock()
20830                    .expect("heartbeat observations")
20831                    .push(observation.clone());
20832            });
20833
20834        worker.register_workflow("snapshot", |ctx, _input| async move {
20835            ctx.wait_signal("finish").await?;
20836            Ok(json!({"status": "finished"}))
20837        });
20838        worker.register_query("snapshot", "current", |ctx, _args| async move {
20839            let current = ctx
20840                .signals("increment")
20841                .iter()
20842                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
20843                .sum::<i64>();
20844            Ok(json!(current))
20845        });
20846        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
20847
20848        worker
20849            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
20850            .await
20851            .expect("pending workflow and query poller must remain live until shutdown");
20852
20853        assert!(
20854            observations.lock().expect("heartbeat observations").len() >= 4,
20855            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
20856        );
20857        assert!(
20858            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
20859            "workflow polling must continue after empty replay acknowledgements"
20860        );
20861        assert!(
20862            server.request_count("/api/worker/query-tasks/poll") >= 2,
20863            "query polling must continue after serving the current query"
20864        );
20865        assert_eq!(
20866            server.request_body("/api/worker/register")["capabilities"],
20867            json!([
20868                CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
20869                MEMO_UPSERTS_CAPABILITY,
20870                TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
20871                QUERY_TASKS_CAPABILITY,
20872                WORKFLOW_UPDATES_CAPABILITY,
20873                MESSAGE_STREAMS_CAPABILITY
20874            ])
20875        );
20876        assert_eq!(
20877            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
20878            json!({
20879                "queries": ["current"],
20880                "query_contracts": [],
20881                "signals": [],
20882                "signal_contracts": [],
20883                "updates": ["replace"],
20884                "update_contracts": [],
20885                "update_validators": [],
20886            })
20887        );
20888
20889        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
20890        assert_eq!(
20891            opened["commands"],
20892            json!([{
20893                "type": "open_signal_wait",
20894                "signal_name": "finish",
20895            }])
20896        );
20897
20898        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
20899            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
20900            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
20901            let failure = server.request_body(&fail_path);
20902            assert_eq!(
20903                failure["failure"]["type"],
20904                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
20905            );
20906            assert_eq!(server.request_count(&completion_path), 0);
20907        }
20908
20909        let query_completion =
20910            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
20911        assert_eq!(query_completion["result"], json!(8));
20912
20913        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
20914        assert_eq!(
20915            server.request_count(terminal_path),
20916            1,
20917            "the matching signal must settle the workflow exactly once"
20918        );
20919        let terminal = server.request_body(terminal_path);
20920        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
20921        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
20922        assert_eq!(
20923            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
20924                .expect("terminal workflow result"),
20925            json!({"status": "finished"})
20926        );
20927    }
20928
20929    #[tokio::test]
20930    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
20931        let server = MockWorkerServer::transient_worker_failures();
20932        let client = Client::builder(server.base_url())
20933            .timeout(Duration::from_secs(2))
20934            .build()
20935            .expect("client");
20936        let mut worker = Worker::new(client, "rust-workers")
20937            .worker_id("retry-worker")
20938            .poll_timeout(Duration::from_millis(10))
20939            .retry_policy(WorkerRetryPolicy {
20940                max_retries: 2,
20941                initial_backoff: Duration::from_millis(1),
20942                max_backoff: Duration::from_millis(1),
20943            });
20944        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
20945        worker.register_activity(
20946            "counter.activity",
20947            |_ctx, _input| async move { Ok(Value::Null) },
20948        );
20949        worker.register_query(
20950            "counter",
20951            "current",
20952            |_ctx, _args| async move { Ok(json!(8)) },
20953        );
20954
20955        worker
20956            .run_until(tokio::time::sleep(Duration::from_millis(75)))
20957            .await
20958            .expect("transient failures must not stop the worker");
20959
20960        for path in [
20961            "/api/worker/heartbeat",
20962            "/api/worker/workflow-tasks/poll",
20963            "/api/worker/activity-tasks/poll",
20964            "/api/worker/query-tasks/poll",
20965        ] {
20966            assert!(
20967                server.request_count(path) >= 2,
20968                "{path} must continue after its transient failure"
20969            );
20970        }
20971    }
20972
20973    #[tokio::test]
20974    async fn worker_bounds_transport_retries() {
20975        let server = MockWorkerServer::unavailable_polls();
20976        let client = Client::builder(server.base_url())
20977            .timeout(Duration::from_secs(2))
20978            .build()
20979            .expect("client");
20980        let mut worker = Worker::new(client, "rust-workers")
20981            .worker_id("bounded-retry-worker")
20982            .poll_timeout(Duration::from_millis(10))
20983            .retry_policy(WorkerRetryPolicy {
20984                max_retries: 2,
20985                initial_backoff: Duration::from_millis(1),
20986                max_backoff: Duration::from_millis(1),
20987            });
20988        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
20989
20990        let error = worker.run().await.expect_err("retry bound must terminate");
20991        assert!(matches!(error, Error::Transport(_)));
20992        assert_eq!(
20993            server.request_count("/api/worker/workflow-tasks/poll"),
20994            3,
20995            "one initial request plus exactly two retries"
20996        );
20997    }
20998
20999    #[tokio::test]
21000    async fn worker_retry_policy_can_disable_poll_retries() {
21001        let server = MockWorkerServer::unavailable_polls();
21002        let client = Client::builder(server.base_url())
21003            .timeout(Duration::from_secs(2))
21004            .build()
21005            .expect("client");
21006        let mut worker = Worker::new(client, "rust-workers")
21007            .worker_id("no-retry-worker")
21008            .poll_timeout(Duration::from_millis(10))
21009            .retry_policy(WorkerRetryPolicy {
21010                max_retries: 0,
21011                initial_backoff: Duration::from_millis(1),
21012                max_backoff: Duration::from_millis(1),
21013            });
21014        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21015
21016        let error = worker
21017            .run_once()
21018            .await
21019            .expect_err("disabled retries must return the first transport failure");
21020        assert!(matches!(error, Error::Transport(_)));
21021        assert_eq!(
21022            server.request_count("/api/worker/workflow-tasks/poll"),
21023            1,
21024            "max_retries=0 must send only the initial request"
21025        );
21026    }
21027
21028    #[tokio::test]
21029    async fn worker_does_not_retry_authentication_failures() {
21030        let server = MockWorkerServer::unauthorized_polls();
21031        let client = Client::builder(server.base_url())
21032            .timeout(Duration::from_secs(2))
21033            .build()
21034            .expect("client");
21035        let mut worker = Worker::new(client, "rust-workers")
21036            .worker_id("unauthorized-worker")
21037            .poll_timeout(Duration::from_millis(10));
21038        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21039
21040        let error = worker
21041            .run()
21042            .await
21043            .expect_err("authentication must terminate");
21044        let Error::Http { status, body } = error else {
21045            panic!("expected stable HTTP authentication error");
21046        };
21047        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
21048        assert!(body.contains("authentication_failed"));
21049        assert_eq!(
21050            server.request_count("/api/worker/workflow-tasks/poll"),
21051            1,
21052            "authentication failures must not be retried"
21053        );
21054    }
21055
21056    #[derive(Clone, Debug)]
21057    struct CapturedRequest {
21058        method: String,
21059        path: String,
21060        authorization: Option<String>,
21061        namespace: Option<String>,
21062        worker_protocol: Option<String>,
21063        control_protocol: Option<String>,
21064        body: String,
21065        received_at: Instant,
21066    }
21067
21068    struct MockWorkerServer {
21069        addr: SocketAddr,
21070        stop: Arc<AtomicBool>,
21071        requests: Arc<Mutex<Vec<CapturedRequest>>>,
21072        thread: Option<thread::JoinHandle<()>>,
21073    }
21074
21075    #[derive(Clone, Copy, Default)]
21076    struct MockWorkerBehavior {
21077        reject_query_protocol: bool,
21078        reject_query_completion: bool,
21079        waiting_query_worker: bool,
21080        decline_registration: bool,
21081        complete_named_signal: bool,
21082        poll_failures_per_path: usize,
21083        heartbeat_failures: usize,
21084        heartbeat_failure_request: Option<usize>,
21085        delayed_heartbeat_request: Option<usize>,
21086        heartbeat_response_delay: Duration,
21087        concurrent_requests: bool,
21088        unauthorized_polls: bool,
21089        reject_registration: bool,
21090        reject_registration_protocol: bool,
21091        reject_deregistration: bool,
21092        reject_deregistration_protocol: bool,
21093        cancelled_activity: bool,
21094        draining_polls: bool,
21095        invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
21096        workflow_completion_status: Option<&'static str>,
21097        workflow_completion_body: Option<&'static str>,
21098    }
21099
21100    impl MockWorkerServer {
21101        fn start() -> Self {
21102            Self::start_with_behavior(MockWorkerBehavior::default())
21103        }
21104
21105        fn reject_query_protocol() -> Self {
21106            Self::start_with_behavior(MockWorkerBehavior {
21107                reject_query_protocol: true,
21108                ..MockWorkerBehavior::default()
21109            })
21110        }
21111
21112        fn reject_query_completion() -> Self {
21113            Self::start_with_behavior(MockWorkerBehavior {
21114                reject_query_completion: true,
21115                ..MockWorkerBehavior::default()
21116            })
21117        }
21118
21119        fn waiting_query_worker() -> Self {
21120            Self::start_with_behavior(MockWorkerBehavior {
21121                waiting_query_worker: true,
21122                complete_named_signal: true,
21123                ..MockWorkerBehavior::default()
21124            })
21125        }
21126
21127        fn transient_worker_failures() -> Self {
21128            Self::start_with_behavior(MockWorkerBehavior {
21129                poll_failures_per_path: 1,
21130                heartbeat_failures: 1,
21131                ..MockWorkerBehavior::default()
21132            })
21133        }
21134
21135        fn consecutive_poll_failures(count: usize) -> Self {
21136            Self::start_with_behavior(MockWorkerBehavior {
21137                poll_failures_per_path: count,
21138                ..MockWorkerBehavior::default()
21139            })
21140        }
21141
21142        fn delayed_heartbeat_worker() -> Self {
21143            Self::start_with_behavior(MockWorkerBehavior {
21144                waiting_query_worker: true,
21145                delayed_heartbeat_request: Some(2),
21146                heartbeat_response_delay: Duration::from_millis(1_500),
21147                concurrent_requests: true,
21148                cancelled_activity: true,
21149                ..MockWorkerBehavior::default()
21150            })
21151        }
21152
21153        fn heartbeat_retry_worker() -> Self {
21154            Self::start_with_behavior(MockWorkerBehavior {
21155                waiting_query_worker: true,
21156                heartbeat_failure_request: Some(2),
21157                concurrent_requests: true,
21158                ..MockWorkerBehavior::default()
21159            })
21160        }
21161
21162        fn unavailable_polls() -> Self {
21163            Self::start_with_behavior(MockWorkerBehavior {
21164                poll_failures_per_path: usize::MAX,
21165                ..MockWorkerBehavior::default()
21166            })
21167        }
21168
21169        fn unauthorized_polls() -> Self {
21170            Self::start_with_behavior(MockWorkerBehavior {
21171                unauthorized_polls: true,
21172                ..MockWorkerBehavior::default()
21173            })
21174        }
21175
21176        fn rejected_registration() -> Self {
21177            Self::start_with_behavior(MockWorkerBehavior {
21178                reject_registration: true,
21179                ..MockWorkerBehavior::default()
21180            })
21181        }
21182
21183        fn rejected_registration_protocol() -> Self {
21184            Self::start_with_behavior(MockWorkerBehavior {
21185                reject_registration_protocol: true,
21186                ..MockWorkerBehavior::default()
21187            })
21188        }
21189
21190        fn declined_registration() -> Self {
21191            Self::start_with_behavior(MockWorkerBehavior {
21192                decline_registration: true,
21193                ..MockWorkerBehavior::default()
21194            })
21195        }
21196
21197        fn rejected_deregistration() -> Self {
21198            Self::start_with_behavior(MockWorkerBehavior {
21199                reject_deregistration: true,
21200                ..MockWorkerBehavior::default()
21201            })
21202        }
21203
21204        fn rejected_deregistration_protocol() -> Self {
21205            Self::start_with_behavior(MockWorkerBehavior {
21206                reject_deregistration_protocol: true,
21207                ..MockWorkerBehavior::default()
21208            })
21209        }
21210
21211        fn unauthorized_polls_and_rejected_deregistration() -> Self {
21212            Self::start_with_behavior(MockWorkerBehavior {
21213                unauthorized_polls: true,
21214                reject_deregistration: true,
21215                ..MockWorkerBehavior::default()
21216            })
21217        }
21218
21219        fn cancelled_activity() -> Self {
21220            Self::start_with_behavior(MockWorkerBehavior {
21221                cancelled_activity: true,
21222                ..MockWorkerBehavior::default()
21223            })
21224        }
21225
21226        fn draining_polls() -> Self {
21227            Self::start_with_behavior(MockWorkerBehavior {
21228                draining_polls: true,
21229                ..MockWorkerBehavior::default()
21230            })
21231        }
21232
21233        fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
21234            Self::start_with_behavior(MockWorkerBehavior {
21235                invalid_task_payload_codec: Some(codec),
21236                ..MockWorkerBehavior::default()
21237            })
21238        }
21239
21240        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
21241            Self::start_with_behavior(MockWorkerBehavior {
21242                workflow_completion_status: Some(status),
21243                workflow_completion_body: Some(body),
21244                ..MockWorkerBehavior::default()
21245            })
21246        }
21247
21248        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
21249            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
21250            listener
21251                .set_nonblocking(true)
21252                .expect("configure mock listener");
21253            let addr = listener.local_addr().expect("mock server address");
21254            let stop = Arc::new(AtomicBool::new(false));
21255            let server_stop = Arc::clone(&stop);
21256            let requests = Arc::new(Mutex::new(Vec::new()));
21257            let server_requests = Arc::clone(&requests);
21258            let thread = thread::spawn(move || {
21259                let mut request_threads = Vec::new();
21260                while !server_stop.load(Ordering::SeqCst) {
21261                    match listener.accept() {
21262                        Ok((mut stream, _)) => {
21263                            if behavior.concurrent_requests {
21264                                let requests = Arc::clone(&server_requests);
21265                                request_threads.push(thread::spawn(move || {
21266                                    handle_mock_worker_request(&mut stream, &requests, behavior)
21267                                }));
21268                            } else {
21269                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
21270                            }
21271                        }
21272                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
21273                            let mut index = 0;
21274                            while index < request_threads.len() {
21275                                if request_threads[index].is_finished() {
21276                                    request_threads
21277                                        .swap_remove(index)
21278                                        .join()
21279                                        .expect("join mock request");
21280                                } else {
21281                                    index += 1;
21282                                }
21283                            }
21284                            thread::sleep(Duration::from_millis(5));
21285                        }
21286                        Err(_) => break,
21287                    }
21288                }
21289                for request_thread in request_threads {
21290                    request_thread.join().expect("join mock request");
21291                }
21292            });
21293
21294            Self {
21295                addr,
21296                stop,
21297                requests,
21298                thread: Some(thread),
21299            }
21300        }
21301
21302        fn base_url(&self) -> String {
21303            format!("http://{}", self.addr)
21304        }
21305
21306        fn worker_protocol_for(&self, path: &str) -> Option<String> {
21307            self.requests
21308                .lock()
21309                .expect("captured requests")
21310                .iter()
21311                .find(|request| request.path == path)
21312                .and_then(|request| request.worker_protocol.clone())
21313        }
21314
21315        fn control_protocol_for(&self, path: &str) -> Option<String> {
21316            self.requests
21317                .lock()
21318                .expect("captured requests")
21319                .iter()
21320                .find(|request| request.path == path)
21321                .and_then(|request| request.control_protocol.clone())
21322        }
21323
21324        fn method_for(&self, path: &str) -> Option<String> {
21325            self.requests
21326                .lock()
21327                .expect("captured requests")
21328                .iter()
21329                .find(|request| request.path == path)
21330                .map(|request| request.method.clone())
21331        }
21332
21333        fn authorization_for(&self, path: &str) -> Option<String> {
21334            self.requests
21335                .lock()
21336                .expect("captured requests")
21337                .iter()
21338                .find(|request| request.path == path)
21339                .and_then(|request| request.authorization.clone())
21340        }
21341
21342        fn namespace_for(&self, path: &str) -> Option<String> {
21343            self.requests
21344                .lock()
21345                .expect("captured requests")
21346                .iter()
21347                .find(|request| request.path == path)
21348                .and_then(|request| request.namespace.clone())
21349        }
21350
21351        fn request_count(&self, path: &str) -> usize {
21352            self.requests
21353                .lock()
21354                .expect("captured requests")
21355                .iter()
21356                .filter(|request| request.path == path)
21357                .count()
21358        }
21359
21360        fn captured_paths(&self) -> Vec<String> {
21361            self.requests
21362                .lock()
21363                .expect("captured requests")
21364                .iter()
21365                .map(|request| request.path.clone())
21366                .collect()
21367        }
21368
21369        fn request_times(&self, path: &str) -> Vec<Instant> {
21370            self.requests
21371                .lock()
21372                .expect("captured requests")
21373                .iter()
21374                .filter(|request| request.path == path)
21375                .map(|request| request.received_at)
21376                .collect()
21377        }
21378
21379        fn request_body(&self, path: &str) -> Value {
21380            let requests = self.requests.lock().expect("captured requests");
21381            let body = &requests
21382                .iter()
21383                .find(|request| request.path == path)
21384                .unwrap_or_else(|| panic!("missing request for {path}"))
21385                .body;
21386            serde_json::from_str(body).unwrap_or_else(|error| {
21387                panic!("invalid JSON request body for {path}: {error}: {body:?}")
21388            })
21389        }
21390
21391        fn request_bodies(&self, path: &str) -> Vec<Value> {
21392            self.requests
21393                .lock()
21394                .expect("captured requests")
21395                .iter()
21396                .filter(|request| request.path == path)
21397                .map(|request| {
21398                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
21399                        panic!(
21400                            "invalid JSON request body for {path}: {error}: {:?}",
21401                            request.body
21402                        )
21403                    })
21404                })
21405                .collect()
21406        }
21407    }
21408
21409    impl Drop for MockWorkerServer {
21410        fn drop(&mut self) {
21411            self.stop.store(true, Ordering::SeqCst);
21412            let _ = TcpStream::connect(self.addr);
21413
21414            if let Some(thread) = self.thread.take() {
21415                thread.join().expect("join mock server");
21416            }
21417        }
21418    }
21419
21420    fn handle_mock_worker_request(
21421        stream: &mut TcpStream,
21422        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
21423        behavior: MockWorkerBehavior,
21424    ) {
21425        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
21426        let mut buffer = [0_u8; 8192];
21427        let mut request = Vec::new();
21428
21429        loop {
21430            match stream.read(&mut buffer) {
21431                Ok(0) => break,
21432                Ok(read) => {
21433                    request.extend_from_slice(&buffer[..read]);
21434                    if mock_request_is_complete(&request) {
21435                        break;
21436                    }
21437                }
21438                Err(error)
21439                    if matches!(
21440                        error.kind(),
21441                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
21442                    ) =>
21443                {
21444                    break;
21445                }
21446                Err(_) => return,
21447            }
21448        }
21449
21450        let request = String::from_utf8_lossy(&request);
21451        let body = request
21452            .split_once("\r\n\r\n")
21453            .map(|(_, body)| body)
21454            .unwrap_or_default();
21455        let path = request
21456            .lines()
21457            .next()
21458            .and_then(|line| line.split_whitespace().nth(1))
21459            .unwrap_or_default();
21460        let method = request
21461            .lines()
21462            .next()
21463            .and_then(|line| line.split_whitespace().next())
21464            .unwrap_or_default();
21465        let authorization = request.lines().find_map(|line| {
21466            let (name, value) = line.split_once(':')?;
21467            name.eq_ignore_ascii_case("Authorization")
21468                .then(|| value.trim().to_string())
21469        });
21470        let namespace = request.lines().find_map(|line| {
21471            let (name, value) = line.split_once(':')?;
21472            name.eq_ignore_ascii_case("X-Namespace")
21473                .then(|| value.trim().to_string())
21474        });
21475        let worker_protocol = request.lines().find_map(|line| {
21476            let (name, value) = line.split_once(':')?;
21477            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
21478                .then(|| value.trim().to_string())
21479        });
21480        let control_protocol = request.lines().find_map(|line| {
21481            let (name, value) = line.split_once(':')?;
21482            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
21483                .then(|| value.trim().to_string())
21484        });
21485        let request_number = {
21486            let mut requests = requests.lock().expect("captured requests");
21487            requests.push(CapturedRequest {
21488                method: method.to_string(),
21489                path: path.to_string(),
21490                authorization,
21491                namespace,
21492                worker_protocol: worker_protocol.clone(),
21493                control_protocol,
21494                body: body.to_string(),
21495                received_at: Instant::now(),
21496            });
21497            requests
21498                .iter()
21499                .filter(|request| request.path == path)
21500                .count()
21501        };
21502
21503        if path == "/api/worker/register" {
21504            if behavior.reject_registration_protocol {
21505                write_mock_response(
21506                    stream,
21507                    "400 Bad Request",
21508                    r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
21509                );
21510                return;
21511            }
21512            if behavior.reject_registration {
21513                write_mock_response(
21514                    stream,
21515                    "503 Service Unavailable",
21516                    r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
21517                );
21518                return;
21519            }
21520        }
21521
21522        if path.starts_with("/api/worker/registrations/") {
21523            if behavior.reject_deregistration_protocol {
21524                write_mock_response(
21525                    stream,
21526                    "400 Bad Request",
21527                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.16","requested_version":"1.17"}"#,
21528                );
21529            } else if behavior.reject_deregistration {
21530                write_mock_response(
21531                    stream,
21532                    "403 Forbidden",
21533                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
21534                );
21535            } else {
21536                write_mock_response(
21537                    stream,
21538                    "200 OK",
21539                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
21540                );
21541            }
21542            return;
21543        }
21544
21545        let is_poll = matches!(
21546            path,
21547            "/api/worker/workflow-tasks/poll"
21548                | "/api/worker/activity-tasks/poll"
21549                | "/api/worker/query-tasks/poll"
21550        );
21551        if is_poll && request_number <= behavior.poll_failures_per_path {
21552            return;
21553        }
21554        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
21555            return;
21556        }
21557        if path == "/api/worker/heartbeat"
21558            && behavior.heartbeat_failure_request == Some(request_number)
21559        {
21560            return;
21561        }
21562        if path == "/api/worker/heartbeat"
21563            && behavior.delayed_heartbeat_request == Some(request_number)
21564        {
21565            thread::sleep(behavior.heartbeat_response_delay);
21566        }
21567        if behavior.unauthorized_polls && is_poll {
21568            write_mock_response(
21569                stream,
21570                "401 Unauthorized",
21571                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
21572            );
21573            return;
21574        }
21575        if behavior.draining_polls && is_poll {
21576            write_mock_response(
21577                stream,
21578                "409 Conflict",
21579                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
21580            );
21581            return;
21582        }
21583
21584        if let Some(codec_case) = behavior.invalid_task_payload_codec {
21585            if is_poll && request_number == 1 {
21586                let mut task = match path {
21587                    "/api/worker/workflow-tasks/poll" => json!({
21588                        "task_id": "codec-workflow",
21589                        "workflow_type": "codec.workflow",
21590                        "payload_codec": DEFAULT_CODEC,
21591                        "workflow_task_attempt": 1,
21592                        "lease_owner": "codec-worker"
21593                    }),
21594                    "/api/worker/activity-tasks/poll" => json!({
21595                        "task_id": "codec-activity",
21596                        "activity_attempt_id": "codec-activity-attempt",
21597                        "activity_type": "codec.activity",
21598                        "payload_codec": DEFAULT_CODEC,
21599                        "attempt_number": 1,
21600                        "lease_owner": "codec-worker"
21601                    }),
21602                    "/api/worker/query-tasks/poll" => json!({
21603                        "query_task_id": "codec-query",
21604                        "query_task_attempt": 1,
21605                        "workflow_type": "codec.workflow",
21606                        "query_name": "known",
21607                        "payload_codec": DEFAULT_CODEC,
21608                        "lease_owner": "codec-worker"
21609                    }),
21610                    _ => unreachable!("is_poll limits task codec probe paths"),
21611                };
21612                codec_case.apply(&mut task);
21613                write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
21614                return;
21615            }
21616
21617            if matches!(
21618                path,
21619                "/api/worker/workflow-tasks/codec-workflow/fail"
21620                    | "/api/worker/activity-tasks/codec-activity/fail"
21621                    | "/api/worker/query-tasks/codec-query/fail"
21622            ) {
21623                write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
21624                return;
21625            }
21626        }
21627
21628        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
21629            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
21630            let body = format!(
21631                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
21632            );
21633            write_mock_response(stream, "400 Bad Request", &body);
21634            return;
21635        }
21636
21637        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
21638        {
21639            write_mock_response(
21640                stream,
21641                "409 Conflict",
21642                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
21643            );
21644            return;
21645        }
21646
21647        if behavior.workflow_completion_status.is_some()
21648            && path == "/api/worker/workflow-tasks/poll"
21649            && request_number == 1
21650        {
21651            write_mock_response(
21652                stream,
21653                "200 OK",
21654                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
21655            );
21656            return;
21657        }
21658
21659        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
21660            if let (Some(status), Some(body)) = (
21661                behavior.workflow_completion_status,
21662                behavior.workflow_completion_body,
21663            ) {
21664                write_mock_response(stream, status, body);
21665                return;
21666            }
21667        }
21668
21669        if behavior.waiting_query_worker {
21670            if behavior.complete_named_signal
21671                && path == "/api/worker/workflow-tasks/poll"
21672                && request_number == 1
21673            {
21674                let body = json!({
21675                    "task": {
21676                        "task_id": "snapshot-open",
21677                        "workflow_id": "snapshot-1",
21678                        "run_id": "snapshot-run-1",
21679                        "workflow_type": "snapshot",
21680                        "payload_codec": DEFAULT_CODEC,
21681                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
21682                            .expect("Avro workflow arguments"),
21683                        "history_events": [],
21684                        "workflow_task_attempt": 1,
21685                        "lease_owner": "rust-snapshot-worker"
21686                    }
21687                })
21688                .to_string();
21689                write_mock_response(stream, "200 OK", &body);
21690                return;
21691            }
21692
21693            let signal_request = request_number - usize::from(behavior.complete_named_signal);
21694            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
21695            if path == "/api/worker/workflow-tasks/poll"
21696                && signal_request >= 1
21697                && signal_request <= signal_request_limit
21698            {
21699                let finish = behavior.complete_named_signal && signal_request == 3;
21700                let amounts = if signal_request == 1 {
21701                    vec![3]
21702                } else {
21703                    vec![3, 5]
21704                };
21705                let task_id = if signal_request == 1 {
21706                    "snapshot-wait-3"
21707                } else if finish {
21708                    "snapshot-finish"
21709                } else {
21710                    "snapshot-wait-5"
21711                };
21712                let mut history_events = std::iter::once(json!({
21713                    "event_type": "SignalWaitOpened",
21714                    "payload": {"sequence": 1, "signal_name": "finish"}
21715                }))
21716                .chain(amounts.iter().enumerate().map(|(index, amount)| {
21717                    json!({
21718                        "event_type": "SignalReceived",
21719                        "payload": {
21720                            "signal_id": format!("increment-{amount}"),
21721                            "signal_name": "increment",
21722                            "workflow_sequence": index + 2,
21723                            "payload_codec": DEFAULT_CODEC,
21724                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
21725                                .expect("Avro signal envelope")
21726                        }
21727                    })
21728                }))
21729                .collect::<Vec<_>>();
21730                let (resume_id, resume_name, resume_arguments) = if finish {
21731                    history_events.push(json!({
21732                        "event_type": "SignalReceived",
21733                        "payload": {
21734                            "signal_id": "finish",
21735                            "signal_name": "finish",
21736                            "workflow_sequence": 4,
21737                            "payload_codec": DEFAULT_CODEC,
21738                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
21739                                .expect("Avro finish signal envelope")
21740                        }
21741                    }));
21742                    (
21743                        "finish".to_string(),
21744                        "finish".to_string(),
21745                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
21746                            .expect("Avro finish resume signal"),
21747                    )
21748                } else {
21749                    let amount = amounts.last().expect("amount");
21750                    (
21751                        format!("increment-{amount}"),
21752                        "increment".to_string(),
21753                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
21754                            .expect("Avro increment resume signal"),
21755                    )
21756                };
21757                let body = json!({
21758                    "task": {
21759                        "task_id": task_id,
21760                        "workflow_id": "snapshot-1",
21761                        "run_id": "snapshot-run-1",
21762                        "workflow_type": "snapshot",
21763                        "payload_codec": DEFAULT_CODEC,
21764                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
21765                            .expect("Avro workflow arguments"),
21766                        "history_events": history_events,
21767                        "workflow_task_attempt": 1,
21768                        "workflow_signal_id": resume_id,
21769                        "signal_name": resume_name,
21770                        "signal_arguments": resume_arguments,
21771                        "lease_owner": "rust-snapshot-worker"
21772                    }
21773                })
21774                .to_string();
21775                write_mock_response(stream, "200 OK", &body);
21776                return;
21777            }
21778
21779            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
21780                let history_events = [3, 5]
21781                    .into_iter()
21782                    .enumerate()
21783                    .map(|(index, amount)| {
21784                        json!({
21785                            "event_type": "SignalReceived",
21786                            "payload": {
21787                                "signal_id": format!("increment-{amount}"),
21788                                "signal_name": "increment",
21789                                "workflow_sequence": index + 2,
21790                                "payload_codec": DEFAULT_CODEC,
21791                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
21792                                    .expect("Avro query signal envelope")
21793                            }
21794                        })
21795                    })
21796                    .collect::<Vec<_>>();
21797                let body = json!({
21798                    "task": {
21799                        "query_task_id": "snapshot-current",
21800                        "query_task_attempt": 1,
21801                        "lease_owner": "rust-snapshot-worker",
21802                        "workflow_id": "snapshot-1",
21803                        "run_id": "snapshot-run-1",
21804                        "workflow_type": "snapshot",
21805                        "query_name": "current",
21806                        "payload_codec": DEFAULT_CODEC,
21807                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
21808                            .expect("Avro workflow arguments"),
21809                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
21810                            .expect("Avro query arguments"),
21811                        "history_events": history_events,
21812                        "run_status": "waiting"
21813                    }
21814                })
21815                .to_string();
21816                write_mock_response(stream, "200 OK", &body);
21817                return;
21818            }
21819
21820            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
21821                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
21822            {
21823                write_mock_response(
21824                    stream,
21825                    "200 OK",
21826                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
21827                );
21828                return;
21829            }
21830
21831            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
21832                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
21833                return;
21834            }
21835
21836            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
21837                write_mock_response(
21838                    stream,
21839                    "200 OK",
21840                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
21841                );
21842                return;
21843            }
21844
21845            if path == "/api/worker/query-tasks/snapshot-current/complete" {
21846                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
21847                return;
21848            }
21849        }
21850
21851        if matches!(
21852            path,
21853            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
21854        ) {
21855            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21856                .expect("typed mock result");
21857            let body = json!({
21858                "result": typed_fidelity_probe().into_json().expect("result projection"),
21859                "result_envelope": result,
21860            })
21861            .to_string();
21862            write_mock_response(stream, "200 OK", &body);
21863            return;
21864        }
21865
21866        if path == "/api/workflows/typed-1" {
21867            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21868                .expect("typed mock result");
21869            let body = json!({
21870                "workflow_id": "typed-1",
21871                "run_id": "run-typed-1",
21872                "workflow_type": "typed.echo",
21873                "status": "completed",
21874                "output": typed_fidelity_probe().into_json().expect("output projection"),
21875                "output_envelope": result,
21876            })
21877            .to_string();
21878            write_mock_response(stream, "200 OK", &body);
21879            return;
21880        }
21881
21882        let (status, body) = match path {
21883            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
21884            "/api/workflows" => (
21885                "201 Created",
21886                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
21887            ),
21888            "/api/worker/register" if behavior.decline_registration => (
21889                "200 OK",
21890                r#"{"worker_id":"declined-worker","registered":false}"#,
21891            ),
21892            "/api/worker/register" if behavior.waiting_query_worker => (
21893                "200 OK",
21894                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
21895            ),
21896            "/api/worker/register" => (
21897                "200 OK",
21898                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
21899            ),
21900            "/api/worker/heartbeat" => ("200 OK", "{}"),
21901            "/api/worker/activity-tasks/poll"
21902                if behavior.cancelled_activity && request_number == 1 =>
21903            {
21904                (
21905                    "200 OK",
21906                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
21907                )
21908            }
21909            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
21910                ("200 OK", r#"{"task":null}"#)
21911            }
21912            "/api/worker/query-tasks/poll"
21913                if behavior.reject_query_completion && request_number == 1 =>
21914            {
21915                (
21916                    "200 OK",
21917                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
21918                )
21919            }
21920            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
21921            "/api/worker/query-tasks/query-capture/complete"
21922            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
21923            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
21924                "200 OK",
21925                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
21926            ),
21927            "/api/worker/activity-tasks/activity-cancel/complete" => (
21928                "409 Conflict",
21929                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
21930            ),
21931            "/api/worker/activity-tasks/activity-typed/complete"
21932            | "/api/worker/activity-tasks/activity-typed/fail"
21933            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
21934            "/api/workflows/counter-1/query/current" => (
21935                "200 OK",
21936                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
21937            ),
21938            "/api/workflows/counter-1/query/missing" => (
21939                "404 Not Found",
21940                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
21941            ),
21942            "/api/workflows/wf-lifecycle/cancel" => (
21943                "200 OK",
21944                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
21945            ),
21946            "/api/workflows/wf-lifecycle/terminate" => (
21947                "200 OK",
21948                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
21949            ),
21950            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
21951                "200 OK",
21952                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
21953            ),
21954            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
21955                "200 OK",
21956                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
21957            ),
21958            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
21959            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
21960                "409 Conflict",
21961                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
21962            ),
21963            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
21964                "200 OK",
21965                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
21966            ),
21967            "/api/workflows/wf-cancelled" => (
21968                "200 OK",
21969                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
21970            ),
21971            "/api/workflows/wf-terminated" => (
21972                "200 OK",
21973                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
21974            ),
21975            "/api/workflows/wf-timed-out" => (
21976                "200 OK",
21977                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
21978            ),
21979            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
21980                "200 OK",
21981                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
21982            ),
21983            "/api/workflows/wf-selected" => (
21984                "200 OK",
21985                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
21986            ),
21987            "/api/workflows/wf-selected/runs/run-selected" => (
21988                "200 OK",
21989                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
21990            ),
21991            _ => ("404 Not Found", r#"{"message":"not found"}"#),
21992        };
21993        write_mock_response(stream, status, body);
21994    }
21995
21996    fn mock_request_is_complete(request: &[u8]) -> bool {
21997        let Some(header_end) = request
21998            .windows(4)
21999            .position(|window| window == b"\r\n\r\n")
22000            .map(|position| position + 4)
22001        else {
22002            return false;
22003        };
22004        let headers = String::from_utf8_lossy(&request[..header_end]);
22005        let content_length = headers.lines().find_map(|line| {
22006            let (name, value) = line.split_once(':')?;
22007            name.eq_ignore_ascii_case("content-length")
22008                .then(|| value.trim().parse::<usize>().ok())
22009                .flatten()
22010        });
22011
22012        request.len() >= header_end + content_length.unwrap_or(0)
22013    }
22014
22015    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
22016        let response = format!(
22017            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
22018            body.len()
22019        );
22020
22021        let _ = stream.write_all(response.as_bytes());
22022        let _ = stream.flush();
22023    }
22024}