1#![doc = include_str!("../README.md")]
2
3mod cancellation_context;
4mod cancellation_replay_clock;
5mod cancellation_scope;
6mod cancellation_scope_history;
7mod cancellation_scope_replay;
8mod cooperative_cancellation;
9mod local_activity;
10mod runtime_payloads;
11mod runtime_uploads;
12mod sticky_worker;
13mod sticky_workflow_cache;
14mod worker_session;
15
16pub use cancellation_context::{
17 CancellationContext, CancellationLineage, ScopedCancellationContext, ScopedCancellationLineage,
18};
19pub use cancellation_scope::CancellationScopeOpenReceipt;
20#[doc(hidden)]
21pub use cancellation_scope_history::{
22 CancellationScopeDeliveryBudget, CancellationScopeDeliveryReceipt,
23};
24#[doc(hidden)]
25pub use cancellation_scope_replay::CancellationScopeRequested;
26
27pub use cooperative_cancellation::{
28 CancellationCallKind, CancellationDelivery, CancellationDeliveryReceipt,
29 CancellationDeliveryReply, CancellationHistory, CancellationRequest, CancellationShield,
30 CooperativeCancellationOptions, CooperativeCancellationRequested, CooperativeWorkflowTask,
31 CooperativeWorkflowTaskPoll, WorkflowCancellationRequest, WorkflowTaskHeartbeat,
32};
33
34use std::{
35 any::{type_name, Any, TypeId},
36 collections::{BTreeMap, BTreeSet, HashMap},
37 future::Future,
38 io::{self, Read},
39 pin::Pin,
40 sync::{
41 atomic::{AtomicBool, Ordering},
42 Arc, Mutex, OnceLock,
43 },
44 task::{Context as TaskContext, Poll},
45 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
46};
47
48use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
49use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
50use chrono::DateTime;
51use futures_util::{
52 future::OptionFuture, stream::FuturesUnordered, task::noop_waker_ref, StreamExt,
53};
54pub use local_activity::LocalActivityOptions;
55use serde::{
56 de::DeserializeOwned,
57 ser::{SerializeMap, SerializeSeq},
58 Deserialize, Deserializer, Serialize, Serializer,
59};
60pub use serde_json::{json, Value};
61use sha2::{Digest, Sha256};
62pub use sticky_workflow_cache::{StickyCacheMetrics, StickyCacheOptions};
63use thiserror::Error;
64pub use uuid::Uuid;
65pub use worker_session::{WorkerSession, WorkerSessionOptions};
66
67pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
68const WORKFLOW_HISTORY_PAGE_SIZE: usize = 500;
69pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
71pub const CONTROL_PLANE_VERSION: &str = "2";
72pub const DEFAULT_CODEC: &str = "avro";
73pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
74pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
76 "condition_wait_occurrence_identity";
77pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
79pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
81pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
83pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
85pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
87pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
89pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
90pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
91pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
92pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
93pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
94pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
96pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
98pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
100pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
102pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
104pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
106pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
108
109pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
110 let Some((major, minor)) = version.split_once('.') else {
111 return false;
112 };
113 major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
114}
115
116fn validate_user_signal_name(signal_name: &str) -> Result<()> {
117 if signal_name == MESSAGE_STREAM_SIGNAL {
118 return Err(Error::Codec(format!(
119 "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
120 )));
121 }
122 Ok(())
123}
124
125const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
126const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
127 "Workflow task waiting for scheduled history.";
128const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
129const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
130const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
131const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
132const MAX_MEMO_ENTRIES: usize = 100;
133const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
134const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
135
136const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
137 "lease_expired",
138 "query_task_not_found",
139 "query_task_not_leased",
140 "query_task_timed_out",
141];
142
143pub fn portable_worker_affinity_capability_manifest() -> Value {
148 json!({
149 "local_activities": {
150 "supported": false,
151 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
152 "reason": "rust_worker_does_not_execute_record_local_activity",
153 },
154 "worker_sessions": {
155 "supported": false,
156 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
157 "reason": "rust_worker_has_no_typed_session_lifecycle",
158 },
159 "sticky_execution": {
160 "supported": false,
161 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
162 "reason": "rust_worker_uses_complete_durable_history_replay",
163 },
164 })
165}
166
167pub const AVRO_VALUE_SCHEMA_JSON: &str =
169 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
170pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
171pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
172const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
173
174static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
175static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
176 OnceLock::new();
177
178#[derive(Clone, Copy)]
179enum RequestProtocol {
180 ControlPlane,
181 Worker(&'static str),
182}
183
184pub type Result<T> = std::result::Result<T, Error>;
185
186#[derive(Debug, Error)]
187pub enum Error {
188 #[error("transport error: {0}")]
189 Transport(#[from] reqwest::Error),
190 #[error(
191 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
192 )]
193 InvalidBaseUrl,
194 #[error("json error: {0}")]
195 Json(#[from] serde_json::Error),
196 #[error("http {status}: {body}")]
197 Http {
198 status: reqwest::StatusCode,
199 body: String,
200 },
201 #[error("codec error: {0}")]
202 Codec(String),
203 #[error(transparent)]
204 QueryFailed(QueryFailure),
205 #[error(transparent)]
206 Protocol(ProtocolFailure),
207 #[error(transparent)]
208 NonDeterministicReplay(ReplayFailure),
209 #[error(transparent)]
210 ChildWorkflowFailed(ChildWorkflowFailure),
211 #[error(transparent)]
212 ActivityFailed(ActivityFailure),
213 #[error(transparent)]
214 ParallelFailed(ParallelFailure),
215 #[error(transparent)]
216 SagaCompensationFailed(SagaCompensationFailure),
217 #[error(transparent)]
218 InvalidParallelGroup(ParallelGroupError),
219 #[error(transparent)]
220 DurableOperationCancelled(DurableOperationCancelled),
221 #[error(transparent)]
222 WorkflowCancellationRequested(WorkflowCancellationRequested),
223 #[error(transparent)]
224 CooperativeCancellationRequested(CooperativeCancellationRequested),
225 #[error(transparent)]
226 CancellationScopeRequested(CancellationScopeRequested),
227 #[error(transparent)]
228 WorkflowCommandRejected(WorkflowCommandRejection),
229 #[error(transparent)]
230 WorkflowFailed(WorkflowTerminalOutcome),
231 #[error(transparent)]
232 WorkflowCancelled(WorkflowTerminalOutcome),
233 #[error(transparent)]
234 WorkflowTerminated(WorkflowTerminalOutcome),
235 #[error(transparent)]
236 WorkflowTimedOut(WorkflowTerminalOutcome),
237 #[error(transparent)]
238 ActivityTaskRejected(ActivityTaskRejection),
239 #[error("workflow handler {0:?} is not registered")]
240 WorkflowNotRegistered(String),
241 #[error("activity handler {0:?} is not registered")]
242 ActivityNotRegistered(String),
243 #[error(
244 "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
245 )]
246 HandlerType {
247 handler_kind: HandlerKind,
248 handler_name: String,
249 value_kind: HandlerValueKind,
250 rust_type: &'static str,
251 message: String,
252 },
253 #[error("workflow future yielded without emitting a durable command")]
254 WorkflowYieldedWithoutCommand,
255 #[error(
256 "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id or task_id"
257 )]
258 MissingWorkflowCommandIdentity,
259 #[error("workflow state lock is poisoned")]
260 WorkflowStatePoisoned,
261 #[error("timer duration is too large for the worker protocol")]
262 TimerDurationOverflow,
263 #[error(transparent)]
264 InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
265 #[error(transparent)]
266 InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
267 #[error("operation timed out")]
268 Timeout,
269 #[error(
270 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
271 )]
272 MissingRoleCredentials {
273 role: &'static str,
274 opposite_role: &'static str,
275 },
276 #[error("worker loop error: {0}")]
277 WorkerLoop(String),
278 #[error(
279 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
280 )]
281 UnsupportedUpdateValidators { workflow_type: String },
282 #[error("{primary}; worker deregistration also failed: {deregistration}")]
283 WorkerShutdown {
284 primary: Box<Error>,
285 deregistration: Box<Error>,
286 },
287 #[error("invalid child workflow options: {0}")]
288 InvalidChildWorkflowOptions(String),
289 #[error("invalid workflow memo update: {0}")]
290 InvalidMemoUpdate(String),
291 #[error(
292 "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
293 )]
294 WorkflowMemoUpdatesUnavailable,
295 #[error("cooperative cancellation is unavailable: {0}")]
296 CooperativeCancellationUnavailable(String),
297 #[error("cancellation_scope_execution_not_supported: Rust worker cannot replay scoped cancellation history")]
298 CancellationScopeExecutionUnavailable,
299 #[error("invalid cooperative cancellation: {0}")]
300 InvalidCooperativeCancellation(String),
301 #[error("activity execution no longer owns its claim: {0}")]
302 ActivityExecutionAbandoned(String),
303 #[error(transparent)]
304 InvalidActivityOptions(ActivityOptionsError),
305 #[error(transparent)]
306 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
307 #[doc(hidden)]
308 #[error("workflow requested continue as new")]
309 ContinueAsNew(ContinueAsNewRequest),
310}
311
312#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
314pub enum RegistrationKind {
315 Workflow,
316 Activity,
317 Query,
318 Update,
319}
320
321impl std::fmt::Display for RegistrationKind {
322 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
323 formatter.write_str(match self {
324 Self::Workflow => "workflow",
325 Self::Activity => "activity",
326 Self::Query => "query",
327 Self::Update => "update",
328 })
329 }
330}
331
332#[derive(Clone, Debug, PartialEq, Eq)]
334pub struct HandlerRegistration {
335 pub method: &'static str,
336 pub handler_type: &'static str,
337 pub file: &'static str,
338 pub line: u32,
339 pub column: u32,
340}
341
342impl std::fmt::Display for HandlerRegistration {
343 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
344 write!(
345 formatter,
346 "{} at {}:{}:{} ({})",
347 self.method, self.file, self.line, self.column, self.handler_type
348 )
349 }
350}
351
352#[derive(Clone, Debug, Error, PartialEq, Eq)]
354#[error("duplicate_registration: {handler_kind} name {handler_name:?} in workflow scope {workflow_type:?}; first {first_definition}; conflicting {second_definition}")]
355pub struct DuplicateRegistrationError {
356 pub handler_kind: RegistrationKind,
357 pub handler_name: String,
358 pub workflow_type: Option<String>,
360 pub first_definition: HandlerRegistration,
361 pub second_definition: HandlerRegistration,
362}
363
364impl From<DuplicateRegistrationError> for Error {
365 fn from(error: DuplicateRegistrationError) -> Self {
366 Self::WorkerLoop(error.to_string())
367 }
368}
369
370#[derive(Clone, Debug, Error, PartialEq, Eq)]
372pub enum ConditionWaitOptionsError {
373 #[error("condition_key must be non-empty")]
374 EmptyKey,
375 #[error("condition_definition_fingerprint must be non-empty")]
376 EmptyPredicateIdentity,
377 #[error("condition timeout is too large for the worker protocol")]
378 TimeoutOverflow,
379}
380
381#[derive(Clone, Debug, PartialEq, Eq)]
388pub struct ConditionWaitOptions {
389 condition_key: String,
390 predicate_identity: String,
391 timeout: Option<Duration>,
392}
393
394impl ConditionWaitOptions {
395 pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
396 Self {
397 condition_key: condition_key.into(),
398 predicate_identity: predicate_identity.into(),
399 timeout: None,
400 }
401 }
402
403 pub fn timeout(mut self, timeout: Duration) -> Self {
404 self.timeout = Some(timeout);
405 self
406 }
407
408 fn validate(
409 &self,
410 ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
411 let condition_key = self.condition_key.trim();
412 if condition_key.is_empty() {
413 return Err(ConditionWaitOptionsError::EmptyKey);
414 }
415 let predicate_identity = self.predicate_identity.trim();
416 if predicate_identity.is_empty() {
417 return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
418 }
419 let timeout_seconds = self
420 .timeout
421 .map(|timeout| {
422 timeout
423 .as_secs()
424 .checked_add(u64::from(timeout.subsec_nanos() > 0))
425 .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
426 })
427 .transpose()?;
428
429 Ok(ValidatedConditionWaitOptions {
430 condition_key: condition_key.to_string(),
431 predicate_identity: predicate_identity.to_string(),
432 timeout_seconds,
433 })
434 }
435}
436
437#[derive(Clone, Debug, PartialEq, Eq)]
438struct ValidatedConditionWaitOptions {
439 condition_key: String,
440 predicate_identity: String,
441 timeout_seconds: Option<u64>,
442}
443
444const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
445
446#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
448#[serde(rename_all = "snake_case")]
449pub enum ConditionWaitResult {
450 Satisfied,
451 TimedOut,
452}
453
454impl ConditionWaitResult {
455 pub fn is_satisfied(self) -> bool {
456 self == Self::Satisfied
457 }
458
459 pub fn is_timed_out(self) -> bool {
460 self == Self::TimedOut
461 }
462}
463
464#[doc(hidden)]
466pub fn __condition_definition_fingerprint(source: &str) -> String {
467 let mut digest = Sha256::new();
468 digest.update(b"durable-workflow-rust.wait-condition.v1\0");
469 digest.update(source.as_bytes());
470 format!("sha256:{:x}", digest.finalize())
471}
472
473#[macro_export]
479macro_rules! wait_condition {
480 ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
481 $ctx.wait_condition(
482 $crate::ConditionWaitOptions::new(
483 $key,
484 $crate::__condition_definition_fingerprint(concat!(
485 module_path!(),
486 "\0",
487 stringify!($predicate)
488 )),
489 )
490 .timeout($timeout),
491 $predicate,
492 )
493 }};
494 ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
495 $ctx.wait_condition(
496 $crate::ConditionWaitOptions::new(
497 $key,
498 $crate::__condition_definition_fingerprint(concat!(
499 module_path!(),
500 "\0",
501 stringify!($predicate)
502 )),
503 ),
504 $predicate,
505 )
506 }};
507}
508
509const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
510const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
511const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
512const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
513const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
514
515#[derive(Clone, Debug, Error, PartialEq, Eq)]
517pub enum SearchAttributeUpdateError {
518 #[error("search-attribute update requires at least one attribute")]
519 Empty,
520 #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
521 InvalidKey(String),
522 #[error("search-attribute update exceeds the limit of 100 attributes")]
523 TooManyAttributes,
524 #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
525 ValueTooLong {
526 key: String,
527 kind: &'static str,
528 limit: usize,
529 },
530 #[error(
531 "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
532 )]
533 EmptyString(String),
534 #[error("search attribute {0:?} has a non-finite float value")]
535 NonFiniteFloat(String),
536 #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
537 InvalidDateTime(String),
538 #[error("search-attribute update exceeds the 65536-byte protocol limit")]
539 PayloadTooLarge,
540}
541
542#[derive(Clone, Debug, PartialEq)]
544pub enum SearchAttributeValue {
545 String(String),
546 Keyword(String),
547 KeywordList(Vec<String>),
548 Int(i64),
549 Float(f64),
550 Bool(bool),
551 DateTime(String),
552 Delete,
553}
554
555impl SearchAttributeValue {
556 fn type_name(&self) -> Option<&'static str> {
557 match self {
558 Self::String(_) => Some("string"),
559 Self::Keyword(_) => Some("keyword"),
560 Self::KeywordList(_) => Some("keyword_list"),
561 Self::Int(_) => Some("int"),
562 Self::Float(_) => Some("float"),
563 Self::Bool(_) => Some("bool"),
564 Self::DateTime(_) => Some("datetime"),
565 Self::Delete => None,
566 }
567 }
568
569 fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
570 let normalize_string = |value: String, kind: &'static str, limit: usize| {
571 let value = value.trim().to_string();
572 if value.is_empty() {
573 return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
574 }
575 if value.len() > limit {
576 return Err(SearchAttributeUpdateError::ValueTooLong {
577 key: key.to_string(),
578 kind,
579 limit,
580 });
581 }
582 Ok(value)
583 };
584
585 match self {
586 Self::String(value) => Ok(Self::String(normalize_string(
587 value,
588 "string",
589 MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
590 )?)),
591 Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
592 value,
593 "keyword",
594 MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
595 )?)),
596 Self::KeywordList(values) => {
597 let values = values
598 .into_iter()
599 .map(|value| {
600 let value = value.trim().to_string();
601 if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
602 return Err(SearchAttributeUpdateError::ValueTooLong {
603 key: key.to_string(),
604 kind: "keyword-list entry",
605 limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
606 });
607 }
608 Ok(value)
609 })
610 .collect::<std::result::Result<Vec<_>, _>>()?;
611 Ok(Self::KeywordList(values))
612 }
613 Self::Float(value) if !value.is_finite() => {
614 Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
615 }
616 Self::DateTime(value) => {
617 let value =
618 normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
619 if DateTime::parse_from_rfc3339(&value).is_err() {
620 return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
621 }
622 Ok(Self::DateTime(value))
623 }
624 value => Ok(value),
625 }
626 }
627
628 fn into_json(self) -> Value {
629 match self {
630 Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
631 Value::String(value)
632 }
633 Self::KeywordList(values) => {
634 Value::Array(values.into_iter().map(Value::String).collect())
635 }
636 Self::Int(value) => json!(value),
637 Self::Float(value) => json!(value),
638 Self::Bool(value) => json!(value),
639 Self::Delete => Value::Null,
640 }
641 }
642}
643
644#[derive(Clone, Debug, Default, PartialEq)]
646pub struct SearchAttributeUpdate {
647 attributes: BTreeMap<String, SearchAttributeValue>,
648}
649
650impl SearchAttributeUpdate {
651 pub fn new() -> Self {
652 Self::default()
653 }
654
655 pub fn set(
656 mut self,
657 key: impl Into<String>,
658 value: SearchAttributeValue,
659 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
660 let key = key.into();
661 validate_search_attribute_key(&key)?;
662 if !self.attributes.contains_key(&key)
663 && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
664 {
665 return Err(SearchAttributeUpdateError::TooManyAttributes);
666 }
667 self.attributes.insert(key.clone(), value.normalized(&key)?);
668 self.validate_size()?;
669 Ok(self)
670 }
671
672 pub fn string(
673 self,
674 key: impl Into<String>,
675 value: impl Into<String>,
676 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
677 self.set(key, SearchAttributeValue::String(value.into()))
678 }
679
680 pub fn keyword(
681 self,
682 key: impl Into<String>,
683 value: impl Into<String>,
684 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
685 self.set(key, SearchAttributeValue::Keyword(value.into()))
686 }
687
688 pub fn keyword_list<I, V>(
689 self,
690 key: impl Into<String>,
691 values: I,
692 ) -> std::result::Result<Self, SearchAttributeUpdateError>
693 where
694 I: IntoIterator<Item = V>,
695 V: Into<String>,
696 {
697 self.set(
698 key,
699 SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
700 )
701 }
702
703 pub fn int(
704 self,
705 key: impl Into<String>,
706 value: i64,
707 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
708 self.set(key, SearchAttributeValue::Int(value))
709 }
710
711 pub fn float(
712 self,
713 key: impl Into<String>,
714 value: f64,
715 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
716 self.set(key, SearchAttributeValue::Float(value))
717 }
718
719 pub fn bool(
720 self,
721 key: impl Into<String>,
722 value: bool,
723 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
724 self.set(key, SearchAttributeValue::Bool(value))
725 }
726
727 pub fn datetime(
728 self,
729 key: impl Into<String>,
730 value: impl Into<String>,
731 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
732 self.set(key, SearchAttributeValue::DateTime(value.into()))
733 }
734
735 pub fn delete(
736 self,
737 key: impl Into<String>,
738 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
739 self.set(key, SearchAttributeValue::Delete)
740 }
741
742 fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
743 let (attributes, _) = self.clone().into_wire_parts();
744 if serde_json::to_vec(&attributes)
745 .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
746 .unwrap_or(true)
747 {
748 return Err(SearchAttributeUpdateError::PayloadTooLarge);
749 }
750 Ok(())
751 }
752
753 fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
754 let mut attributes = serde_json::Map::new();
755 let mut attribute_types = BTreeMap::new();
756 for (key, value) in self.attributes {
757 if let Some(type_name) = value.type_name() {
758 attribute_types.insert(key.clone(), type_name.to_string());
759 }
760 attributes.insert(key, value.into_json());
761 }
762 (Value::Object(attributes), attribute_types)
763 }
764
765 fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
766 if self.attributes.is_empty() {
767 return Err(SearchAttributeUpdateError::Empty);
768 }
769 self.validate_size()
770 }
771}
772
773fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
774 let valid = !key.is_empty()
775 && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
776 && key
777 .bytes()
778 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
779 if valid {
780 Ok(())
781 } else {
782 Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
783 }
784}
785
786#[derive(Clone, Copy, Debug, PartialEq, Eq)]
788pub enum HandlerKind {
789 Workflow,
790 Activity,
791}
792
793impl std::fmt::Display for HandlerKind {
794 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
795 formatter.write_str(match self {
796 Self::Workflow => "workflow",
797 Self::Activity => "activity",
798 })
799 }
800}
801
802#[derive(Clone, Copy, Debug, PartialEq, Eq)]
804pub enum HandlerValueKind {
805 Input,
806 Result,
807}
808
809impl std::fmt::Display for HandlerValueKind {
810 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
811 formatter.write_str(match self {
812 Self::Input => "input",
813 Self::Result => "result",
814 })
815 }
816}
817
818#[derive(Clone, Copy, Debug, PartialEq, Eq)]
820pub enum WorkflowCommandKind {
821 Cancel,
822 Terminate,
823}
824
825impl WorkflowCommandKind {
826 fn as_str(self) -> &'static str {
827 match self {
828 Self::Cancel => "cancel",
829 Self::Terminate => "terminate",
830 }
831 }
832}
833
834#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
836pub struct WorkflowCommandOptions {
837 #[serde(skip_serializing_if = "Option::is_none")]
838 pub reason: Option<String>,
839 #[serde(skip_serializing_if = "Option::is_none")]
840 pub request_id: Option<String>,
841}
842
843#[derive(Clone, Debug, PartialEq, Eq)]
850pub struct WorkflowStartOptions {
851 pub execution_timeout_seconds: u64,
852 pub run_timeout_seconds: u64,
853}
854
855impl Default for WorkflowStartOptions {
856 fn default() -> Self {
857 Self {
858 execution_timeout_seconds: 3600,
859 run_timeout_seconds: 600,
860 }
861 }
862}
863
864impl WorkflowStartOptions {
865 pub fn new() -> Self {
866 Self::default()
867 }
868
869 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
870 self.execution_timeout_seconds = seconds;
871 self
872 }
873
874 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
875 self.run_timeout_seconds = seconds;
876 self
877 }
878
879 fn validate(&self) -> Result<()> {
880 if self.execution_timeout_seconds == 0 {
881 return Err(Error::Codec(
882 "execution_timeout_seconds must be at least 1".to_string(),
883 ));
884 }
885 if self.run_timeout_seconds == 0 {
886 return Err(Error::Codec(
887 "run_timeout_seconds must be at least 1".to_string(),
888 ));
889 }
890 if self.run_timeout_seconds > self.execution_timeout_seconds {
891 return Err(Error::Codec(
892 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
893 ));
894 }
895
896 Ok(())
897 }
898}
899
900#[derive(Clone, Debug, Default, PartialEq, Eq)]
905pub struct ContinueAsNewOptions {
906 pub workflow_type: Option<String>,
907 pub task_queue: Option<String>,
908}
909
910impl ContinueAsNewOptions {
911 pub fn new() -> Self {
912 Self::default()
913 }
914
915 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
916 self.workflow_type = Some(workflow_type.into());
917 self
918 }
919
920 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
921 self.task_queue = Some(task_queue.into());
922 self
923 }
924
925 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
926 for (field, value) in [
927 ("workflow_type", self.workflow_type.as_deref()),
928 ("task_queue", self.task_queue.as_deref()),
929 ] {
930 if value.is_some_and(|value| value.trim().is_empty()) {
931 return Err(ContinueAsNewOptionsError {
932 field,
933 message: format!("{field} must not be empty"),
934 });
935 }
936 }
937 Ok(())
938 }
939}
940
941#[derive(Clone, Debug, Error, PartialEq, Eq)]
943#[error("invalid continue-as-new option {field}: {message}")]
944pub struct ContinueAsNewOptionsError {
945 pub field: &'static str,
946 pub message: String,
947}
948
949#[derive(Clone, Debug, Default, PartialEq, Eq)]
951pub struct WorkflowHistoryBudget {
952 pub event_count: u64,
953 pub size_bytes: Option<u64>,
954 pub continue_as_new_recommended: bool,
955 pub pressure: Option<String>,
956}
957
958#[doc(hidden)]
959#[derive(Clone, Debug)]
960pub struct ContinueAsNewRequest {
961 arguments: AvroValue,
962 options: ContinueAsNewOptions,
963}
964
965impl WorkflowCommandOptions {
966 pub fn new() -> Self {
967 Self::default()
968 }
969
970 pub fn reason(mut self, reason: impl Into<String>) -> Self {
971 self.reason = Some(reason.into());
972 self
973 }
974
975 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
976 self.request_id = Some(request_id.into());
977 self
978 }
979}
980
981#[derive(Clone, Debug, PartialEq)]
983pub struct WorkflowCommandResult {
984 pub command: WorkflowCommandKind,
985 pub workflow_id: String,
986 pub run_id: Option<String>,
987 pub outcome: Option<String>,
988 pub reason: Option<String>,
989 pub command_status: Option<String>,
990 pub raw: Value,
991}
992
993#[derive(Clone, Debug, PartialEq)]
995pub struct WorkflowRedriveResult {
996 pub workflow_id: String,
997 pub source_run_id: String,
998 pub run_id: String,
999 pub resume_step_sequence: u64,
1000 pub raw: Value,
1001}
1002
1003#[derive(Clone, Debug, Error)]
1005#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
1006pub struct WorkflowCommandRejection {
1007 pub command: WorkflowCommandKind,
1008 pub status: u16,
1009 pub reason: String,
1010 pub message: String,
1011 pub workflow_id: String,
1012 pub run_id: Option<String>,
1013 pub target_scope: Option<String>,
1014 pub body: Value,
1015}
1016
1017#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1019pub enum WorkflowTerminalKind {
1020 Failed,
1021 Cancelled,
1022 Terminated,
1023 TimedOut,
1024}
1025
1026#[derive(Clone, Debug, Error)]
1032#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
1033pub struct WorkflowTerminalOutcome {
1034 pub kind: WorkflowTerminalKind,
1035 pub workflow_id: String,
1036 pub run_id: Option<String>,
1037 pub reason: String,
1038 pub failure_category: Option<String>,
1039 pub failure_id: Option<String>,
1040 pub exception_type: Option<String>,
1041 pub exception_class: Option<String>,
1042 pub non_retryable: Option<bool>,
1043 pub message: Option<String>,
1044 pub exception: Option<Value>,
1045 pub raw: Value,
1046}
1047
1048#[derive(Clone, Debug, Error)]
1050#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
1051pub struct ActivityTaskRejection {
1052 pub operation: String,
1053 pub status: u16,
1054 pub reason: String,
1055 pub task_id: String,
1056 pub activity_attempt_id: String,
1057 pub cancel_requested: bool,
1058 pub can_continue: Option<bool>,
1059 pub run_closed_reason: Option<String>,
1060 pub body: Value,
1061}
1062
1063#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1065pub enum ActivityOptionsErrorKind {
1066 EmptyTaskQueue,
1067 EmptyRetryPolicy,
1068 InvalidMaxAttempts,
1069 BackoffWithoutRetryBudget,
1070 TooManyBackoffIntervals,
1071 InvalidBackoffCoefficient,
1072 BackoffGenerationTooLarge,
1073 BackoffOverflow,
1074 EmptyNonRetryableErrorType,
1075 TimeoutNotPositive,
1076 MissingTotalTimeout,
1077 TimeoutOverflow,
1078 TimeoutOrder,
1079}
1080
1081#[derive(Clone, Debug, Error, PartialEq, Eq)]
1083#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
1084pub struct ActivityOptionsError {
1085 pub kind: ActivityOptionsErrorKind,
1086 pub field: Option<&'static str>,
1087 pub message: String,
1088}
1089
1090impl ActivityOptionsError {
1091 fn new(
1092 kind: ActivityOptionsErrorKind,
1093 field: Option<&'static str>,
1094 message: impl Into<String>,
1095 ) -> Self {
1096 Self {
1097 kind,
1098 field,
1099 message: message.into(),
1100 }
1101 }
1102}
1103
1104#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1106pub enum ActivityFailureKind {
1107 Failed,
1108 Cancelled,
1109 TimedOut,
1110}
1111
1112#[derive(Clone, Debug, Error, PartialEq)]
1117#[error("activity failed ({reason}): {message}")]
1118pub struct ActivityFailure {
1119 pub kind: ActivityFailureKind,
1120 pub reason: String,
1121 pub message: String,
1122 pub activity_execution_id: Option<String>,
1123 pub activity_attempt_id: Option<String>,
1124 pub activity_type: Option<String>,
1125 pub activity_class: Option<String>,
1126 pub attempt_number: Option<u64>,
1127 pub failure_id: Option<String>,
1128 pub failure_category: Option<String>,
1129 pub timeout_kind: Option<String>,
1130 pub non_retryable: bool,
1131 pub exception_type: Option<String>,
1132 pub exception_class: Option<String>,
1133 pub code: Option<Value>,
1134 pub exception: Option<Value>,
1135}
1136
1137#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1139pub enum ChildWorkflowFailureKind {
1140 Failed,
1141 Cancelled,
1142 Terminated,
1143}
1144
1145#[derive(Clone, Debug, Error)]
1151#[error("child workflow failed ({reason}): {message}")]
1152pub struct ChildWorkflowFailure {
1153 pub kind: ChildWorkflowFailureKind,
1154 pub reason: String,
1155 pub message: String,
1156 pub parent_workflow_id: Option<String>,
1157 pub parent_workflow_run_id: Option<String>,
1158 pub child_workflow_id: Option<String>,
1159 pub child_workflow_run_id: Option<String>,
1160 pub child_workflow_type: Option<String>,
1161 pub failure_id: Option<String>,
1162 pub failure_category: Option<String>,
1163 pub exception_type: Option<String>,
1164 pub exception_class: Option<String>,
1165 pub non_retryable: bool,
1166 pub code: Option<Value>,
1167 pub exception: Option<Value>,
1168}
1169
1170#[derive(Clone, Debug, PartialEq, Eq)]
1172pub struct WorkflowIdentity {
1173 pub workflow_id: Option<String>,
1174 pub run_id: Option<String>,
1175}
1176
1177#[derive(Clone, Debug, PartialEq)]
1179pub struct ChildWorkflowResult {
1180 pub parent: WorkflowIdentity,
1181 pub child: WorkflowIdentity,
1182 pub child_workflow_type: Option<String>,
1183 pub result: Value,
1184}
1185
1186#[derive(Clone, Debug, PartialEq)]
1188pub struct ChildWorkflowAvroResult {
1189 pub parent: WorkflowIdentity,
1190 pub child: WorkflowIdentity,
1191 pub child_workflow_type: Option<String>,
1192 pub result: AvroValue,
1193}
1194
1195#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1197#[serde(untagged)]
1198pub enum SelectionKey {
1199 Index(usize),
1200 Name(String),
1201}
1202
1203impl From<usize> for SelectionKey {
1204 fn from(value: usize) -> Self {
1205 Self::Index(value)
1206 }
1207}
1208
1209impl From<String> for SelectionKey {
1210 fn from(value: String) -> Self {
1211 Self::Name(value)
1212 }
1213}
1214
1215impl From<&str> for SelectionKey {
1216 fn from(value: &str) -> Self {
1217 Self::Name(value.to_string())
1218 }
1219}
1220
1221#[derive(Clone, Debug, Error, PartialEq, Eq)]
1223#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1224pub struct DurableOperationCancelled {
1225 pub selection_group_id: String,
1226 pub member_key: SelectionKey,
1227 pub member_index: usize,
1228 pub operation_kind: String,
1229 pub operation_identity: String,
1230}
1231
1232#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1238pub struct ParallelGroupMetadata {
1239 pub parallel_group_id: String,
1240 pub parallel_group_kind: String,
1241 pub parallel_group_base_sequence: u64,
1242 pub parallel_group_size: usize,
1243 pub parallel_group_index: usize,
1244 #[serde(default, skip_serializing_if = "Option::is_none")]
1245 pub parallel_group_mode: Option<String>,
1246 #[serde(default, skip_serializing_if = "Option::is_none")]
1247 pub selection_member_key: Option<SelectionKey>,
1248 #[serde(default, skip_serializing_if = "Option::is_none")]
1249 pub selection_member_index: Option<usize>,
1250 #[serde(default, skip_serializing_if = "Option::is_none")]
1251 pub selection_member_base_sequence: Option<u64>,
1252 #[serde(default, skip_serializing_if = "Option::is_none")]
1253 pub selection_member_size: Option<usize>,
1254 #[serde(default, skip_serializing_if = "Option::is_none")]
1255 pub selection_member_kind: Option<String>,
1256}
1257
1258#[derive(Clone, Debug, PartialEq)]
1260pub enum ParallelResult {
1261 Activity(Value),
1262 ChildWorkflow(ChildWorkflowResult),
1263 Timer,
1264 Signal(Vec<Value>),
1265 Condition(ConditionWaitResult),
1266 Group(Vec<ParallelResult>),
1267}
1268
1269#[derive(Clone, Debug, PartialEq)]
1271pub enum ParallelAvroResult {
1272 Activity(AvroValue),
1273 ChildWorkflow(ChildWorkflowAvroResult),
1274 Timer,
1275 Signal(Vec<AvroValue>),
1276 Condition(ConditionWaitResult),
1277 Group(Vec<ParallelAvroResult>),
1278}
1279
1280impl ParallelAvroResult {
1281 fn into_json_result(self) -> Result<ParallelResult> {
1282 match self {
1283 Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1284 Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1285 parent: result.parent,
1286 child: result.child,
1287 child_workflow_type: result.child_workflow_type,
1288 result: result.result.into_json()?,
1289 })),
1290 Self::Timer => Ok(ParallelResult::Timer),
1291 Self::Signal(values) => Ok(ParallelResult::Signal(
1292 values
1293 .into_iter()
1294 .map(AvroValue::into_json)
1295 .collect::<Result<Vec<_>>>()?,
1296 )),
1297 Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1298 Self::Group(results) => Ok(ParallelResult::Group(
1299 results
1300 .into_iter()
1301 .map(Self::into_json_result)
1302 .collect::<Result<Vec<_>>>()?,
1303 )),
1304 }
1305 }
1306}
1307
1308#[derive(Clone, Debug, PartialEq)]
1310pub struct ParallelCompletion {
1311 pub member_path: Vec<usize>,
1312 pub result: ParallelResult,
1313}
1314
1315#[derive(Debug, Error)]
1323#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1324pub struct ParallelFailure {
1325 pub group_id: String,
1326 pub member_path: Vec<usize>,
1327 pub group_path: Vec<ParallelGroupMetadata>,
1328 pub completed: Vec<ParallelCompletion>,
1329 #[source]
1330 pub cause: Box<Error>,
1331}
1332
1333#[derive(Clone, Debug, Error, PartialEq, Eq)]
1335#[error("invalid deterministic parallel group ({reason}): {message}")]
1336pub struct ParallelGroupError {
1337 pub reason: &'static str,
1338 pub member_path: Vec<usize>,
1339 pub message: String,
1340}
1341
1342#[derive(Clone, Debug, Error, PartialEq, Eq)]
1344#[error("workflow cancellation was requested")]
1345pub struct WorkflowCancellationRequested;
1346
1347#[derive(Debug, Error)]
1349#[error(
1350 "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1351)]
1352pub struct SagaCompensationFailure {
1353 pub initiating_failure: Box<Error>,
1354 pub compensation_failure: Box<Error>,
1355 pub compensation_activity_type: String,
1356 pub compensation_registration_order: usize,
1357}
1358
1359#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1361pub enum ParentClosePolicy {
1362 #[default]
1363 Abandon,
1364 RequestCancel,
1366 RequestCancellation,
1368 Terminate,
1369}
1370
1371impl ParentClosePolicy {
1372 fn as_str(self) -> &'static str {
1373 match self {
1374 Self::Abandon => "abandon",
1375 Self::RequestCancel => "request_cancel",
1376 Self::RequestCancellation => "request_cancellation",
1377 Self::Terminate => "terminate",
1378 }
1379 }
1380}
1381
1382#[non_exhaustive]
1384#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1385pub enum CancellationPolicy {
1386 TryCancel,
1388 WaitCancellationCompleted,
1390 #[default]
1392 Abandon,
1393}
1394
1395impl CancellationPolicy {
1396 fn as_str(self) -> &'static str {
1397 match self {
1398 Self::TryCancel => "try_cancel",
1399 Self::WaitCancellationCompleted => "wait_cancellation_completed",
1400 Self::Abandon => "abandon",
1401 }
1402 }
1403}
1404
1405#[derive(Clone, Debug, Default, PartialEq, Eq)]
1407pub struct ChildWorkflowRetryPolicy {
1408 pub max_attempts: Option<u32>,
1409 pub backoff_seconds: Vec<u64>,
1410 pub non_retryable_error_types: Vec<String>,
1411}
1412
1413#[derive(Clone, Debug, PartialEq, Eq)]
1417pub struct ChildWorkflowOptions {
1418 pub task_queue: String,
1419 pub parent_close_policy: ParentClosePolicy,
1420 pub cancellation_policy: CancellationPolicy,
1421 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1422 pub execution_timeout_seconds: Option<u64>,
1423 pub run_timeout_seconds: Option<u64>,
1424}
1425
1426impl ChildWorkflowOptions {
1427 pub fn new(task_queue: impl Into<String>) -> Self {
1428 Self {
1429 task_queue: task_queue.into(),
1430 parent_close_policy: ParentClosePolicy::Abandon,
1431 cancellation_policy: CancellationPolicy::Abandon,
1432 retry_policy: None,
1433 execution_timeout_seconds: None,
1434 run_timeout_seconds: None,
1435 }
1436 }
1437
1438 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1439 self.parent_close_policy = policy;
1440 self
1441 }
1442
1443 pub fn cancellation_policy(mut self, policy: CancellationPolicy) -> Self {
1444 self.cancellation_policy = policy;
1445 self
1446 }
1447
1448 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1449 self.retry_policy = Some(policy);
1450 self
1451 }
1452
1453 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1454 self.execution_timeout_seconds = Some(seconds);
1455 self
1456 }
1457
1458 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1459 self.run_timeout_seconds = Some(seconds);
1460 self
1461 }
1462}
1463
1464#[derive(Clone, Debug, PartialEq, Eq)]
1466pub enum ActivityBackoff {
1467 Explicit(Vec<Duration>),
1470 Exponential {
1472 initial_interval: Duration,
1473 coefficient: u32,
1474 maximum_interval: Option<Duration>,
1475 },
1476}
1477
1478#[derive(Clone, Debug, Default, PartialEq, Eq)]
1480pub struct ActivityRetryPolicy {
1481 pub max_attempts: Option<u32>,
1482 pub backoff: Option<ActivityBackoff>,
1483 pub non_retryable_error_types: Vec<String>,
1484}
1485
1486impl ActivityRetryPolicy {
1487 pub fn new(max_attempts: u32) -> Self {
1489 Self {
1490 max_attempts: Some(max_attempts),
1491 ..Self::default()
1492 }
1493 }
1494
1495 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1496 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1497 self
1498 }
1499
1500 pub fn exponential_backoff(
1501 mut self,
1502 initial_interval: Duration,
1503 coefficient: u32,
1504 maximum_interval: Option<Duration>,
1505 ) -> Self {
1506 self.backoff = Some(ActivityBackoff::Exponential {
1507 initial_interval,
1508 coefficient,
1509 maximum_interval,
1510 });
1511 self
1512 }
1513
1514 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1515 self.non_retryable_error_types.push(error_type.into());
1516 self
1517 }
1518
1519 pub fn non_retryable_error_types(
1520 mut self,
1521 error_types: impl IntoIterator<Item = impl Into<String>>,
1522 ) -> Self {
1523 self.non_retryable_error_types
1524 .extend(error_types.into_iter().map(Into::into));
1525 self
1526 }
1527}
1528
1529#[derive(Clone, Debug, Default, PartialEq, Eq)]
1534pub struct ActivityOptions {
1535 pub task_queue: Option<String>,
1536 pub retry_policy: Option<ActivityRetryPolicy>,
1537 pub start_to_close_timeout: Option<Duration>,
1538 pub schedule_to_start_timeout: Option<Duration>,
1539 pub schedule_to_close_timeout: Option<Duration>,
1540 pub heartbeat_timeout: Option<Duration>,
1541 pub cancellation_policy: Option<CancellationPolicy>,
1542}
1543
1544impl ActivityOptions {
1545 pub fn new() -> Self {
1546 Self::default()
1547 }
1548
1549 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1550 self.task_queue = Some(task_queue.into());
1551 self
1552 }
1553
1554 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1555 self.retry_policy = Some(policy);
1556 self
1557 }
1558
1559 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1560 self.start_to_close_timeout = Some(timeout);
1561 self
1562 }
1563
1564 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1565 self.schedule_to_start_timeout = Some(timeout);
1566 self
1567 }
1568
1569 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1570 self.schedule_to_close_timeout = Some(timeout);
1571 self
1572 }
1573
1574 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1575 self.heartbeat_timeout = Some(timeout);
1576 self
1577 }
1578
1579 pub fn cancellation_policy(mut self, policy: CancellationPolicy) -> Self {
1580 self.cancellation_policy = Some(policy);
1581 self
1582 }
1583
1584 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1585 if self.cancellation_policy == Some(CancellationPolicy::Abandon)
1586 && self.schedule_to_close_timeout.is_none()
1587 {
1588 return Err(ActivityOptionsError::new(
1589 ActivityOptionsErrorKind::MissingTotalTimeout,
1590 Some("schedule_to_close_timeout"),
1591 "remote Activity Abandon requires a finite positive total timeout",
1592 ));
1593 }
1594 if self
1595 .task_queue
1596 .as_deref()
1597 .is_some_and(|queue| queue.trim().is_empty())
1598 {
1599 return Err(ActivityOptionsError::new(
1600 ActivityOptionsErrorKind::EmptyTaskQueue,
1601 Some("task_queue"),
1602 "task_queue must not be empty",
1603 ));
1604 }
1605
1606 for (field, value) in [
1607 ("start_to_close_timeout", self.start_to_close_timeout),
1608 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1609 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1610 ("heartbeat_timeout", self.heartbeat_timeout),
1611 ] {
1612 if value.is_some_and(|value| value.is_zero()) {
1613 return Err(ActivityOptionsError::new(
1614 ActivityOptionsErrorKind::TimeoutNotPositive,
1615 Some(field),
1616 format!("{field} must be positive"),
1617 ));
1618 }
1619 }
1620
1621 validate_timeout_order(
1622 "heartbeat_timeout",
1623 self.heartbeat_timeout,
1624 "start_to_close_timeout",
1625 self.start_to_close_timeout,
1626 )?;
1627 validate_timeout_order(
1628 "start_to_close_timeout",
1629 self.start_to_close_timeout,
1630 "schedule_to_close_timeout",
1631 self.schedule_to_close_timeout,
1632 )?;
1633 validate_timeout_order(
1634 "schedule_to_start_timeout",
1635 self.schedule_to_start_timeout,
1636 "schedule_to_close_timeout",
1637 self.schedule_to_close_timeout,
1638 )?;
1639
1640 Ok(ValidatedActivityOptions {
1641 task_queue: self.task_queue.clone(),
1642 retry_policy: self
1643 .retry_policy
1644 .as_ref()
1645 .map(validate_activity_retry_policy)
1646 .transpose()?,
1647 start_to_close_timeout: timeout_seconds(
1648 "start_to_close_timeout",
1649 self.start_to_close_timeout,
1650 )?,
1651 schedule_to_start_timeout: timeout_seconds(
1652 "schedule_to_start_timeout",
1653 self.schedule_to_start_timeout,
1654 )?,
1655 schedule_to_close_timeout: timeout_seconds(
1656 "schedule_to_close_timeout",
1657 self.schedule_to_close_timeout,
1658 )?,
1659 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1660 cancellation_policy: self.cancellation_policy,
1661 })
1662 }
1663}
1664
1665pub enum ParallelOperation {
1671 Activity {
1672 activity_type: String,
1673 options: ActivityOptions,
1674 arguments: Result<AvroValue>,
1675 },
1676 ChildWorkflow {
1677 workflow_type: String,
1678 options: ChildWorkflowOptions,
1679 arguments: Result<AvroValue>,
1680 },
1681 Timer(Duration),
1682 Signal(String),
1683 Condition {
1684 options: ConditionWaitOptions,
1685 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1686 },
1687 Group(Vec<ParallelOperation>),
1688}
1689
1690impl ParallelOperation {
1691 pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1692 Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1693 }
1694
1695 pub fn activity_with_options<T: Serialize>(
1696 activity_type: impl Into<String>,
1697 options: ActivityOptions,
1698 args: T,
1699 ) -> Self {
1700 Self::Activity {
1701 activity_type: activity_type.into(),
1702 options,
1703 arguments: AvroValue::from_serialize(&args),
1704 }
1705 }
1706
1707 pub fn child_workflow<T: Serialize>(
1708 workflow_type: impl Into<String>,
1709 options: ChildWorkflowOptions,
1710 args: T,
1711 ) -> Self {
1712 Self::ChildWorkflow {
1713 workflow_type: workflow_type.into(),
1714 options,
1715 arguments: AvroValue::from_serialize(&args),
1716 }
1717 }
1718
1719 pub fn timer(duration: Duration) -> Self {
1720 Self::Timer(duration)
1721 }
1722
1723 pub fn signal(signal_name: impl Into<String>) -> Self {
1724 Self::Signal(signal_name.into())
1725 }
1726
1727 pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1728 where
1729 F: Fn() -> Result<bool> + Send + 'static,
1730 {
1731 Self::Condition {
1732 options,
1733 predicate: Box::new(predicate),
1734 }
1735 }
1736
1737 pub fn group(operations: Vec<ParallelOperation>) -> Self {
1738 Self::Group(operations)
1739 }
1740}
1741
1742#[derive(Clone, Debug)]
1743struct ValidatedActivityOptions {
1744 task_queue: Option<String>,
1745 retry_policy: Option<Value>,
1746 start_to_close_timeout: Option<u64>,
1747 schedule_to_start_timeout: Option<u64>,
1748 schedule_to_close_timeout: Option<u64>,
1749 heartbeat_timeout: Option<u64>,
1750 cancellation_policy: Option<CancellationPolicy>,
1751}
1752
1753fn validate_timeout_order(
1754 smaller_name: &'static str,
1755 smaller: Option<Duration>,
1756 larger_name: &'static str,
1757 larger: Option<Duration>,
1758) -> std::result::Result<(), ActivityOptionsError> {
1759 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1760 return Err(ActivityOptionsError::new(
1761 ActivityOptionsErrorKind::TimeoutOrder,
1762 Some(smaller_name),
1763 format!("{smaller_name} must be <= {larger_name}"),
1764 ));
1765 }
1766 Ok(())
1767}
1768
1769fn timeout_seconds(
1770 field: &'static str,
1771 value: Option<Duration>,
1772) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1773 value
1774 .map(|value| {
1775 activity_protocol_seconds(value).ok_or_else(|| {
1776 ActivityOptionsError::new(
1777 ActivityOptionsErrorKind::TimeoutOverflow,
1778 Some(field),
1779 format!("{field} is too large for the worker protocol"),
1780 )
1781 })
1782 })
1783 .transpose()
1784}
1785
1786fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1787 value
1788 .as_secs()
1789 .checked_add(u64::from(value.subsec_nanos() > 0))
1790}
1791
1792fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1793 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1794}
1795
1796fn validate_activity_retry_policy(
1797 policy: &ActivityRetryPolicy,
1798) -> std::result::Result<Value, ActivityOptionsError> {
1799 if policy.max_attempts.is_none()
1800 && policy.backoff.is_none()
1801 && policy.non_retryable_error_types.is_empty()
1802 {
1803 return Err(ActivityOptionsError::new(
1804 ActivityOptionsErrorKind::EmptyRetryPolicy,
1805 Some("retry_policy"),
1806 "retry_policy must configure at least one field",
1807 ));
1808 }
1809 if policy.max_attempts == Some(0) {
1810 return Err(ActivityOptionsError::new(
1811 ActivityOptionsErrorKind::InvalidMaxAttempts,
1812 Some("retry_policy.max_attempts"),
1813 "max_attempts must be >= 1",
1814 ));
1815 }
1816 if policy
1817 .non_retryable_error_types
1818 .iter()
1819 .any(|error_type| error_type.trim().is_empty())
1820 {
1821 return Err(ActivityOptionsError::new(
1822 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1823 Some("retry_policy.non_retryable_error_types"),
1824 "non_retryable_error_types must not contain empty values",
1825 ));
1826 }
1827
1828 let backoff_seconds = match &policy.backoff {
1829 None => None,
1830 Some(backoff) => {
1831 let max_attempts = policy.max_attempts.ok_or_else(|| {
1832 ActivityOptionsError::new(
1833 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1834 Some("retry_policy.backoff"),
1835 "backoff requires max_attempts",
1836 )
1837 })?;
1838 let retry_count = max_attempts.saturating_sub(1) as usize;
1839 let intervals = match backoff {
1840 ActivityBackoff::Explicit(intervals) => {
1841 if intervals.len() > retry_count {
1842 return Err(ActivityOptionsError::new(
1843 ActivityOptionsErrorKind::TooManyBackoffIntervals,
1844 Some("retry_policy.backoff"),
1845 "backoff interval count must not exceed max_attempts - 1",
1846 ));
1847 }
1848 intervals.clone()
1849 }
1850 ActivityBackoff::Exponential {
1851 initial_interval,
1852 coefficient,
1853 maximum_interval,
1854 } => {
1855 if *coefficient < 1 {
1856 return Err(ActivityOptionsError::new(
1857 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1858 Some("retry_policy.backoff.coefficient"),
1859 "backoff coefficient must be >= 1",
1860 ));
1861 }
1862 if retry_count > 10_000 {
1863 return Err(ActivityOptionsError::new(
1864 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1865 Some("retry_policy.max_attempts"),
1866 "generated backoff supports at most 10000 retry intervals",
1867 ));
1868 }
1869 let mut current = *initial_interval;
1870 let mut intervals = Vec::with_capacity(retry_count);
1871 for _ in 0..retry_count {
1872 let interval = maximum_interval
1873 .map(|maximum| current.min(maximum))
1874 .unwrap_or(current);
1875 intervals.push(interval);
1876 if maximum_interval.is_some_and(|maximum| interval == maximum) {
1877 break;
1878 }
1879 current = current.checked_mul(*coefficient).ok_or_else(|| {
1880 ActivityOptionsError::new(
1881 ActivityOptionsErrorKind::BackoffOverflow,
1882 Some("retry_policy.backoff"),
1883 "generated backoff interval overflowed",
1884 )
1885 })?;
1886 }
1887 intervals
1888 }
1889 };
1890 Some(
1891 intervals
1892 .into_iter()
1893 .map(|interval| {
1894 activity_protocol_seconds(interval).ok_or_else(|| {
1895 ActivityOptionsError::new(
1896 ActivityOptionsErrorKind::BackoffOverflow,
1897 Some("retry_policy.backoff"),
1898 "backoff interval is too large for the worker protocol",
1899 )
1900 })
1901 })
1902 .collect::<std::result::Result<Vec<_>, _>>()?,
1903 )
1904 }
1905 };
1906
1907 let mut encoded = serde_json::Map::new();
1908 if let Some(max_attempts) = policy.max_attempts {
1909 encoded.insert("max_attempts".to_string(), json!(max_attempts));
1910 }
1911 if let Some(backoff_seconds) = backoff_seconds {
1912 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1913 }
1914 if !policy.non_retryable_error_types.is_empty() {
1915 let mut canonical_error_types = Vec::new();
1916 for error_type in policy
1917 .non_retryable_error_types
1918 .iter()
1919 .map(|error_type| error_type.trim())
1920 {
1921 if !canonical_error_types.contains(&error_type) {
1922 canonical_error_types.push(error_type);
1923 }
1924 }
1925 encoded.insert(
1926 "non_retryable_error_types".to_string(),
1927 json!(canonical_error_types),
1928 );
1929 }
1930 Ok(Value::Object(encoded))
1931}
1932
1933#[derive(Clone, Debug, Error)]
1936#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1937pub struct ReplayFailure {
1938 pub reason: String,
1939 pub sequence: Option<u64>,
1940 pub expected: Option<String>,
1941 pub actual: Option<String>,
1942 pub message: String,
1943}
1944
1945impl ReplayFailure {
1946 fn new(
1947 reason: impl Into<String>,
1948 sequence: Option<u64>,
1949 expected: Option<String>,
1950 actual: Option<String>,
1951 message: impl Into<String>,
1952 ) -> Self {
1953 Self {
1954 reason: reason.into(),
1955 sequence,
1956 expected,
1957 actual,
1958 message: message.into(),
1959 }
1960 }
1961}
1962
1963#[derive(Clone, Debug, Error)]
1965#[error("query failed ({reason}, HTTP {status}): {message}")]
1966pub struct QueryFailure {
1967 pub status: u16,
1968 pub reason: String,
1969 pub message: String,
1970 pub body: Value,
1971}
1972
1973#[derive(Clone, Debug, Error)]
1975#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1976pub struct ProtocolFailure {
1977 pub status: u16,
1978 pub reason: String,
1979 pub message: String,
1980 pub supported_version: Option<String>,
1981 pub requested_version: Option<String>,
1982 pub body: Value,
1983}
1984
1985#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1986pub struct PayloadEnvelope {
1987 pub codec: String,
1988 pub blob: String,
1989}
1990
1991impl PayloadEnvelope {
1992 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1993 encode_payload(value, DEFAULT_CODEC)
1994 }
1995
1996 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1999 encode_avro_value(value)
2000 }
2001}
2002
2003#[derive(Clone, Debug)]
2005pub enum AvroValue {
2006 Null,
2007 Boolean(bool),
2008 Long(i64),
2009 Double(f64),
2010 Bytes(Vec<u8>),
2011 String(String),
2012 Array(Vec<AvroValue>),
2013 Map(BTreeMap<String, AvroValue>),
2014}
2015
2016impl PartialEq for AvroValue {
2017 fn eq(&self, other: &Self) -> bool {
2018 match (self, other) {
2019 (Self::Null, Self::Null) => true,
2020 (Self::Boolean(left), Self::Boolean(right)) => left == right,
2021 (Self::Long(left), Self::Long(right)) => left == right,
2022 (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
2023 (Self::Bytes(left), Self::Bytes(right)) => left == right,
2024 (Self::String(left), Self::String(right)) => left == right,
2025 (Self::Array(left), Self::Array(right)) => left == right,
2026 (Self::Map(left), Self::Map(right)) => left == right,
2027 _ => false,
2028 }
2029 }
2030}
2031
2032impl AvroValue {
2033 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
2034 Self::from_serde_value(
2035 serde_value::to_value(value).map_err(|error| {
2036 Error::Codec(format!("could not adapt value for Avro: {error}"))
2037 })?,
2038 )
2039 }
2040
2041 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
2042 use serde_value::Value as SerdeValue;
2043
2044 match value {
2045 SerdeValue::Unit => Ok(Self::Null),
2046 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
2047 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
2048 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
2049 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
2050 SerdeValue::I64(value) => Ok(Self::Long(value)),
2051 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
2052 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
2053 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
2054 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
2055 Error::Codec(
2056 "integer_overflow: Avro Value long must be within signed 64-bit range"
2057 .to_string(),
2058 )
2059 }),
2060 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
2061 SerdeValue::F64(value) => Self::finite_double(value),
2062 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
2063 SerdeValue::String(value) => Ok(Self::String(value)),
2064 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
2065 SerdeValue::Option(None) => Ok(Self::Null),
2066 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
2067 Self::from_serde_value(*value)
2068 }
2069 SerdeValue::Seq(values) => values
2070 .into_iter()
2071 .map(Self::from_serde_value)
2072 .collect::<Result<Vec<_>>>()
2073 .map(Self::Array),
2074 SerdeValue::Map(values) => values
2075 .into_iter()
2076 .map(|(key, value)| {
2077 let SerdeValue::String(key) = key else {
2078 return Err(Error::Codec(
2079 "invalid_map_key: Avro Value map keys must be strings".to_string(),
2080 ));
2081 };
2082
2083 Ok((key, Self::from_serde_value(value)?))
2084 })
2085 .collect::<Result<BTreeMap<_, _>>>()
2086 .map(Self::Map),
2087 }
2088 }
2089
2090 fn finite_double(value: f64) -> Result<Self> {
2091 if !value.is_finite() {
2092 return Err(Error::Codec(
2093 "non_finite_float: Avro Value doubles must be finite".to_string(),
2094 ));
2095 }
2096
2097 Ok(Self::Double(value))
2098 }
2099
2100 fn into_json(self) -> Result<Value> {
2101 match self {
2102 Self::Null => Ok(Value::Null),
2103 Self::Boolean(value) => Ok(Value::Bool(value)),
2104 Self::Long(value) => Ok(Value::Number(value.into())),
2105 Self::Double(value) => serde_json::Number::from_f64(value)
2106 .map(Value::Number)
2107 .ok_or_else(|| {
2108 Error::Codec(
2109 "non_finite_float: decoded Avro Value double is not finite".to_string(),
2110 )
2111 }),
2112 Self::Bytes(value) => Ok(json!({
2113 "$type": "bytes",
2114 "base64": BASE64.encode(value),
2115 })),
2116 Self::String(value) => Ok(Value::String(value)),
2117 Self::Array(values) => values
2118 .into_iter()
2119 .map(Self::into_json)
2120 .collect::<Result<Vec<_>>>()
2121 .map(Value::Array),
2122 Self::Map(values) => values
2123 .into_iter()
2124 .map(|(key, value)| Ok((key, value.into_json()?)))
2125 .collect::<Result<serde_json::Map<_, _>>>()
2126 .map(Value::Object),
2127 }
2128 }
2129
2130 fn into_serde_value(self) -> serde_value::Value {
2131 use serde_value::Value as SerdeValue;
2132
2133 match self {
2134 Self::Null => SerdeValue::Unit,
2135 Self::Boolean(value) => SerdeValue::Bool(value),
2136 Self::Long(value) => SerdeValue::I64(value),
2137 Self::Double(value) => SerdeValue::F64(value),
2138 Self::Bytes(value) => SerdeValue::Bytes(value),
2139 Self::String(value) => SerdeValue::String(value),
2140 Self::Array(values) => {
2141 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
2142 }
2143 Self::Map(values) => SerdeValue::Map(
2144 values
2145 .into_iter()
2146 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
2147 .collect(),
2148 ),
2149 }
2150 }
2151
2152 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
2153 self.into_serde_value().deserialize_into().map_err(|error| {
2154 Error::Codec(format!(
2155 "avro_value_type_mismatch: could not adapt decoded value: {error}"
2156 ))
2157 })
2158 }
2159}
2160
2161impl Serialize for AvroValue {
2162 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
2163 where
2164 S: Serializer,
2165 {
2166 match self {
2167 Self::Null => serializer.serialize_unit(),
2168 Self::Boolean(value) => serializer.serialize_bool(*value),
2169 Self::Long(value) => serializer.serialize_i64(*value),
2170 Self::Double(value) => serializer.serialize_f64(*value),
2171 Self::Bytes(value) => serializer.serialize_bytes(value),
2172 Self::String(value) => serializer.serialize_str(value),
2173 Self::Array(values) => {
2174 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2175 for value in values {
2176 sequence.serialize_element(value)?;
2177 }
2178 sequence.end()
2179 }
2180 Self::Map(values) => {
2181 let mut map = serializer.serialize_map(Some(values.len()))?;
2182 for (key, value) in values {
2183 map.serialize_entry(key, value)?;
2184 }
2185 map.end()
2186 }
2187 }
2188 }
2189}
2190
2191pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2192 let datum = avro_value_to_datum(value)?;
2193 let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2194 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2195 let mut bytes = Vec::with_capacity(datum.len() + 10);
2196 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2197 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2198 bytes.extend_from_slice(&datum);
2199 Ok(PayloadEnvelope {
2200 codec: DEFAULT_CODEC.to_string(),
2201 blob: BASE64.encode(bytes),
2202 })
2203}
2204
2205pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2206 if envelope.codec != DEFAULT_CODEC {
2207 return Err(unsupported_payload_codec(&envelope.codec));
2208 }
2209 decode_avro_value_blob(&envelope.blob)
2210}
2211
2212pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2213 let blob = match codec {
2214 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2215 other => return Err(unsupported_payload_codec(other)),
2216 };
2217
2218 Ok(PayloadEnvelope {
2219 codec: codec.to_string(),
2220 blob,
2221 })
2222}
2223
2224pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2225 match envelope.codec.as_str() {
2226 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2227 other => Err(unsupported_payload_codec(other)),
2228 }
2229}
2230
2231fn handler_type_error<T>(
2232 handler_kind: HandlerKind,
2233 handler_name: &str,
2234 value_kind: HandlerValueKind,
2235 message: impl Into<String>,
2236) -> Error {
2237 Error::HandlerType {
2238 handler_kind,
2239 handler_name: handler_name.to_string(),
2240 value_kind,
2241 rust_type: type_name::<T>(),
2242 message: message.into(),
2243 }
2244}
2245
2246fn decode_handler_input<T: DeserializeOwned>(
2247 arguments: AvroValue,
2248 handler_kind: HandlerKind,
2249 handler_name: &str,
2250) -> Result<T> {
2251 let argument = match arguments {
2252 AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2253 arguments.pop().expect("one typed handler argument")
2254 }
2255 AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2256 AvroValue::Array(arguments) => {
2257 return Err(handler_type_error::<T>(
2258 handler_kind,
2259 handler_name,
2260 HandlerValueKind::Input,
2261 format!(
2262 "typed handlers accept one request value, but the task carried {} arguments",
2263 arguments.len()
2264 ),
2265 ));
2266 }
2267 argument => argument,
2268 };
2269
2270 argument.deserialize().map_err(|error| {
2271 handler_type_error::<T>(
2272 handler_kind,
2273 handler_name,
2274 HandlerValueKind::Input,
2275 error.to_string(),
2276 )
2277 })
2278}
2279
2280fn encode_handler_result<T: Serialize>(
2281 result: &T,
2282 handler_kind: HandlerKind,
2283 handler_name: &str,
2284) -> Result<AvroValue> {
2285 AvroValue::from_serialize(result).map_err(|error| {
2286 handler_type_error::<T>(
2287 handler_kind,
2288 handler_name,
2289 HandlerValueKind::Result,
2290 error.to_string(),
2291 )
2292 })
2293}
2294
2295fn decode_handler_result<T: DeserializeOwned>(
2296 result: AvroValue,
2297 handler_kind: HandlerKind,
2298 handler_name: &str,
2299) -> Result<T> {
2300 result.deserialize().map_err(|error| {
2301 handler_type_error::<T>(
2302 handler_kind,
2303 handler_name,
2304 HandlerValueKind::Result,
2305 error.to_string(),
2306 )
2307 })
2308}
2309
2310#[cfg(test)]
2311fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2312 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2313}
2314
2315fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2316 validate_payload_codec(fallback_codec)?;
2317
2318 if value.is_null() {
2319 return Ok(Value::Null);
2320 }
2321
2322 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2323 return decode_blob(blob, codec);
2324 }
2325
2326 if let Some(blob) = value.as_str() {
2327 return decode_blob(blob, fallback_codec);
2328 }
2329
2330 Err(untagged_payload_value())
2331}
2332
2333fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2334 let envelope = match codec {
2335 DEFAULT_CODEC => encode_avro_value(value)?,
2336 other => return Err(unsupported_payload_codec(other)),
2337 };
2338 Ok(serde_json::to_value(envelope)?)
2339}
2340
2341fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2342 validate_payload_codec(fallback_codec)?;
2343
2344 if value.is_null() {
2345 return Ok(AvroValue::Null);
2346 }
2347
2348 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2349 validate_payload_codec(codec)?;
2350 return decode_avro_value_blob(blob);
2351 }
2352
2353 if let Some(blob) = value.as_str() {
2354 return match fallback_codec {
2355 DEFAULT_CODEC => decode_avro_value_blob(blob),
2356 other => Err(unsupported_payload_codec(other)),
2357 };
2358 }
2359
2360 Err(untagged_payload_value())
2361}
2362
2363fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2364 match value {
2365 AvroValue::Null => AvroValue::Array(Vec::new()),
2366 AvroValue::Array(_) => value,
2367 other => AvroValue::Array(vec![other]),
2368 }
2369}
2370
2371fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2372 match codec {
2373 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2374 other => Err(unsupported_payload_codec(other)),
2375 }
2376}
2377
2378fn validate_payload_codec(codec: &str) -> Result<()> {
2379 match codec {
2380 DEFAULT_CODEC => Ok(()),
2381 MISSING_TASK_PAYLOAD_CODEC => {
2382 Err(invalid_task_payload_codec("task payload_codec is missing"))
2383 }
2384 NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2385 NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2386 "task payload_codec must be a string",
2387 )),
2388 other => Err(unsupported_payload_codec(other)),
2389 }
2390}
2391
2392fn invalid_task_payload_codec(reason: &str) -> Error {
2393 Error::Codec(format!(
2394 "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2395 ))
2396}
2397
2398fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2399 let Some(object) = value.as_object() else {
2400 return Ok(None);
2401 };
2402 if !object.contains_key("codec") && !object.contains_key("blob") {
2403 return Ok(None);
2404 }
2405
2406 let codec = object
2407 .get("codec")
2408 .and_then(Value::as_str)
2409 .ok_or_else(invalid_payload_envelope)?;
2410 validate_payload_codec(codec)?;
2411 let blob = object
2412 .get("blob")
2413 .and_then(Value::as_str)
2414 .ok_or_else(invalid_payload_envelope)?;
2415 Ok(Some((codec, blob)))
2416}
2417
2418fn invalid_payload_envelope() -> Error {
2419 Error::Codec(
2420 "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2421 .to_string(),
2422 )
2423}
2424
2425fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2426 for command in commands {
2427 let Some(command) = command.as_object() else {
2428 continue;
2429 };
2430 let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2431 continue;
2432 };
2433 let Some(payload_field) = workflow_command_payload_field(command_type) else {
2434 continue;
2435 };
2436
2437 if let Some(codec) = command.get("payload_codec") {
2438 let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2439 validate_payload_codec(codec)?;
2440 }
2441
2442 let payload = command
2443 .get(payload_field)
2444 .ok_or_else(invalid_payload_envelope)?;
2445 if runtime_payloads::Reference::parse(payload)?.is_none() {
2446 validate_outbound_payload_envelope(payload)?;
2447 }
2448 }
2449 Ok(())
2450}
2451
2452fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2453 if commands.iter().any(|command| {
2454 command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2455 && command
2456 .get("condition_wait_occurrence_id")
2457 .and_then(Value::as_str)
2458 .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2459 }) {
2460 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2461 } else if commands.iter().any(|command| {
2462 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2463 && command.get("attribute_types").is_some()
2464 }) {
2465 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2466 } else if commands
2467 .iter()
2468 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2469 {
2470 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2471 } else if commands
2472 .iter()
2473 .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2474 {
2475 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2476 } else if commands.iter().any(|command| {
2477 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2478 }) {
2479 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2480 } else {
2481 WORKER_PROTOCOL_VERSION
2482 }
2483}
2484
2485fn workflow_completion_protocol_version_with_message_streams(
2486 commands: &[Value],
2487 has_message_stream_metadata: bool,
2488) -> &'static str {
2489 let command_protocol = workflow_completion_protocol_version(commands);
2490 if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2491 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2492 } else {
2493 command_protocol
2494 }
2495}
2496
2497fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2498 match command_type {
2499 "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2500 "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2501 "start_service_operation" => Some("request_payload"),
2502 "upsert_memo" => Some("entries"),
2503 _ => None,
2504 }
2505}
2506
2507fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2508 let Some((codec, blob)) = payload_envelope_parts(value)? else {
2509 return Err(untagged_payload_value());
2510 };
2511 validate_payload_codec(codec)?;
2512 decode_avro_value_blob(blob)?;
2513 Ok(())
2514}
2515
2516fn unsupported_payload_codec(codec: &str) -> Error {
2517 Error::Codec(format!(
2518 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2519 ))
2520}
2521
2522fn untagged_payload_value() -> Error {
2523 Error::Codec(
2524 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2525 .to_string(),
2526 )
2527}
2528
2529fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2530 let bytes = BASE64.decode(blob).map_err(|err| {
2531 Error::Codec(format!(
2532 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2533 ))
2534 })?;
2535
2536 if serde_json::from_slice::<Value>(&bytes).is_ok() {
2537 return Err(unsupported_payload_codec("json"));
2538 }
2539
2540 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2541 return Err(Error::Codec(
2542 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2543 ));
2544 }
2545
2546 let fingerprint: [u8; 8] = bytes[2..10]
2547 .try_into()
2548 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2549 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2550 return Err(Error::Codec(format!(
2551 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2552 fingerprint
2553 .iter()
2554 .map(|byte| format!("{byte:02x}"))
2555 .collect::<String>()
2556 )));
2557 }
2558
2559 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2560 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2564 if datum_reader.truncated {
2565 return Err(Error::Codec(
2566 "invalid_payload_framing: truncated Avro Value datum".to_string(),
2567 ));
2568 }
2569 let datum = datum.map_err(|err| {
2570 Error::Codec(format!(
2571 "invalid_payload_framing: malformed Avro Value datum: {err}"
2572 ))
2573 })?;
2574 if datum_reader.remaining() != 0 {
2575 return Err(Error::Codec(format!(
2576 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2577 datum_reader.remaining()
2578 )));
2579 }
2580 avro_value_from_datum(datum)
2581}
2582
2583struct StrictAvroDatumReader<'a> {
2584 bytes: &'a [u8],
2585 offset: usize,
2586 truncated: bool,
2587}
2588
2589impl<'a> StrictAvroDatumReader<'a> {
2590 fn new(bytes: &'a [u8]) -> Self {
2591 Self {
2592 bytes,
2593 offset: 0,
2594 truncated: false,
2595 }
2596 }
2597
2598 fn remaining(&self) -> usize {
2599 self.bytes.len() - self.offset
2600 }
2601}
2602
2603impl Read for StrictAvroDatumReader<'_> {
2604 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2605 let count = buffer.len().min(self.remaining());
2606 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2607 self.offset += count;
2608 if count < buffer.len() {
2609 self.truncated = true;
2610 }
2611
2612 Ok(count)
2613 }
2614}
2615
2616fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2617 let branch = match value {
2618 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2619 AvroValue::Boolean(value) => AvroDatum::Union(
2620 1,
2621 Box::new(AvroDatum::Record(vec![(
2622 "boolean".to_string(),
2623 AvroDatum::Boolean(*value),
2624 )])),
2625 ),
2626 AvroValue::Long(value) => AvroDatum::Union(
2627 2,
2628 Box::new(AvroDatum::Record(vec![(
2629 "long".to_string(),
2630 AvroDatum::Long(*value),
2631 )])),
2632 ),
2633 AvroValue::Double(value) => {
2634 if !value.is_finite() {
2635 return Err(Error::Codec(
2636 "non_finite_float: Avro Value doubles must be finite".to_string(),
2637 ));
2638 }
2639 AvroDatum::Union(
2640 3,
2641 Box::new(AvroDatum::Record(vec![(
2642 "double".to_string(),
2643 AvroDatum::Double(*value),
2644 )])),
2645 )
2646 }
2647 AvroValue::Bytes(value) => AvroDatum::Union(
2648 4,
2649 Box::new(AvroDatum::Record(vec![(
2650 "bytes".to_string(),
2651 AvroDatum::Bytes(value.clone()),
2652 )])),
2653 ),
2654 AvroValue::String(value) => AvroDatum::Union(
2655 5,
2656 Box::new(AvroDatum::Record(vec![(
2657 "string".to_string(),
2658 AvroDatum::String(value.clone()),
2659 )])),
2660 ),
2661 AvroValue::Array(values) => AvroDatum::Union(
2662 6,
2663 Box::new(AvroDatum::Record(vec![(
2664 "items".to_string(),
2665 AvroDatum::Array(
2666 values
2667 .iter()
2668 .map(avro_value_to_datum)
2669 .collect::<Result<Vec<_>>>()?,
2670 ),
2671 )])),
2672 ),
2673 AvroValue::Map(values) => AvroDatum::Union(
2674 7,
2675 Box::new(AvroDatum::Record(vec![(
2676 "entries".to_string(),
2677 AvroDatum::Array(
2678 values
2679 .iter()
2680 .map(|(key, value)| {
2681 Ok(AvroDatum::Record(vec![
2682 ("key".to_string(), AvroDatum::String(key.clone())),
2683 ("value".to_string(), avro_value_to_datum(value)?),
2684 ]))
2685 })
2686 .collect::<Result<Vec<_>>>()?,
2687 ),
2688 )])),
2689 ),
2690 };
2691 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2692}
2693
2694fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2695 let AvroDatum::Record(mut outer) = datum else {
2696 return Err(Error::Codec(
2697 "invalid_payload_framing: datum is not a Value record".to_string(),
2698 ));
2699 };
2700 let (_, branch) = outer
2701 .pop()
2702 .filter(|(name, _)| name == "value")
2703 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2704 let AvroDatum::Union(_, branch) = branch else {
2705 return Err(Error::Codec(
2706 "invalid_payload_framing: invalid Value union".to_string(),
2707 ));
2708 };
2709 match *branch {
2710 AvroDatum::Null => Ok(AvroValue::Null),
2711 AvroDatum::Record(mut fields) => {
2712 let (name, value) = fields.pop().ok_or_else(|| {
2713 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2714 })?;
2715 match (name.as_str(), value) {
2716 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2717 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2718 ("double", AvroDatum::Double(value)) if value.is_finite() => {
2719 Ok(AvroValue::Double(value))
2720 }
2721 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2722 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2723 ("items", AvroDatum::Array(values)) => values
2724 .into_iter()
2725 .map(avro_value_from_datum)
2726 .collect::<Result<Vec<_>>>()
2727 .map(AvroValue::Array),
2728 ("entries", AvroDatum::Map(values)) => values
2729 .into_iter()
2730 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2731 .collect::<Result<BTreeMap<_, _>>>()
2732 .map(AvroValue::Map),
2733 _ => Err(Error::Codec(
2734 "invalid_payload_framing: unknown Value branch".to_string(),
2735 )),
2736 }
2737 }
2738 _ => Err(Error::Codec(
2739 "invalid_payload_framing: invalid Value branch".to_string(),
2740 )),
2741 }
2742}
2743
2744fn avro_value_schema() -> Result<&'static Schema> {
2745 match AVRO_VALUE_SCHEMA.get_or_init(|| {
2746 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2747 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2748 }) {
2749 Ok(schema) => Ok(schema),
2750 Err(message) => Err(Error::Codec(message.clone())),
2751 }
2752}
2753
2754fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2755 match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2756 let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2761 .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2762 let entries_schema = schema
2763 .pointer_mut("/fields/0/type/7/fields/0/type")
2764 .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2765 if *entries_schema != json!({"type": "map", "values": "Value"}) {
2766 return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2767 }
2768 *entries_schema = json!({
2769 "type": "array",
2770 "items": {
2771 "type": "record",
2772 "name": "MapEntry",
2773 "fields": [
2774 {"name": "key", "type": "string"},
2775 {"name": "value", "type": "Value"}
2776 ]
2777 }
2778 });
2779 Schema::parse_str(&schema.to_string())
2780 .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2781 }) {
2782 Ok(schema) => Ok(schema),
2783 Err(message) => Err(Error::Codec(message.clone())),
2784 }
2785}
2786
2787#[derive(Clone, Debug)]
2788pub struct Client {
2789 http: reqwest::Client,
2790 base_url: String,
2791 token: Option<String>,
2792 control_token: Option<String>,
2793 worker_token: Option<String>,
2794 namespace: String,
2795 max_external_payload_bytes: usize,
2796 worker_storage_admission: Option<WorkerStorageAdmission>,
2797 cooperative_worker_protocol: bool,
2798 local_activities_enabled: bool,
2799 worker_sessions_enabled: bool,
2800 max_concurrent_worker_sessions: usize,
2801 worker_build_id: Option<String>,
2802 sticky_cache: Option<Arc<Mutex<sticky_workflow_cache::StickyWorkflowCache>>>,
2803 runtime_upload_policy: Arc<Mutex<runtime_uploads::PolicyCache>>,
2804}
2805
2806impl Client {
2807 pub fn new(base_url: impl Into<String>) -> Result<Self> {
2808 Self::builder(base_url).build()
2809 }
2810
2811 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2812 ClientBuilder {
2813 base_url: base_url.into(),
2814 token: None,
2815 control_token: None,
2816 worker_token: None,
2817 namespace: "default".to_string(),
2818 timeout: Duration::from_secs(60),
2819 max_external_payload_bytes: 64 * 1024 * 1024,
2820 }
2821 }
2822
2823 pub async fn health(&self) -> Result<Value> {
2824 self.request_json(
2825 reqwest::Method::GET,
2826 "/health",
2827 RequestProtocol::ControlPlane,
2828 Option::<&Value>::None,
2829 )
2830 .await
2831 }
2832
2833 pub async fn cluster_info(&self) -> Result<Value> {
2834 self.request_json(
2835 reqwest::Method::GET,
2836 "/cluster/info",
2837 RequestProtocol::ControlPlane,
2838 Option::<&Value>::None,
2839 )
2840 .await
2841 }
2842
2843 pub async fn start_workflow<T: Serialize>(
2844 &self,
2845 workflow_type: &str,
2846 task_queue: &str,
2847 workflow_id: &str,
2848 input: T,
2849 ) -> Result<WorkflowHandle> {
2850 self.start_workflow_with_options(
2851 workflow_type,
2852 task_queue,
2853 workflow_id,
2854 WorkflowStartOptions::default(),
2855 input,
2856 )
2857 .await
2858 }
2859
2860 pub async fn start_workflow_with_options<T: Serialize>(
2863 &self,
2864 workflow_type: &str,
2865 task_queue: &str,
2866 workflow_id: &str,
2867 options: WorkflowStartOptions,
2868 input: T,
2869 ) -> Result<WorkflowHandle> {
2870 options.validate()?;
2871 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2872 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2873 let body = json!({
2874 "workflow_id": workflow_id,
2875 "workflow_type": workflow_type,
2876 "task_queue": task_queue,
2877 "input": input_envelope,
2878 "execution_timeout_seconds": options.execution_timeout_seconds,
2879 "run_timeout_seconds": options.run_timeout_seconds
2880 });
2881
2882 let data: Value = self
2883 .request_json(
2884 reqwest::Method::POST,
2885 "/workflows",
2886 RequestProtocol::ControlPlane,
2887 Some(&body),
2888 )
2889 .await?;
2890
2891 Ok(WorkflowHandle {
2892 client: self.clone(),
2893 workflow_id: data
2894 .get("workflow_id")
2895 .and_then(Value::as_str)
2896 .unwrap_or(workflow_id)
2897 .to_string(),
2898 run_id: data
2899 .get("run_id")
2900 .and_then(Value::as_str)
2901 .map(str::to_string),
2902 workflow_type: data
2903 .get("workflow_type")
2904 .and_then(Value::as_str)
2905 .unwrap_or(workflow_type)
2906 .to_string(),
2907 })
2908 }
2909
2910 pub async fn signal_workflow<T: Serialize>(
2911 &self,
2912 workflow_id: &str,
2913 signal_name: &str,
2914 input: T,
2915 ) -> Result<Value> {
2916 self.signal_workflow_target(workflow_id, None, signal_name, input)
2917 .await
2918 }
2919
2920 pub async fn append_message_stream<T: Serialize>(
2922 &self,
2923 workflow_id: &str,
2924 stream_name: &str,
2925 message_id: &str,
2926 input: T,
2927 ) -> Result<Value> {
2928 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2929 let body = json!({
2930 "message_id": message_id,
2931 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2932 });
2933 self.request_json(
2934 reqwest::Method::POST,
2935 &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2936 RequestProtocol::ControlPlane,
2937 Some(&body),
2938 )
2939 .await
2940 }
2941
2942 pub async fn signal_workflow_run<T: Serialize>(
2944 &self,
2945 workflow_id: &str,
2946 run_id: &str,
2947 signal_name: &str,
2948 input: T,
2949 ) -> Result<Value> {
2950 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2951 .await
2952 }
2953
2954 async fn signal_workflow_target<T: Serialize>(
2955 &self,
2956 workflow_id: &str,
2957 run_id: Option<&str>,
2958 signal_name: &str,
2959 input: T,
2960 ) -> Result<Value> {
2961 validate_user_signal_name(signal_name)?;
2962 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2963 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2964 let body = json!({
2965 "input": input_envelope
2966 });
2967 let path = match run_id {
2968 Some(run_id) => {
2969 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2970 }
2971 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2972 };
2973 self.request_json(
2974 reqwest::Method::POST,
2975 &path,
2976 RequestProtocol::ControlPlane,
2977 Some(&body),
2978 )
2979 .await
2980 }
2981
2982 pub async fn cancel_workflow(
2988 &self,
2989 workflow_id: &str,
2990 options: WorkflowCommandOptions,
2991 ) -> Result<WorkflowCommandResult> {
2992 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2993 .await
2994 }
2995
2996 pub async fn cancel_workflow_run(
2998 &self,
2999 workflow_id: &str,
3000 run_id: &str,
3001 options: WorkflowCommandOptions,
3002 ) -> Result<WorkflowCommandResult> {
3003 self.workflow_command(
3004 workflow_id,
3005 Some(run_id),
3006 WorkflowCommandKind::Cancel,
3007 options,
3008 )
3009 .await
3010 }
3011
3012 pub async fn terminate_workflow(
3014 &self,
3015 workflow_id: &str,
3016 options: WorkflowCommandOptions,
3017 ) -> Result<WorkflowCommandResult> {
3018 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
3019 .await
3020 }
3021
3022 pub async fn terminate_workflow_run(
3024 &self,
3025 workflow_id: &str,
3026 run_id: &str,
3027 options: WorkflowCommandOptions,
3028 ) -> Result<WorkflowCommandResult> {
3029 self.workflow_command(
3030 workflow_id,
3031 Some(run_id),
3032 WorkflowCommandKind::Terminate,
3033 options,
3034 )
3035 .await
3036 }
3037
3038 pub async fn redrive_workflow_run(
3040 &self,
3041 workflow_id: &str,
3042 failed_run_id: &str,
3043 request_id: Option<&str>,
3044 ) -> Result<WorkflowRedriveResult> {
3045 let body = request_id
3046 .map(|id| json!({"request_id": id}))
3047 .unwrap_or_else(|| json!({}));
3048 let data: Value = self
3049 .request_json(
3050 reqwest::Method::POST,
3051 &format!("/workflows/{workflow_id}/runs/{failed_run_id}/redrive"),
3052 RequestProtocol::ControlPlane,
3053 Some(&body),
3054 )
3055 .await?;
3056 if data.get("command_status").and_then(Value::as_str) != Some("accepted")
3057 || data.get("outcome").and_then(Value::as_str) != Some("redriven")
3058 {
3059 return Err(Error::Codec(
3060 "redrive response was not accepted".to_string(),
3061 ));
3062 }
3063 if data.get("workflow_id").and_then(Value::as_str) != Some(workflow_id)
3064 || data.get("continued_from_run_id").and_then(Value::as_str) != Some(failed_run_id)
3065 {
3066 return Err(Error::Codec(
3067 "redrive response does not match the requested source run".to_string(),
3068 ));
3069 }
3070 let run_id = data
3071 .get("run_id")
3072 .and_then(Value::as_str)
3073 .filter(|id| !id.is_empty() && *id != failed_run_id)
3074 .ok_or_else(|| {
3075 Error::Codec("redrive response is missing a distinct successor run_id".to_string())
3076 })?;
3077 let resume_step_sequence = data
3078 .get("resume_step_sequence")
3079 .and_then(Value::as_u64)
3080 .filter(|sequence| *sequence > 0)
3081 .ok_or_else(|| {
3082 Error::Codec("redrive response is missing resume_step_sequence".to_string())
3083 })?;
3084 Ok(WorkflowRedriveResult {
3085 workflow_id: workflow_id.to_string(),
3086 source_run_id: failed_run_id.to_string(),
3087 run_id: run_id.to_string(),
3088 resume_step_sequence,
3089 raw: data,
3090 })
3091 }
3092
3093 async fn workflow_command(
3094 &self,
3095 workflow_id: &str,
3096 run_id: Option<&str>,
3097 command: WorkflowCommandKind,
3098 options: WorkflowCommandOptions,
3099 ) -> Result<WorkflowCommandResult> {
3100 let path = match run_id {
3101 Some(run_id) => format!(
3102 "/workflows/{workflow_id}/runs/{run_id}/{}",
3103 command.as_str()
3104 ),
3105 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
3106 };
3107 let data = match self
3108 .request_json(
3109 reqwest::Method::POST,
3110 &path,
3111 RequestProtocol::ControlPlane,
3112 Some(&options),
3113 )
3114 .await
3115 {
3116 Ok(data) => data,
3117 Err(Error::Http { status, body }) => {
3118 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
3119 command,
3120 status,
3121 body,
3122 workflow_id,
3123 run_id,
3124 )));
3125 }
3126 Err(error) => return Err(error),
3127 };
3128
3129 Ok(workflow_command_result(command, data, workflow_id, run_id))
3130 }
3131
3132 pub async fn query_workflow<T: Serialize>(
3138 &self,
3139 workflow_id: &str,
3140 query_name: &str,
3141 input: T,
3142 ) -> Result<Value> {
3143 self.query_workflow_target(workflow_id, None, query_name, input)
3144 .await
3145 }
3146
3147 pub async fn query_workflow_run<T: Serialize>(
3149 &self,
3150 workflow_id: &str,
3151 run_id: &str,
3152 query_name: &str,
3153 input: T,
3154 ) -> Result<Value> {
3155 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
3156 .await
3157 }
3158
3159 pub async fn query_workflow_avro_value<T: Serialize>(
3161 &self,
3162 workflow_id: &str,
3163 query_name: &str,
3164 input: T,
3165 ) -> Result<AvroValue> {
3166 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
3167 .await
3168 }
3169
3170 pub async fn query_workflow_run_avro_value<T: Serialize>(
3172 &self,
3173 workflow_id: &str,
3174 run_id: &str,
3175 query_name: &str,
3176 input: T,
3177 ) -> Result<AvroValue> {
3178 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
3179 .await
3180 }
3181
3182 async fn query_workflow_avro_value_target<T: Serialize>(
3183 &self,
3184 workflow_id: &str,
3185 run_id: Option<&str>,
3186 query_name: &str,
3187 input: T,
3188 ) -> Result<AvroValue> {
3189 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3190 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
3191 let path = match run_id {
3192 Some(run_id) => {
3193 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3194 }
3195 None => format!("/workflows/{workflow_id}/query/{query_name}"),
3196 };
3197 let response: Value = match self
3198 .request_json(
3199 reqwest::Method::POST,
3200 &path,
3201 RequestProtocol::ControlPlane,
3202 Some(&body),
3203 )
3204 .await
3205 {
3206 Ok(response) => response,
3207 Err(Error::Http { status, body }) => {
3208 return Err(Error::QueryFailed(query_failure(status, body)));
3209 }
3210 Err(error) => return Err(error),
3211 };
3212
3213 let envelope = response
3214 .get("result_envelope")
3215 .filter(|envelope| !envelope.is_null())
3216 .ok_or_else(|| {
3217 Error::Codec(
3218 "missing_payload_envelope: typed query result requires result_envelope"
3219 .to_string(),
3220 )
3221 })?;
3222 decode_wire_avro_value(envelope, DEFAULT_CODEC)
3223 }
3224
3225 async fn query_workflow_target<T: Serialize>(
3226 &self,
3227 workflow_id: &str,
3228 run_id: Option<&str>,
3229 query_name: &str,
3230 input: T,
3231 ) -> Result<Value> {
3232 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3233 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
3234 let body = json!({
3235 "input": input_envelope
3236 });
3237 let path = match run_id {
3238 Some(run_id) => {
3239 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3240 }
3241 None => format!("/workflows/{workflow_id}/query/{query_name}"),
3242 };
3243 let response: Value = match self
3244 .request_json(
3245 reqwest::Method::POST,
3246 &path,
3247 RequestProtocol::ControlPlane,
3248 Some(&body),
3249 )
3250 .await
3251 {
3252 Ok(response) => response,
3253 Err(Error::Http { status, body }) => {
3254 return Err(Error::QueryFailed(query_failure(status, body)));
3255 }
3256 Err(error) => return Err(error),
3257 };
3258
3259 if let Some(envelope) = response
3260 .get("result_envelope")
3261 .filter(|envelope| !envelope.is_null())
3262 {
3263 return decode_wire_value(envelope, DEFAULT_CODEC);
3264 }
3265
3266 Ok(response.get("result").cloned().unwrap_or(Value::Null))
3267 }
3268
3269 pub async fn update_workflow<T: Serialize>(
3271 &self,
3272 workflow_id: &str,
3273 update_name: &str,
3274 input: T,
3275 request_id: Option<&str>,
3276 ) -> Result<Value> {
3277 let response = self
3278 .update_workflow_response(workflow_id, update_name, input, request_id)
3279 .await?;
3280 if let Some(envelope) = response
3281 .get("result_envelope")
3282 .filter(|envelope| !envelope.is_null())
3283 {
3284 return decode_wire_value(envelope, DEFAULT_CODEC);
3285 }
3286 Ok(response.get("result").cloned().unwrap_or(response))
3287 }
3288
3289 pub async fn update_workflow_avro_value<T: Serialize>(
3291 &self,
3292 workflow_id: &str,
3293 update_name: &str,
3294 input: T,
3295 request_id: Option<&str>,
3296 ) -> Result<AvroValue> {
3297 let response = self
3298 .update_workflow_response(workflow_id, update_name, input, request_id)
3299 .await?;
3300 let envelope = response
3301 .get("result_envelope")
3302 .filter(|envelope| !envelope.is_null())
3303 .ok_or_else(|| {
3304 Error::Codec(
3305 "missing_payload_envelope: typed update result requires result_envelope"
3306 .to_string(),
3307 )
3308 })?;
3309 decode_wire_avro_value(envelope, DEFAULT_CODEC)
3310 }
3311
3312 async fn update_workflow_response<T: Serialize>(
3313 &self,
3314 workflow_id: &str,
3315 update_name: &str,
3316 input: T,
3317 request_id: Option<&str>,
3318 ) -> Result<Value> {
3319 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3320 let mut body = json!({
3321 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3322 "wait_for": "completed",
3323 });
3324 if let Some(request_id) = request_id {
3325 body["request_id"] = json!(request_id);
3326 }
3327 self.request_json(
3328 reqwest::Method::POST,
3329 &format!("/workflows/{workflow_id}/update/{update_name}"),
3330 RequestProtocol::ControlPlane,
3331 Some(&body),
3332 )
3333 .await
3334 }
3335
3336 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3337 let path = format!("/workflows/{workflow_id}");
3338 let mut data: WorkflowDescription = self
3339 .request_json(
3340 reqwest::Method::GET,
3341 &path,
3342 RequestProtocol::ControlPlane,
3343 Option::<&Value>::None,
3344 )
3345 .await?;
3346 data.decode_payloads()?;
3347 Ok(data)
3348 }
3349
3350 pub async fn describe_workflow_run(
3352 &self,
3353 workflow_id: &str,
3354 run_id: &str,
3355 ) -> Result<WorkflowDescription> {
3356 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3357 let mut data: WorkflowDescription = self
3358 .request_json(
3359 reqwest::Method::GET,
3360 &path,
3361 RequestProtocol::ControlPlane,
3362 Option::<&Value>::None,
3363 )
3364 .await?;
3365 data.decode_payloads()?;
3366 Ok(data)
3367 }
3368
3369 fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3370 let mut path = format!(
3371 "/workflows/{}/runs/{}/streams",
3372 percent_encode_path_segment(workflow_id),
3373 percent_encode_path_segment(run_id),
3374 );
3375 if let Some(stream_name) = stream_name {
3376 path.push('/');
3377 path.push_str(&percent_encode_path_segment(stream_name));
3378 }
3379 path
3380 }
3381
3382 pub async fn list_workflow_streams(
3384 &self,
3385 workflow_id: &str,
3386 run_id: &str,
3387 ) -> Result<Vec<WorkflowStreamDescription>> {
3388 let response: WorkflowStreamListResponse = self
3389 .request_json(
3390 reqwest::Method::GET,
3391 &Self::workflow_stream_path(workflow_id, run_id, None),
3392 RequestProtocol::ControlPlane,
3393 Option::<&Value>::None,
3394 )
3395 .await?;
3396 Ok(response.streams)
3397 }
3398
3399 pub async fn describe_workflow_stream(
3401 &self,
3402 workflow_id: &str,
3403 run_id: &str,
3404 stream_name: &str,
3405 ) -> Result<WorkflowStreamDescription> {
3406 let response: WorkflowStreamDescriptionResponse = self
3407 .request_json(
3408 reqwest::Method::GET,
3409 &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3410 RequestProtocol::ControlPlane,
3411 Option::<&Value>::None,
3412 )
3413 .await?;
3414 Ok(response.stream)
3415 }
3416
3417 pub async fn subscribe_workflow_stream(
3424 &self,
3425 workflow_id: &str,
3426 run_id: &str,
3427 stream_name: &str,
3428 from_offset: u64,
3429 max_items: usize,
3430 wait: Duration,
3431 ) -> Result<WorkflowStreamPage> {
3432 let max_items = max_items.clamp(1, 500);
3433 let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3434 let path = format!(
3435 "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3436 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3437 );
3438 let response: WorkflowStreamPageResponse = self
3439 .request_json_with_timeout(
3440 reqwest::Method::GET,
3441 &path,
3442 RequestProtocol::ControlPlane,
3443 Option::<&Value>::None,
3444 Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3445 )
3446 .await?;
3447
3448 let items = response
3449 .items
3450 .into_iter()
3451 .map(|raw| {
3452 let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3453 let envelope = raw.get("payload").cloned();
3454 let payload = envelope
3455 .as_ref()
3456 .filter(|value| value.get("blob").is_some())
3457 .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3458 .transpose()?
3459 .map(AvroValue::into_json)
3460 .transpose()?;
3461 Ok(WorkflowStreamItem {
3462 offset,
3463 payload,
3464 payload_envelope: envelope,
3465 payload_reference: raw
3466 .get("payload_reference")
3467 .and_then(Value::as_str)
3468 .map(str::to_string),
3469 payload_codec: raw
3470 .get("payload_codec")
3471 .and_then(Value::as_str)
3472 .map(str::to_string),
3473 idempotency_key: raw
3474 .get("idempotency_key")
3475 .and_then(Value::as_str)
3476 .map(str::to_string),
3477 item_type: raw
3478 .get("item_type")
3479 .and_then(Value::as_str)
3480 .map(str::to_string),
3481 content_type: raw
3482 .get("content_type")
3483 .and_then(Value::as_str)
3484 .map(str::to_string),
3485 origin: raw
3486 .get("origin")
3487 .and_then(Value::as_str)
3488 .map(str::to_string),
3489 origin_reference: raw
3490 .get("origin_reference")
3491 .and_then(Value::as_str)
3492 .map(str::to_string),
3493 emitted_at: raw
3494 .get("emitted_at")
3495 .and_then(Value::as_str)
3496 .map(str::to_string),
3497 raw,
3498 })
3499 })
3500 .collect::<Result<Vec<_>>>()?;
3501 Ok(WorkflowStreamPage {
3502 stream: response.stream,
3503 items,
3504 next_offset: response.next_offset,
3505 terminal: response.terminal,
3506 })
3507 }
3508
3509 pub async fn append_workflow_stream(
3511 &self,
3512 workflow_id: &str,
3513 run_id: &str,
3514 stream_name: &str,
3515 items: &[WorkflowStreamAppendItem],
3516 max_pending_items: Option<u64>,
3517 ) -> Result<WorkflowStreamAppendResult> {
3518 if items.is_empty() {
3519 return Err(Error::Codec(
3520 "workflow_stream_items_empty: append requires at least one item".to_string(),
3521 ));
3522 }
3523 let mut body = json!({
3524 "items": items
3525 .iter()
3526 .map(|item| item.wire_value(None))
3527 .collect::<Vec<_>>(),
3528 });
3529 if let Some(max_pending_items) = max_pending_items {
3530 if max_pending_items == 0 {
3531 return Err(Error::Codec(
3532 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3533 .to_string(),
3534 ));
3535 }
3536 body["max_pending_items"] = json!(max_pending_items);
3537 }
3538 let response: WorkflowStreamAppendResponse = self
3539 .request_json(
3540 reqwest::Method::POST,
3541 &format!(
3542 "{}/items",
3543 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3544 ),
3545 RequestProtocol::ControlPlane,
3546 Some(&body),
3547 )
3548 .await?;
3549 Ok(WorkflowStreamAppendResult {
3550 stream: response.stream,
3551 accepted_offsets: response.accepted_offsets,
3552 accepted: response.accepted,
3553 deduped: response.deduped,
3554 })
3555 }
3556
3557 pub async fn close_workflow_stream(
3559 &self,
3560 workflow_id: &str,
3561 run_id: &str,
3562 stream_name: &str,
3563 error_reason: Option<&str>,
3564 retention_seconds: Option<u64>,
3565 ) -> Result<WorkflowStreamDescription> {
3566 let mut body = json!({});
3567 if let Some(error_reason) = error_reason {
3568 body["error_reason"] = json!(error_reason);
3569 }
3570 if let Some(retention_seconds) = retention_seconds {
3571 if retention_seconds == 0 {
3572 return Err(Error::Codec(
3573 "workflow_stream_retention_invalid: retention_seconds must be positive"
3574 .to_string(),
3575 ));
3576 }
3577 body["retention_seconds"] = json!(retention_seconds);
3578 }
3579 let response: WorkflowStreamDescriptionResponse = self
3580 .request_json(
3581 reqwest::Method::POST,
3582 &format!(
3583 "{}/close",
3584 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3585 ),
3586 RequestProtocol::ControlPlane,
3587 Some(&body),
3588 )
3589 .await?;
3590 Ok(response.stream)
3591 }
3592
3593 pub async fn register_worker(
3594 &self,
3595 worker_id: &str,
3596 task_queue: &str,
3597 supported_workflow_types: Vec<String>,
3598 supported_activity_types: Vec<String>,
3599 max_concurrent_workflow_tasks: usize,
3600 max_concurrent_activity_tasks: usize,
3601 ) -> Result<RegisterWorkerResponse> {
3602 self.register_worker_with_capabilities(
3603 worker_id,
3604 task_queue,
3605 supported_workflow_types,
3606 supported_activity_types,
3607 max_concurrent_workflow_tasks,
3608 max_concurrent_activity_tasks,
3609 Vec::new(),
3610 )
3611 .await
3612 }
3613
3614 pub async fn register_worker_with_capabilities(
3616 &self,
3617 worker_id: &str,
3618 task_queue: &str,
3619 supported_workflow_types: Vec<String>,
3620 supported_activity_types: Vec<String>,
3621 max_concurrent_workflow_tasks: usize,
3622 max_concurrent_activity_tasks: usize,
3623 capabilities: Vec<String>,
3624 ) -> Result<RegisterWorkerResponse> {
3625 self.register_worker_with_command_contracts(
3626 worker_id,
3627 task_queue,
3628 supported_workflow_types,
3629 supported_activity_types,
3630 max_concurrent_workflow_tasks,
3631 max_concurrent_activity_tasks,
3632 capabilities,
3633 Value::Object(serde_json::Map::new()),
3634 )
3635 .await
3636 }
3637
3638 #[allow(clippy::too_many_arguments)]
3645 pub async fn register_worker_with_command_contracts(
3646 &self,
3647 worker_id: &str,
3648 task_queue: &str,
3649 supported_workflow_types: Vec<String>,
3650 supported_activity_types: Vec<String>,
3651 max_concurrent_workflow_tasks: usize,
3652 max_concurrent_activity_tasks: usize,
3653 capabilities: Vec<String>,
3654 workflow_command_contracts: Value,
3655 ) -> Result<RegisterWorkerResponse> {
3656 self.register_worker_with_definition_fingerprints(
3657 worker_id,
3658 task_queue,
3659 supported_workflow_types,
3660 supported_activity_types,
3661 max_concurrent_workflow_tasks,
3662 max_concurrent_activity_tasks,
3663 capabilities,
3664 workflow_command_contracts,
3665 None,
3666 )
3667 .await
3668 }
3669
3670 #[allow(clippy::too_many_arguments)]
3671 async fn register_worker_with_definition_fingerprints(
3672 &self,
3673 worker_id: &str,
3674 task_queue: &str,
3675 supported_workflow_types: Vec<String>,
3676 supported_activity_types: Vec<String>,
3677 max_concurrent_workflow_tasks: usize,
3678 max_concurrent_activity_tasks: usize,
3679 capabilities: Vec<String>,
3680 workflow_command_contracts: Value,
3681 workflow_definition_fingerprints: Option<BTreeMap<String, String>>,
3682 ) -> Result<RegisterWorkerResponse> {
3683 if let Some(contracts) = workflow_command_contracts.as_object() {
3684 for (workflow_type, contract) in contracts {
3685 let Some(update_validators) = contract.get("update_validators") else {
3686 continue;
3687 };
3688 if !update_validators
3689 .as_array()
3690 .is_some_and(|validators| validators.is_empty())
3691 {
3692 return Err(Error::UnsupportedUpdateValidators {
3693 workflow_type: workflow_type.clone(),
3694 });
3695 }
3696 }
3697 }
3698
3699 let mut body = json!({
3700 "worker_id": worker_id,
3701 "task_queue": task_queue,
3702 "runtime": "rust",
3703 "sdk_version": SDK_VERSION,
3704 "supported_workflow_types": supported_workflow_types,
3705 "supported_activity_types": supported_activity_types,
3706 "capabilities": capabilities,
3707 "capability_manifest": portable_worker_affinity_capability_manifest(),
3708 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3709 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3710 });
3711 if let Some(build_id) = &self.worker_build_id {
3712 if build_id.is_empty() || build_id.len() > 255 {
3713 return Err(Error::WorkerLoop(
3714 "worker build ID must contain 1 to 255 bytes".into(),
3715 ));
3716 }
3717 body["build_id"] = json!(build_id);
3718 }
3719 if self.sticky_cache.is_some() {
3720 body["capability_manifest"]["sticky_execution"] = json!({"supported":true,
3721 "minimum_protocol_version":PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
3722 "implementation":"bounded_durable_history_cache"});
3723 }
3724 if self.local_activities_enabled {
3725 body["capability_manifest"]["local_activities"] = json!({"supported":true,
3726 "minimum_protocol_version":PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION});
3727 }
3728 if self.worker_sessions_enabled {
3729 body["capability_manifest"]["worker_sessions"] = json!({"supported":true,
3730 "minimum_protocol_version":PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION});
3731 body["max_concurrent_worker_sessions"] = json!(self.max_concurrent_worker_sessions);
3732 }
3733 if workflow_command_contracts
3734 .as_object()
3735 .is_some_and(|contracts| !contracts.is_empty())
3736 {
3737 body["workflow_command_contracts"] = workflow_command_contracts;
3738 }
3739 if let Some(fingerprints) = workflow_definition_fingerprints {
3740 body["workflow_definition_fingerprints"] = json!(fingerprints);
3741 }
3742
3743 let response: Value = self
3744 .request_json(
3745 reqwest::Method::POST,
3746 "/worker/register",
3747 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3748 Some(&body),
3749 )
3750 .await?;
3751 if self.sticky_cache.is_some() {
3752 self.confirm_sticky_registration(&response, worker_id, task_queue)
3753 .await?;
3754 }
3755 if self.local_activities_enabled || self.worker_sessions_enabled {
3756 let compatible = response["registered"].as_bool() == Some(true)
3757 && response["worker_id"].as_str() == Some(worker_id)
3758 && response["namespace"].as_str() == Some(self.namespace.as_str())
3759 && response["task_queue"].as_str() == Some(task_queue)
3760 && response["protocol_version"]
3761 .as_str()
3762 .and_then(|version| version.strip_prefix("1."))
3763 .and_then(|minor| minor.parse::<u64>().ok())
3764 .is_some_and(|minor| minor >= 18)
3765 && (!self.local_activities_enabled
3766 || response["capability_manifest"]["local_activities"]["supported"].as_bool()
3767 == Some(true))
3768 && (!self.worker_sessions_enabled
3769 || (response["capability_manifest"]["worker_sessions"]["supported"].as_bool()
3770 == Some(true)
3771 && response["capabilities"].as_array().is_some_and(|accepted| {
3772 capabilities.iter().all(|capability| {
3773 accepted
3774 .iter()
3775 .any(|value| value.as_str() == Some(capability.as_str()))
3776 })
3777 })));
3778 if !compatible {
3779 let error = Error::WorkerLoop(if self.worker_sessions_enabled {
3780 "worker_session_registration_unconfirmed: Server must acknowledge this worker, namespace, queue and session capabilities".into()
3781 } else {
3782 "local_activity_registration_unconfirmed: Server must acknowledge this worker, namespace, queue and local capability".into()
3783 });
3784 if response["registered"] == true
3785 && response["worker_id"].as_str() == Some(worker_id)
3786 {
3787 if let Err(deregistration) =
3788 self.deregister_worker_registration(worker_id).await
3789 {
3790 return Err(Error::WorkerShutdown {
3791 primary: Box::new(error),
3792 deregistration: Box::new(deregistration),
3793 });
3794 }
3795 }
3796 return Err(error);
3797 }
3798 }
3799 if self.cooperative_worker_protocol {
3800 let compatible = response["registered"].as_bool() == Some(true)
3801 && response["worker_id"].as_str() == Some(worker_id)
3802 && response["namespace"].as_str() == Some(self.namespace.as_str())
3803 && response["task_queue"].as_str() == Some(task_queue)
3804 && response["protocol_version"]
3805 .as_str()
3806 .is_some_and(cooperative_cancellation::supports_protocol)
3807 && response["server_capabilities"]["cooperative_cancellation"].as_bool()
3808 == Some(true)
3809 && response["capabilities"].as_array().is_some_and(|accepted| {
3810 capabilities.iter().all(|capability| {
3811 accepted
3812 .iter()
3813 .any(|value| value.as_str() == Some(capability.as_str()))
3814 })
3815 });
3816 if !compatible {
3817 let error = Error::CooperativeCancellationUnavailable(
3818 "registration must acknowledge this namespace, queue, worker, capabilities and compatible runtime protocol 1.20".into(),
3819 );
3820 if response["registered"].as_bool() == Some(true)
3821 && response["worker_id"].as_str() == Some(worker_id)
3822 {
3823 if let Err(deregistration) =
3824 self.deregister_worker_registration(worker_id).await
3825 {
3826 return Err(Error::WorkerShutdown {
3827 primary: Box::new(error),
3828 deregistration: Box::new(deregistration),
3829 });
3830 }
3831 }
3832 return Err(error);
3833 }
3834 }
3835 Ok(serde_json::from_value(response)?)
3836 }
3837
3838 pub async fn deregister_worker_registration(
3844 &self,
3845 worker_id: &str,
3846 ) -> Result<WorkerDeregistrationEnvelope> {
3847 let path = format!(
3848 "/worker/registrations/{}",
3849 percent_encode_path_segment(worker_id)
3850 );
3851 self.request_json(
3852 reqwest::Method::DELETE,
3853 &path,
3854 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3855 Option::<&Value>::None,
3856 )
3857 .await
3858 }
3859
3860 pub async fn poll_query_task(
3862 &self,
3863 worker_id: &str,
3864 task_queue: &str,
3865 timeout: Duration,
3866 ) -> Result<Option<QueryTask>> {
3867 Ok(self
3868 .poll_query_task_response(worker_id, task_queue, timeout)
3869 .await?
3870 .task)
3871 }
3872
3873 pub async fn poll_query_task_response(
3875 &self,
3876 worker_id: &str,
3877 task_queue: &str,
3878 timeout: Duration,
3879 ) -> Result<PollQueryTaskResponse> {
3880 let poll_request_id = unique_request_id("rust-query-poll");
3881 self.poll_query_task_response_with_request_id(
3882 worker_id,
3883 task_queue,
3884 timeout,
3885 &poll_request_id,
3886 1,
3887 )
3888 .await
3889 }
3890
3891 async fn poll_query_task_response_with_request_id(
3892 &self,
3893 worker_id: &str,
3894 task_queue: &str,
3895 timeout: Duration,
3896 poll_request_id: &str,
3897 transport_retries: usize,
3898 ) -> Result<PollQueryTaskResponse> {
3899 let timeout_seconds = long_poll_timeout_seconds(timeout);
3900 let body = json!({
3901 "worker_id": worker_id,
3902 "task_queue": task_queue,
3903 "poll_request_id": poll_request_id,
3904 "timeout_seconds": timeout_seconds,
3905 });
3906 self.poll_request_json(
3907 "/worker/query-tasks/poll",
3908 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3909 &body,
3910 timeout + Duration::from_secs(5),
3911 transport_retries,
3912 )
3913 .await
3914 }
3915
3916 pub async fn complete_query_task<T: Serialize>(
3918 &self,
3919 query_task_id: &str,
3920 lease_owner: &str,
3921 query_task_attempt: u64,
3922 result: T,
3923 codec: &str,
3924 ) -> Result<Value> {
3925 let typed_result = AvroValue::from_serialize(&result)?;
3926 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3927 self.complete_query_task_with_envelope(
3928 query_task_id,
3929 lease_owner,
3930 query_task_attempt,
3931 typed_result.into_json()?,
3932 result_envelope,
3933 )
3934 .await
3935 }
3936
3937 async fn complete_query_task_with_envelope(
3938 &self,
3939 query_task_id: &str,
3940 lease_owner: &str,
3941 query_task_attempt: u64,
3942 result: Value,
3943 result_envelope: Value,
3944 ) -> Result<Value> {
3945 let body = json!({
3946 "lease_owner": lease_owner,
3947 "query_task_attempt": query_task_attempt,
3948 "result": result,
3949 "result_envelope": result_envelope,
3950 });
3951 let path = format!("/worker/query-tasks/{query_task_id}/complete");
3952 let response = self
3953 .request_json(
3954 reqwest::Method::POST,
3955 &path,
3956 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3957 Some(&body),
3958 )
3959 .await;
3960 query_task_response(response)
3961 }
3962
3963 pub async fn fail_query_task(
3965 &self,
3966 query_task_id: &str,
3967 lease_owner: &str,
3968 query_task_attempt: u64,
3969 message: impl Into<String>,
3970 reason: impl Into<String>,
3971 failure_type: impl Into<String>,
3972 ) -> Result<Value> {
3973 let body = json!({
3974 "lease_owner": lease_owner,
3975 "query_task_attempt": query_task_attempt,
3976 "failure": {
3977 "message": message.into(),
3978 "reason": reason.into(),
3979 "type": failure_type.into(),
3980 }
3981 });
3982 let path = format!("/worker/query-tasks/{query_task_id}/fail");
3983 let response = self
3984 .request_json(
3985 reqwest::Method::POST,
3986 &path,
3987 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3988 Some(&body),
3989 )
3990 .await;
3991 query_task_response(response)
3992 }
3993
3994 pub async fn heartbeat_worker(
3995 &self,
3996 worker_id: &str,
3997 workflow_available: usize,
3998 activity_available: usize,
3999 ) -> Result<Value> {
4000 self.heartbeat_worker_with_sessions(worker_id, workflow_available, activity_available, None)
4001 .await
4002 }
4003
4004 async fn heartbeat_worker_with_sessions(
4005 &self,
4006 worker_id: &str,
4007 workflow_available: usize,
4008 activity_available: usize,
4009 session_available: Option<usize>,
4010 ) -> Result<Value> {
4011 let mut body = json!({
4012 "worker_id": worker_id,
4013 "task_slots": {
4014 "workflow_available": workflow_available,
4015 "activity_available": activity_available
4016 },
4017 "process_metrics": {
4018 "process_id": std::process::id(),
4019 "process_uptime_seconds": 0
4020 }
4021 });
4022
4023 if let Some(available) = session_available {
4024 body["task_slots"]["session_available"] = json!(available);
4025 }
4026
4027 self.request_json(
4028 reqwest::Method::POST,
4029 "/worker/heartbeat",
4030 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4031 Some(&body),
4032 )
4033 .await
4034 }
4035
4036 pub async fn poll_workflow_task(
4037 &self,
4038 worker_id: &str,
4039 task_queue: &str,
4040 timeout: Duration,
4041 ) -> Result<Option<WorkflowTask>> {
4042 Ok(self
4043 .poll_workflow_task_response(worker_id, task_queue, timeout)
4044 .await?
4045 .task)
4046 }
4047
4048 pub async fn poll_workflow_task_response(
4049 &self,
4050 worker_id: &str,
4051 task_queue: &str,
4052 timeout: Duration,
4053 ) -> Result<PollWorkflowTaskResponse> {
4054 let poll_request_id = unique_request_id("rust-workflow-poll");
4055 self.poll_workflow_task_response_with_request_id(
4056 worker_id,
4057 task_queue,
4058 timeout,
4059 &poll_request_id,
4060 1,
4061 )
4062 .await
4063 }
4064
4065 async fn poll_workflow_task_response_with_request_id(
4066 &self,
4067 worker_id: &str,
4068 task_queue: &str,
4069 timeout: Duration,
4070 poll_request_id: &str,
4071 transport_retries: usize,
4072 ) -> Result<PollWorkflowTaskResponse> {
4073 let body = json!({
4074 "worker_id": worker_id,
4075 "task_queue": task_queue,
4076 "poll_request_id": poll_request_id,
4077 "timeout_seconds": long_poll_timeout_seconds(timeout),
4078 "history_page_size": WORKFLOW_HISTORY_PAGE_SIZE,
4079 });
4080 let mut data: PollWorkflowTaskResponse = self
4081 .poll_request_json(
4082 "/worker/workflow-tasks/poll",
4083 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4084 &body,
4085 timeout + Duration::from_secs(5),
4086 transport_retries,
4087 )
4088 .await?;
4089
4090 if let Some(task) = data.task.as_mut() {
4091 self.fetch_remaining_workflow_history(worker_id, task)
4092 .await?;
4093 }
4094
4095 Ok(data)
4096 }
4097
4098 async fn fetch_remaining_workflow_history(
4099 &self,
4100 worker_id: &str,
4101 task: &mut WorkflowTask,
4102 ) -> Result<()> {
4103 let mut next_token = task.next_history_page_token.clone();
4104
4105 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
4106 let lease_owner = task
4107 .lease_owner
4108 .clone()
4109 .unwrap_or_else(|| worker_id.to_string());
4110 let page = self
4111 .workflow_task_history_page(
4112 &task.task_id,
4113 &lease_owner,
4114 task.workflow_task_attempt,
4115 &token,
4116 )
4117 .await?;
4118
4119 task.append_history_page(page);
4120
4121 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
4122 return Err(Error::Codec(
4123 "workflow history pagination returned the same page token".to_string(),
4124 ));
4125 }
4126
4127 next_token = task.next_history_page_token.clone();
4128 }
4129
4130 Ok(())
4131 }
4132
4133 async fn workflow_task_history_page(
4134 &self,
4135 task_id: &str,
4136 lease_owner: &str,
4137 workflow_task_attempt: u64,
4138 next_history_page_token: &str,
4139 ) -> Result<WorkflowTaskHistoryPage> {
4140 let body = json!({
4141 "lease_owner": lease_owner,
4142 "workflow_task_attempt": workflow_task_attempt,
4143 "next_history_page_token": next_history_page_token
4144 });
4145 let path = format!("/worker/workflow-tasks/{task_id}/history");
4146
4147 self.request_json(
4148 reqwest::Method::POST,
4149 &path,
4150 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4151 Some(&body),
4152 )
4153 .await
4154 }
4155
4156 pub async fn complete_workflow_task(
4157 &self,
4158 task_id: &str,
4159 lease_owner: &str,
4160 workflow_task_attempt: u64,
4161 commands: Vec<Value>,
4162 ) -> Result<Value> {
4163 self.complete_workflow_task_with_message_streams(
4164 task_id,
4165 lease_owner,
4166 workflow_task_attempt,
4167 commands,
4168 Vec::new(),
4169 Vec::new(),
4170 None,
4171 )
4172 .await
4173 }
4174
4175 #[allow(clippy::too_many_arguments)]
4176 async fn complete_workflow_task_with_message_streams(
4177 &self,
4178 task_id: &str,
4179 lease_owner: &str,
4180 workflow_task_attempt: u64,
4181 commands: Vec<Value>,
4182 message_stream_cursors: Vec<Value>,
4183 message_stream_waits: Vec<Value>,
4184 sticky_cache: Option<Value>,
4185 ) -> Result<Value> {
4186 validate_workflow_task_commands(&commands)?;
4187 let has_message_stream_metadata =
4188 !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
4189 if has_message_stream_metadata
4190 && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
4191 {
4192 return Err(Error::Codec(
4193 "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
4194 .to_string(),
4195 ));
4196 }
4197 let protocol_version = workflow_completion_protocol_version_with_message_streams(
4198 &commands,
4199 has_message_stream_metadata,
4200 );
4201 let protocol_version = if sticky_cache.is_some()
4202 && protocol_version
4203 .strip_prefix("1.")
4204 .and_then(|minor| minor.parse::<u64>().ok())
4205 .is_some_and(|minor| minor < 18)
4206 {
4207 PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION
4208 } else {
4209 protocol_version
4210 };
4211 let mut body = json!({
4212 "lease_owner": lease_owner,
4213 "workflow_task_attempt": workflow_task_attempt,
4214 "commands": commands
4215 });
4216 if !message_stream_cursors.is_empty() {
4217 body["message_stream_cursors"] = Value::Array(message_stream_cursors);
4218 }
4219 if !message_stream_waits.is_empty() {
4220 body["message_stream_waits"] = Value::Array(message_stream_waits);
4221 }
4222 if let Some(claim) = sticky_cache {
4223 body["sticky_cache"] = claim;
4224 }
4225 let path = format!("/worker/workflow-tasks/{task_id}/complete");
4226 self.request_json(
4227 reqwest::Method::POST,
4228 &path,
4229 RequestProtocol::Worker(protocol_version),
4230 Some(&body),
4231 )
4232 .await
4233 }
4234
4235 pub async fn fail_workflow_task(
4236 &self,
4237 task_id: &str,
4238 lease_owner: &str,
4239 workflow_task_attempt: u64,
4240 message: impl Into<String>,
4241 ) -> Result<Value> {
4242 self.fail_workflow_task_with_type(
4243 task_id,
4244 lease_owner,
4245 workflow_task_attempt,
4246 message,
4247 "RustWorkflowTaskFailure",
4248 )
4249 .await
4250 }
4251
4252 async fn fail_workflow_task_with_type(
4253 &self,
4254 task_id: &str,
4255 lease_owner: &str,
4256 workflow_task_attempt: u64,
4257 message: impl Into<String>,
4258 failure_type: &str,
4259 ) -> Result<Value> {
4260 let body = json!({
4261 "lease_owner": lease_owner,
4262 "workflow_task_attempt": workflow_task_attempt,
4263 "failure": {
4264 "message": message.into(),
4265 "type": failure_type
4266 }
4267 });
4268 let path = format!("/worker/workflow-tasks/{task_id}/fail");
4269 self.request_json(
4270 reqwest::Method::POST,
4271 &path,
4272 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4273 Some(&body),
4274 )
4275 .await
4276 }
4277
4278 pub async fn poll_activity_task(
4279 &self,
4280 worker_id: &str,
4281 task_queue: &str,
4282 timeout: Duration,
4283 ) -> Result<Option<ActivityTask>> {
4284 Ok(self
4285 .poll_activity_task_response(worker_id, task_queue, timeout)
4286 .await?
4287 .task)
4288 }
4289
4290 pub async fn poll_activity_task_response(
4292 &self,
4293 worker_id: &str,
4294 task_queue: &str,
4295 timeout: Duration,
4296 ) -> Result<PollActivityTaskResponse> {
4297 let poll_request_id = unique_request_id("rust-activity-poll");
4298 self.poll_activity_task_response_with_request_id(
4299 worker_id,
4300 task_queue,
4301 timeout,
4302 &poll_request_id,
4303 1,
4304 )
4305 .await
4306 .map(worker_session::SessionPollResponse::ordinary)
4307 }
4308
4309 async fn poll_activity_task_response_with_request_id(
4310 &self,
4311 worker_id: &str,
4312 task_queue: &str,
4313 timeout: Duration,
4314 poll_request_id: &str,
4315 transport_retries: usize,
4316 ) -> Result<worker_session::SessionPollResponse> {
4317 let body = json!({
4318 "worker_id": worker_id,
4319 "task_queue": task_queue,
4320 "poll_request_id": poll_request_id,
4321 "timeout_seconds": long_poll_timeout_seconds(timeout),
4322 });
4323 let data: worker_session::SessionPollResponse = self
4324 .poll_request_json(
4325 "/worker/activity-tasks/poll",
4326 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4327 &body,
4328 timeout + Duration::from_secs(5),
4329 transport_retries,
4330 )
4331 .await?;
4332 Ok(data)
4333 }
4334
4335 pub async fn complete_activity_task<T: Serialize>(
4336 &self,
4337 task_id: &str,
4338 activity_attempt_id: &str,
4339 lease_owner: &str,
4340 result: T,
4341 codec: &str,
4342 ) -> Result<Value> {
4343 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
4344 let body = json!({
4345 "activity_attempt_id": activity_attempt_id,
4346 "lease_owner": lease_owner,
4347 "result": result
4348 });
4349 let path = format!("/worker/activity-tasks/{task_id}/complete");
4350 activity_task_response(
4351 self.request_json(
4352 reqwest::Method::POST,
4353 &path,
4354 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4355 Some(&body),
4356 )
4357 .await,
4358 "complete",
4359 task_id,
4360 activity_attempt_id,
4361 )
4362 }
4363
4364 pub async fn fail_activity_task(
4365 &self,
4366 task_id: &str,
4367 activity_attempt_id: &str,
4368 lease_owner: &str,
4369 message: impl Into<String>,
4370 non_retryable: bool,
4371 ) -> Result<Value> {
4372 let body = json!({
4373 "activity_attempt_id": activity_attempt_id,
4374 "lease_owner": lease_owner,
4375 "failure": {
4376 "message": message.into(),
4377 "type": "RustActivityFailure",
4378 "non_retryable": non_retryable
4379 }
4380 });
4381 let path = format!("/worker/activity-tasks/{task_id}/fail");
4382 activity_task_response(
4383 self.request_json(
4384 reqwest::Method::POST,
4385 &path,
4386 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4387 Some(&body),
4388 )
4389 .await,
4390 "fail",
4391 task_id,
4392 activity_attempt_id,
4393 )
4394 }
4395
4396 pub async fn heartbeat_activity_task<T: Serialize>(
4401 &self,
4402 task_id: &str,
4403 activity_attempt_id: &str,
4404 lease_owner: &str,
4405 details: T,
4406 ) -> Result<ActivityHeartbeatResponse> {
4407 serde_json::from_value(
4408 self.heartbeat_activity_task_value(task_id, activity_attempt_id, lease_owner, details)
4409 .await?,
4410 )
4411 .map_err(Error::from)
4412 }
4413
4414 async fn heartbeat_activity_task_value<T: Serialize>(
4415 &self,
4416 task_id: &str,
4417 activity_attempt_id: &str,
4418 lease_owner: &str,
4419 details: T,
4420 ) -> Result<Value> {
4421 let details = AvroValue::from_serialize(&details)?.into_json()?;
4422 let body = json!({
4423 "activity_attempt_id": activity_attempt_id,
4424 "lease_owner": lease_owner,
4425 "details": details
4426 });
4427 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
4428 activity_task_response(
4429 self.request_json(
4430 reqwest::Method::POST,
4431 &path,
4432 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4433 Some(&body),
4434 )
4435 .await,
4436 "heartbeat",
4437 task_id,
4438 activity_attempt_id,
4439 )
4440 }
4441
4442 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4443 &self,
4444 method: reqwest::Method,
4445 path: &str,
4446 protocol: RequestProtocol,
4447 body: Option<&B>,
4448 ) -> Result<T> {
4449 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4450 .await
4451 }
4452
4453 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4454 &self,
4455 method: reqwest::Method,
4456 path: &str,
4457 protocol: RequestProtocol,
4458 body: Option<&B>,
4459 timeout: Duration,
4460 ) -> Result<T> {
4461 let protocol = match protocol {
4462 RequestProtocol::Worker(version)
4463 if self.cooperative_worker_protocol
4464 && version
4465 .strip_prefix("1.")
4466 .and_then(|minor| minor.parse::<u64>().ok())
4467 .is_some_and(|minor| minor < 20) =>
4468 {
4469 RequestProtocol::Worker("1.20")
4470 }
4471 protocol => protocol,
4472 };
4473 let auth_token = self.auth_token(protocol)?;
4474 let mut request = self
4475 .http
4476 .request(method.clone(), format!("{}/api{}", self.base_url, path))
4477 .timeout(timeout)
4478 .header(reqwest::header::ACCEPT, "application/json")
4479 .header(reqwest::header::CONTENT_TYPE, "application/json")
4480 .header("X-Namespace", &self.namespace);
4481
4482 match protocol {
4483 RequestProtocol::Worker(version) => {
4484 request = request.header("X-Durable-Workflow-Protocol-Version", version);
4485 }
4486 RequestProtocol::ControlPlane => {
4487 request = request.header(
4488 "X-Durable-Workflow-Control-Plane-Version",
4489 CONTROL_PLANE_VERSION,
4490 );
4491 }
4492 }
4493
4494 if let Some(token) = auth_token {
4495 request = request.bearer_auth(token);
4496 }
4497
4498 if let Some(body) = body {
4499 let mut body = serde_json::to_value(body)?;
4500 if matches!(
4501 method,
4502 reqwest::Method::POST | reqwest::Method::PUT | reqwest::Method::PATCH
4503 ) {
4504 self.externalize_runtime_payloads(&mut body, path, protocol)
4505 .await?;
4506 }
4507 request = request.json(&body);
4508 }
4509
4510 let request = request.build()?;
4511 let request_body = request
4512 .body()
4513 .and_then(reqwest::Body::as_bytes)
4514 .and_then(|body| serde_json::from_slice::<Value>(body).ok());
4515 let poll_request_id = path.ends_with("/poll").then(|| {
4516 request_body
4517 .as_ref()
4518 .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4519 .unwrap_or_default()
4520 });
4521 let mut storage_retries = 0_usize;
4522
4523 loop {
4524 let response = self
4526 .http
4527 .execute(request.try_clone().ok_or_else(|| {
4528 Error::WorkerLoop("worker request body cannot be retried".to_string())
4529 })?)
4530 .await?;
4531 let status = response.status();
4532 let bytes = response.bytes().await?;
4533
4534 if !status.is_success() {
4535 let body = String::from_utf8_lossy(&bytes).to_string();
4536 if let Some(protocol) = protocol_failure(status, &body) {
4537 return Err(Error::Protocol(protocol));
4538 }
4539 let error = Error::Http { status, body };
4540 if self
4541 .wait_for_storage_admission(
4542 &error,
4543 protocol,
4544 Some(path),
4545 poll_request_id.as_deref(),
4546 request_body.as_ref(),
4547 &mut storage_retries,
4548 )
4549 .await
4550 {
4551 continue;
4552 }
4553 return Err(error);
4554 }
4555
4556 if bytes.is_empty() {
4557 return Ok(serde_json::from_value(Value::Null)?);
4558 }
4559
4560 let mut value: Value = serde_json::from_slice(&bytes)?;
4561 self.resolve_runtime_payloads(&mut value, path, protocol)
4562 .await?;
4563 return Ok(serde_json::from_value(value)?);
4564 }
4565 }
4566
4567 async fn wait_for_storage_admission(
4568 &self,
4569 error: &Error,
4570 protocol: RequestProtocol,
4571 path: Option<&str>,
4572 poll_request_id: Option<&str>,
4573 request_body: Option<&Value>,
4574 retries: &mut usize,
4575 ) -> bool {
4576 let Some(admission) = self
4577 .worker_storage_admission
4578 .as_ref()
4579 .filter(|_| matches!(protocol, RequestProtocol::Worker(_)))
4580 else {
4581 return false;
4582 };
4583 let Some(advertised_delay) = worker_storage_admission_retry_after(error, poll_request_id)
4584 .or_else(|| {
4585 path.and_then(|path| {
4586 worker_backend_unavailable_retry_after(error, path, poll_request_id)
4587 .or_else(|| worker_fenced_completion_retry_after(error, path, request_body))
4588 })
4589 })
4590 else {
4591 return false;
4592 };
4593 *retries = retries.saturating_add(1);
4594 let delay = worker_retry_delay(admission.policy, *retries)
4595 .max(advertised_delay)
4596 .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4597 let deadline = tokio::time::Instant::now() + delay;
4598 loop {
4599 if admission.stop.load(Ordering::SeqCst) {
4600 return false;
4601 }
4602 let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
4603 if remaining.is_zero() {
4604 return true;
4605 }
4606 tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4607 }
4608 }
4609
4610 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4611 &self,
4612 path: &str,
4613 protocol: RequestProtocol,
4614 body: &B,
4615 timeout: Duration,
4616 max_retries: usize,
4617 ) -> Result<T> {
4618 let mut retries = 0;
4619
4620 loop {
4621 let response = self
4622 .request_json_with_timeout(
4623 reqwest::Method::POST,
4624 path,
4625 protocol,
4626 Some(body),
4627 timeout,
4628 )
4629 .await;
4630
4631 match response {
4632 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4633 response => return worker_poll_response(response),
4634 }
4635 }
4636 }
4637
4638 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4639 match protocol {
4640 RequestProtocol::Worker(_) => {
4641 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4642 return Ok(Some(token));
4643 }
4644 if self.control_token.is_some() {
4645 return Err(Error::MissingRoleCredentials {
4646 role: "worker",
4647 opposite_role: "control",
4648 });
4649 }
4650 Ok(None)
4651 }
4652 RequestProtocol::ControlPlane => {
4653 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4654 return Ok(Some(token));
4655 }
4656 if self.worker_token.is_some() {
4657 return Err(Error::MissingRoleCredentials {
4658 role: "control",
4659 opposite_role: "worker",
4660 });
4661 }
4662 Ok(None)
4663 }
4664 }
4665 }
4666}
4667
4668fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4669 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4670 let reason = body
4671 .get("reason")
4672 .and_then(Value::as_str)
4673 .unwrap_or("query_rejected")
4674 .to_string();
4675 let message = body
4676 .get("message")
4677 .or_else(|| body.get("error"))
4678 .and_then(Value::as_str)
4679 .unwrap_or("workflow query was rejected")
4680 .to_string();
4681
4682 QueryFailure {
4683 status: status.as_u16(),
4684 reason,
4685 message,
4686 body,
4687 }
4688}
4689
4690fn workflow_command_result(
4691 command: WorkflowCommandKind,
4692 data: Value,
4693 workflow_id: &str,
4694 run_id: Option<&str>,
4695) -> WorkflowCommandResult {
4696 WorkflowCommandResult {
4697 command,
4698 workflow_id: data
4699 .get("workflow_id")
4700 .and_then(Value::as_str)
4701 .unwrap_or(workflow_id)
4702 .to_string(),
4703 run_id: data
4704 .get("run_id")
4705 .and_then(Value::as_str)
4706 .or(run_id)
4707 .map(str::to_string),
4708 outcome: data
4709 .get("outcome")
4710 .and_then(Value::as_str)
4711 .map(str::to_string),
4712 reason: data
4713 .get("reason")
4714 .and_then(Value::as_str)
4715 .map(str::to_string),
4716 command_status: data
4717 .get("command_status")
4718 .and_then(Value::as_str)
4719 .map(str::to_string),
4720 raw: data,
4721 }
4722}
4723
4724fn workflow_command_rejection(
4725 command: WorkflowCommandKind,
4726 status: reqwest::StatusCode,
4727 raw_body: String,
4728 workflow_id: &str,
4729 run_id: Option<&str>,
4730) -> WorkflowCommandRejection {
4731 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4732 WorkflowCommandRejection {
4733 command,
4734 status: status.as_u16(),
4735 reason: body
4736 .get("reason")
4737 .and_then(Value::as_str)
4738 .unwrap_or("workflow_command_rejected")
4739 .to_string(),
4740 message: body
4741 .get("message")
4742 .or_else(|| body.get("error"))
4743 .and_then(Value::as_str)
4744 .unwrap_or("workflow lifecycle command was rejected")
4745 .to_string(),
4746 workflow_id: body
4747 .get("workflow_id")
4748 .and_then(Value::as_str)
4749 .unwrap_or(workflow_id)
4750 .to_string(),
4751 run_id: body
4752 .get("run_id")
4753 .and_then(Value::as_str)
4754 .or(run_id)
4755 .map(str::to_string),
4756 target_scope: body
4757 .get("target_scope")
4758 .and_then(Value::as_str)
4759 .map(str::to_string),
4760 body,
4761 }
4762}
4763
4764fn query_task_response(response: Result<Value>) -> Result<Value> {
4765 match response {
4766 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4767 response => response,
4768 }
4769}
4770
4771fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4772 match response {
4773 Err(Error::Http { status, body })
4774 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4775 {
4776 Ok(serde_json::from_str(&body)?)
4777 }
4778 response => response,
4779 }
4780}
4781
4782fn worker_poll_body_is_stop(body: &str) -> bool {
4783 serde_json::from_str::<Value>(body)
4784 .ok()
4785 .is_some_and(|body| {
4786 worker_poll_is_stop(
4787 body.get("poll_status").and_then(Value::as_str),
4788 body.get("reason").and_then(Value::as_str),
4789 )
4790 })
4791}
4792
4793fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4794 matches!(poll_status, Some("draining" | "stopped"))
4795 || matches!(reason, Some("worker_draining" | "worker_stopped"))
4796}
4797
4798fn query_task_rejection_is_final(error: &Error) -> bool {
4799 matches!(
4800 error,
4801 Error::QueryFailed(failure)
4802 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4803 )
4804}
4805
4806fn activity_task_response<T>(
4807 response: Result<T>,
4808 operation: &str,
4809 task_id: &str,
4810 activity_attempt_id: &str,
4811) -> Result<T> {
4812 match response {
4813 Err(Error::Http { status, body }) => {
4814 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4815 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4816 operation: operation.to_string(),
4817 status: status.as_u16(),
4818 reason: body
4819 .get("reason")
4820 .and_then(Value::as_str)
4821 .unwrap_or("activity_task_rejected")
4822 .to_string(),
4823 task_id: body
4824 .get("task_id")
4825 .and_then(Value::as_str)
4826 .unwrap_or(task_id)
4827 .to_string(),
4828 activity_attempt_id: body
4829 .get("activity_attempt_id")
4830 .and_then(Value::as_str)
4831 .unwrap_or(activity_attempt_id)
4832 .to_string(),
4833 cancel_requested: body
4834 .get("cancel_requested")
4835 .and_then(Value::as_bool)
4836 .unwrap_or(false),
4837 can_continue: body.get("can_continue").and_then(Value::as_bool),
4838 run_closed_reason: body
4839 .get("run_closed_reason")
4840 .and_then(Value::as_str)
4841 .map(str::to_string),
4842 body,
4843 }))
4844 }
4845 response => response,
4846 }
4847}
4848
4849fn activity_task_rejection_is_final(error: &Error) -> bool {
4850 matches!(
4851 error,
4852 Error::ActivityTaskRejected(rejection)
4853 if matches!(
4854 rejection.reason.as_str(),
4855 "run_cancelled"
4856 | "run_terminated"
4857 | "attempt_closed"
4858 | "stale_attempt"
4859 | "activity_cancelled"
4860 | "task_cancelled"
4861 | "run_closed"
4862 | "activity_not_running"
4863 | "attempt_not_found"
4864 )
4865 )
4866}
4867
4868fn workflow_task_completion_is_terminal_timeout(
4869 error: &Error,
4870 task_id: &str,
4871 workflow_task_attempt: u64,
4872 run_id: Option<&str>,
4873) -> bool {
4874 let Error::Http { status, body } = error else {
4875 return false;
4876 };
4877 if *status != reqwest::StatusCode::CONFLICT {
4878 return false;
4879 }
4880
4881 let Some(run_id) = run_id else {
4882 return false;
4883 };
4884 let Ok(body) = serde_json::from_str::<Value>(body) else {
4885 return false;
4886 };
4887
4888 body.get("recorded").and_then(Value::as_bool) == Some(false)
4889 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4890 && body.get("run_status").and_then(Value::as_str) == Some("failed")
4891 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4892 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4893 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4894}
4895
4896fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4897 let body: Value = serde_json::from_str(raw_body).ok()?;
4898 let reason = body.get("reason")?.as_str()?;
4899 if !matches!(
4900 reason,
4901 "missing_protocol_version"
4902 | "unsupported_protocol_version"
4903 | "missing_control_plane_version"
4904 | "unsupported_control_plane_version"
4905 ) {
4906 return None;
4907 }
4908
4909 Some(ProtocolFailure {
4910 status: status.as_u16(),
4911 reason: reason.to_string(),
4912 message: body
4913 .get("message")
4914 .or_else(|| body.get("error"))
4915 .and_then(Value::as_str)
4916 .unwrap_or("protocol version rejected")
4917 .to_string(),
4918 supported_version: body
4919 .get("supported_version")
4920 .and_then(Value::as_str)
4921 .map(str::to_string),
4922 requested_version: body
4923 .get("requested_version")
4924 .and_then(Value::as_str)
4925 .map(str::to_string),
4926 body,
4927 })
4928}
4929
4930fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4931 timeout
4932 .as_secs()
4933 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4934 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4935}
4936
4937fn worker_operation_is_retryable(error: &Error) -> bool {
4938 if worker_poll_capacity_retry_after(error).is_some()
4939 || worker_storage_admission_body(error).is_some()
4940 || worker_backend_unavailable_body(error).is_some()
4941 || worker_operation_is_explicitly_non_retryable(error)
4942 {
4943 return false;
4944 }
4945
4946 match error {
4947 Error::Transport(error) => {
4948 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4949 }
4950 Error::Http { status, .. } => {
4951 matches!(
4952 *status,
4953 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4954 ) || status.is_server_error()
4955 }
4956 _ => false,
4957 }
4958}
4959
4960fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4961 let body: Value = match error {
4962 Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4963 Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4964 _ => return None,
4965 };
4966 matches!(
4967 body.get("reason").and_then(Value::as_str),
4968 Some("storage_pressure" | "storage_admission_unavailable")
4969 )
4970 .then_some(body)
4971}
4972
4973fn worker_storage_admission_retry_after(
4974 error: &Error,
4975 poll_request_id: Option<&str>,
4976) -> Option<Duration> {
4977 let Error::Http { status, .. } = error else {
4978 return None;
4979 };
4980 let body = worker_storage_admission_body(error)?;
4981 let delay = body.get("retry_after_seconds")?.as_u64()?;
4982 if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4983 || delay == 0
4984 || body.get("retryable") != Some(&Value::Bool(true))
4985 || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4986 || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4987 || body
4988 .get("request_admitted")
4989 .is_some_and(|admitted| admitted != &Value::Bool(false))
4990 {
4991 return None;
4992 }
4993 match poll_request_id {
4994 Some(id) => {
4995 if id.is_empty()
4996 || body.get("task") != Some(&Value::Null)
4997 || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4998 || body.get("poll_status") != body.get("reason")
4999 || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
5000 || body.get("claim_admitted") != Some(&Value::Bool(false))
5001 {
5002 return None;
5003 }
5004 }
5005 None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
5006 None => {}
5007 }
5008 Some(Duration::from_secs(delay))
5009}
5010
5011fn worker_backend_unavailable_retry_after(
5012 error: &Error,
5013 path: &str,
5014 poll_request_id: Option<&str>,
5015) -> Option<Duration> {
5016 let operation = match path {
5017 "/worker/register" => "register_worker",
5018 "/worker/heartbeat" => "heartbeat_worker",
5019 "/worker/workflow-tasks/poll" => "poll_workflow_task",
5020 "/worker/activity-tasks/poll" => "poll_activity_task",
5021 "/worker/query-tasks/poll" => "poll_query_task",
5022 "/worker/update-validation-tasks/poll" => "poll_update_validation_task",
5023 _ => return None,
5024 };
5025 let body = worker_backend_unavailable_body(error)?;
5026 let delay = body.get("retry_after_seconds")?.as_u64()?;
5027 if delay == 0
5028 || body.get("operation")?.as_str()? != operation
5029 || body.get("outcome")?.as_str()? != "unknown"
5030 || body.get("retryable") != Some(&Value::Bool(true))
5031 || body.get("worker_id")?.as_str()?.is_empty()
5032 {
5033 return None;
5034 }
5035 if let Some(id) = poll_request_id {
5036 if id.is_empty()
5037 || body.get("task") != Some(&Value::Null)
5038 || body.get("poll_status")?.as_str()? != "backend_unavailable"
5039 || body.get("poll_request_id")?.as_str()? != id
5040 || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
5041 {
5042 return None;
5043 }
5044 } else if path.ends_with("/poll") {
5045 return None;
5046 }
5047 Some(Duration::from_secs(delay))
5048}
5049
5050fn worker_fenced_completion_retry_after(
5051 error: &Error,
5052 path: &str,
5053 request_body: Option<&Value>,
5054) -> Option<Duration> {
5055 let (operation, task_id, attempt_field) = if let Some(task_id) = path
5056 .strip_prefix("/worker/workflow-tasks/")
5057 .and_then(|tail| tail.strip_suffix("/complete"))
5058 {
5059 ("complete_workflow_task", task_id, "workflow_task_attempt")
5060 } else if let Some(task_id) = path
5061 .strip_prefix("/worker/activity-tasks/")
5062 .and_then(|tail| tail.strip_suffix("/complete"))
5063 {
5064 ("complete_activity_task", task_id, "activity_attempt_id")
5065 } else {
5066 return None;
5067 };
5068 if task_id.is_empty() || task_id.contains('/') {
5069 return None;
5070 }
5071
5072 let request = request_body?.as_object()?;
5073 let lease_owner = request.get("lease_owner")?.as_str()?;
5074 if lease_owner.is_empty() {
5075 return None;
5076 }
5077 let attempt = request.get(attempt_field)?;
5078 if (attempt_field == "workflow_task_attempt" && attempt.as_u64().filter(|id| *id > 0).is_none())
5079 || (attempt_field == "activity_attempt_id"
5080 && attempt.as_str().filter(|id| !id.is_empty()).is_none())
5081 {
5082 return None;
5083 }
5084
5085 let body = worker_backend_unavailable_body(error)?;
5086 let delay = body.get("retry_after_seconds")?.as_u64()?;
5087 if delay == 0
5088 || body.get("operation")?.as_str()? != operation
5089 || body.get("outcome")?.as_str()? != "unknown"
5090 || body.get("retryable") != Some(&Value::Bool(true))
5091 || body.get("worker_id")?.as_str()? != lease_owner
5092 || body.get("task_queue") != Some(&Value::Null)
5093 || body.get("task_id")?.as_str()? != task_id
5094 || body.get("lease_owner")?.as_str()? != lease_owner
5095 || body.get(attempt_field) != Some(attempt)
5096 {
5097 return None;
5098 }
5099
5100 Some(Duration::from_secs(delay))
5101}
5102
5103fn worker_backend_unavailable_body(error: &Error) -> Option<Value> {
5104 let Error::Http { status, body } = error else {
5105 return None;
5106 };
5107 if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE {
5108 return None;
5109 }
5110 let body: Value = serde_json::from_str(body).ok()?;
5111 (body.get("reason")?.as_str()? == "backend_unavailable").then_some(body)
5112}
5113
5114fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
5115 let Error::Http { body, .. } = error else {
5116 return false;
5117 };
5118
5119 serde_json::from_str::<Value>(body)
5120 .ok()
5121 .and_then(|body| body.get("retryable").and_then(Value::as_bool))
5122 == Some(false)
5123}
5124
5125fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
5126 let Error::Http { status, body } = error else {
5127 return None;
5128 };
5129 if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
5130 return None;
5131 }
5132
5133 let body = serde_json::from_str::<Value>(body).ok()?;
5134 let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
5135 == Some("long_poll_capacity_exhausted")
5136 || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
5137 if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
5138 return None;
5139 }
5140
5141 Some(Duration::from_secs(
5142 body.get("retry_after_seconds")
5143 .and_then(Value::as_u64)
5144 .unwrap_or_default(),
5145 ))
5146}
5147
5148fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
5149 let exponent = retry.saturating_sub(1).min(31) as u32;
5150 policy
5151 .initial_backoff
5152 .saturating_mul(1_u32 << exponent)
5153 .min(policy.max_backoff)
5154}
5155
5156#[derive(Debug)]
5157pub struct ClientBuilder {
5158 base_url: String,
5159 token: Option<String>,
5160 control_token: Option<String>,
5161 worker_token: Option<String>,
5162 namespace: String,
5163 timeout: Duration,
5164 max_external_payload_bytes: usize,
5165}
5166
5167impl ClientBuilder {
5168 pub fn token(mut self, token: Option<String>) -> Self {
5169 self.token = token;
5170 self
5171 }
5172
5173 pub fn control_token(mut self, token: Option<String>) -> Self {
5174 self.control_token = token;
5175 self
5176 }
5177
5178 pub fn worker_token(mut self, token: Option<String>) -> Self {
5179 self.worker_token = token;
5180 self
5181 }
5182
5183 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
5184 self.namespace = namespace.into();
5185 self
5186 }
5187
5188 pub fn timeout(mut self, timeout: Duration) -> Self {
5189 self.timeout = timeout;
5190 self
5191 }
5192
5193 pub fn max_external_payload_bytes(mut self, bytes: usize) -> Self {
5196 self.max_external_payload_bytes = bytes;
5197 self
5198 }
5199
5200 pub fn build(self) -> Result<Client> {
5201 let base_url = self.base_url.trim_end_matches('/').to_string();
5202 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
5203 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
5204 .unwrap_or_else(|_| base_url.ends_with("/api"));
5205
5206 if has_sdk_api_suffix {
5207 return Err(Error::InvalidBaseUrl);
5208 }
5209
5210 Ok(Client {
5211 http: reqwest::Client::builder()
5212 .timeout(self.timeout)
5213 .redirect(reqwest::redirect::Policy::none())
5214 .build()?,
5215 base_url,
5216 token: self.token,
5217 control_token: self.control_token,
5218 worker_token: self.worker_token,
5219 namespace: self.namespace,
5220 max_external_payload_bytes: self.max_external_payload_bytes,
5221 worker_storage_admission: None,
5222 cooperative_worker_protocol: false,
5223 local_activities_enabled: false,
5224 worker_sessions_enabled: false,
5225 max_concurrent_worker_sessions: 10,
5226 worker_build_id: None,
5227 sticky_cache: None,
5228 runtime_upload_policy: Arc::new(Mutex::new([None, None])),
5229 })
5230 }
5231}
5232
5233#[derive(Clone, Debug)]
5234pub struct WorkflowHandle {
5235 client: Client,
5236 pub workflow_id: String,
5237 pub run_id: Option<String>,
5238 pub workflow_type: String,
5239}
5240
5241impl WorkflowHandle {
5242 pub async fn describe(&self) -> Result<WorkflowDescription> {
5244 self.client.describe_workflow(&self.workflow_id).await
5245 }
5246
5247 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
5249 let run_id = self.run_id.as_deref().ok_or_else(|| {
5250 Error::Codec("run_id is required for selected-run description".to_string())
5251 })?;
5252 self.client
5253 .describe_workflow_run(&self.workflow_id, run_id)
5254 .await
5255 }
5256
5257 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
5258 self.client
5259 .signal_workflow(&self.workflow_id, signal_name, input)
5260 .await
5261 }
5262
5263 pub async fn append_message<T: Serialize>(
5264 &self,
5265 stream_name: &str,
5266 message_id: &str,
5267 input: T,
5268 ) -> Result<Value> {
5269 self.client
5270 .append_message_stream(&self.workflow_id, stream_name, message_id, input)
5271 .await
5272 }
5273
5274 pub async fn signal_selected_run<T: Serialize>(
5276 &self,
5277 signal_name: &str,
5278 input: T,
5279 ) -> Result<Value> {
5280 let run_id = self.run_id.as_deref().ok_or_else(|| {
5281 Error::Codec("run_id is required for selected-run signaling".to_string())
5282 })?;
5283 self.client
5284 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
5285 .await
5286 }
5287
5288 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
5290 self.client
5291 .cancel_workflow(&self.workflow_id, options)
5292 .await
5293 }
5294
5295 pub async fn cancel_selected_run(
5297 &self,
5298 options: WorkflowCommandOptions,
5299 ) -> Result<WorkflowCommandResult> {
5300 let run_id = self.run_id.as_deref().ok_or_else(|| {
5301 Error::Codec("run_id is required for selected-run cancellation".to_string())
5302 })?;
5303 self.client
5304 .cancel_workflow_run(&self.workflow_id, run_id, options)
5305 .await
5306 }
5307
5308 pub async fn terminate(
5310 &self,
5311 options: WorkflowCommandOptions,
5312 ) -> Result<WorkflowCommandResult> {
5313 self.client
5314 .terminate_workflow(&self.workflow_id, options)
5315 .await
5316 }
5317
5318 pub async fn terminate_selected_run(
5320 &self,
5321 options: WorkflowCommandOptions,
5322 ) -> Result<WorkflowCommandResult> {
5323 let run_id = self.run_id.as_deref().ok_or_else(|| {
5324 Error::Codec("run_id is required for selected-run termination".to_string())
5325 })?;
5326 self.client
5327 .terminate_workflow_run(&self.workflow_id, run_id, options)
5328 .await
5329 }
5330
5331 pub async fn redrive(&self, request_id: Option<&str>) -> Result<WorkflowRedriveResult> {
5333 let run_id = self
5334 .run_id
5335 .as_deref()
5336 .ok_or_else(|| Error::Codec("run_id is required for redrive".to_string()))?;
5337 self.client
5338 .redrive_workflow_run(&self.workflow_id, run_id, request_id)
5339 .await
5340 }
5341
5342 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
5344 self.client
5345 .query_workflow(&self.workflow_id, query_name, input)
5346 .await
5347 }
5348
5349 pub async fn query_avro_value<T: Serialize>(
5350 &self,
5351 query_name: &str,
5352 input: T,
5353 ) -> Result<AvroValue> {
5354 self.client
5355 .query_workflow_avro_value(&self.workflow_id, query_name, input)
5356 .await
5357 }
5358
5359 pub async fn update<T: Serialize>(
5360 &self,
5361 update_name: &str,
5362 input: T,
5363 request_id: Option<&str>,
5364 ) -> Result<Value> {
5365 self.client
5366 .update_workflow(&self.workflow_id, update_name, input, request_id)
5367 .await
5368 }
5369
5370 pub async fn update_avro_value<T: Serialize>(
5371 &self,
5372 update_name: &str,
5373 input: T,
5374 request_id: Option<&str>,
5375 ) -> Result<AvroValue> {
5376 self.client
5377 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
5378 .await
5379 }
5380
5381 pub async fn query_selected_run<T: Serialize>(
5383 &self,
5384 query_name: &str,
5385 input: T,
5386 ) -> Result<Value> {
5387 let run_id = self
5388 .run_id
5389 .as_deref()
5390 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
5391 self.client
5392 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
5393 .await
5394 }
5395
5396 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
5398 self.result_target(options, None).await
5399 }
5400
5401 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
5403 self.result_avro_value_target(options, None).await
5404 }
5405
5406 pub async fn result_typed<T: DeserializeOwned>(
5408 &self,
5409 options: WorkflowResultOptions,
5410 ) -> Result<T> {
5411 let result = self.result_avro_value(options).await?;
5412 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
5413 }
5414
5415 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
5417 let run_id = self.run_id.as_deref().ok_or_else(|| {
5418 Error::Codec("run_id is required for selected-run result".to_string())
5419 })?;
5420 self.result_target(options, Some(run_id)).await
5421 }
5422
5423 pub async fn result_selected_run_avro_value(
5425 &self,
5426 options: WorkflowResultOptions,
5427 ) -> Result<AvroValue> {
5428 let run_id = self.run_id.as_deref().ok_or_else(|| {
5429 Error::Codec("run_id is required for selected-run result".to_string())
5430 })?;
5431 self.result_avro_value_target(options, Some(run_id)).await
5432 }
5433
5434 pub async fn result_selected_run_typed<T: DeserializeOwned>(
5436 &self,
5437 options: WorkflowResultOptions,
5438 ) -> Result<T> {
5439 let result = self.result_selected_run_avro_value(options).await?;
5440 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
5441 }
5442
5443 async fn result_avro_value_target(
5444 &self,
5445 options: WorkflowResultOptions,
5446 selected_run_id: Option<&str>,
5447 ) -> Result<AvroValue> {
5448 let started = Instant::now();
5449
5450 loop {
5451 let description = match selected_run_id {
5452 Some(run_id) => {
5453 self.client
5454 .describe_workflow_run(&self.workflow_id, run_id)
5455 .await?
5456 }
5457 None => self.describe().await?,
5458 };
5459 if description.is_completed() {
5460 return description.output_avro_value.ok_or_else(|| {
5461 Error::Codec(
5462 "missing_payload_envelope: typed workflow result requires output_envelope"
5463 .to_string(),
5464 )
5465 });
5466 }
5467 if description.is_terminal() {
5468 let outcome =
5469 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
5470 return Err(match outcome.kind {
5471 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
5472 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5473 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5474 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5475 });
5476 }
5477 if started.elapsed() >= options.timeout {
5478 return Err(Error::Timeout);
5479 }
5480 tokio::time::sleep(options.poll_interval).await;
5481 }
5482 }
5483
5484 async fn result_target(
5485 &self,
5486 options: WorkflowResultOptions,
5487 selected_run_id: Option<&str>,
5488 ) -> Result<Value> {
5489 let started = Instant::now();
5490
5491 loop {
5492 let description = match selected_run_id {
5493 Some(run_id) => {
5494 self.client
5495 .describe_workflow_run(&self.workflow_id, run_id)
5496 .await?
5497 }
5498 None => self.describe().await?,
5499 };
5500 if description.is_completed() {
5501 return Ok(description.output.unwrap_or(Value::Null));
5502 }
5503
5504 if description.is_terminal() {
5505 let outcome =
5506 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
5507 return Err(match outcome.kind {
5508 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
5509 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5510 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5511 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5512 });
5513 }
5514
5515 if started.elapsed() >= options.timeout {
5516 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
5517 kind: WorkflowTerminalKind::TimedOut,
5518 workflow_id: description
5519 .workflow_id
5520 .clone()
5521 .unwrap_or_else(|| self.workflow_id.clone()),
5522 run_id: description
5523 .run_id
5524 .clone()
5525 .or_else(|| selected_run_id.map(str::to_string)),
5526 reason: "result_wait_timeout".to_string(),
5527 failure_category: Some("client_timeout".to_string()),
5528 failure_id: None,
5529 exception_type: None,
5530 exception_class: None,
5531 non_retryable: None,
5532 message: Some(format!(
5533 "workflow result was not terminal within {:?}",
5534 options.timeout
5535 )),
5536 exception: None,
5537 raw: description.raw_value(),
5538 }));
5539 }
5540
5541 tokio::time::sleep(options.poll_interval).await;
5542 }
5543 }
5544}
5545
5546#[derive(Clone, Copy, Debug)]
5547pub struct WorkflowResultOptions {
5548 pub poll_interval: Duration,
5549 pub timeout: Duration,
5550}
5551
5552impl Default for WorkflowResultOptions {
5553 fn default() -> Self {
5554 Self {
5555 poll_interval: Duration::from_millis(500),
5556 timeout: Duration::from_secs(30),
5557 }
5558 }
5559}
5560
5561#[derive(Clone, Debug, Deserialize)]
5562pub struct WorkflowDescription {
5563 pub workflow_id: Option<String>,
5564 pub run_id: Option<String>,
5565 pub workflow_type: Option<String>,
5566 pub status: Option<String>,
5567 #[serde(default)]
5568 pub closed_reason: Option<String>,
5569 #[serde(default)]
5570 pub error: Option<String>,
5571 #[serde(default)]
5572 pub failure: Option<Value>,
5573 #[serde(default)]
5574 pub exception: Option<Value>,
5575 #[serde(default)]
5576 pub failures: Vec<Value>,
5577 #[serde(default)]
5578 pub output: Option<Value>,
5579 #[serde(default)]
5580 pub output_envelope: Option<Value>,
5581 #[serde(skip)]
5582 pub output_avro_value: Option<AvroValue>,
5583 #[serde(flatten)]
5584 pub raw: HashMap<String, Value>,
5585}
5586
5587#[derive(Clone, Debug, Deserialize)]
5589pub struct WorkflowStreamDescription {
5590 pub stream_name: String,
5591 pub status: String,
5592 pub last_offset: i64,
5593 pub total_items: u64,
5594 pub pending_items: u64,
5595 #[serde(default)]
5596 pub opened_at: Option<String>,
5597 #[serde(default)]
5598 pub last_appended_at: Option<String>,
5599 #[serde(default)]
5600 pub closed_at: Option<String>,
5601 #[serde(default)]
5602 pub error_reason: Option<String>,
5603 #[serde(default)]
5604 pub retention_seconds: Option<u64>,
5605 #[serde(flatten)]
5606 pub raw: HashMap<String, Value>,
5607}
5608
5609impl WorkflowStreamDescription {
5610 pub fn is_terminal(&self) -> bool {
5611 matches!(self.status.as_str(), "closed" | "errored")
5612 }
5613}
5614
5615#[derive(Clone, Debug, Default)]
5617pub struct WorkflowStreamAppendItem {
5618 pub payload_envelope: Option<Value>,
5619 pub payload_reference: Option<String>,
5620 pub item_type: Option<String>,
5621 pub content_type: Option<String>,
5622 pub idempotency_key: Option<String>,
5623}
5624
5625impl WorkflowStreamAppendItem {
5626 pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5628 let value = AvroValue::from_serialize(&payload)?;
5629 Ok(Self {
5630 payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5631 ..Self::default()
5632 })
5633 }
5634
5635 pub fn from_reference(reference: impl Into<String>) -> Self {
5637 Self {
5638 payload_reference: Some(reference.into()),
5639 ..Self::default()
5640 }
5641 }
5642
5643 pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5644 self.item_type = Some(item_type.into());
5645 self
5646 }
5647
5648 pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5649 self.content_type = Some(content_type.into());
5650 self
5651 }
5652
5653 pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5654 self.idempotency_key = Some(idempotency_key.into());
5655 self
5656 }
5657
5658 fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5659 let mut item = serde_json::Map::new();
5660 if let Some(payload) = &self.payload_envelope {
5661 item.insert("payload".to_string(), payload.clone());
5662 item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5663 }
5664 if let Some(reference) = &self.payload_reference {
5665 item.insert("payload_reference".to_string(), json!(reference));
5666 }
5667 if let Some(item_type) = &self.item_type {
5668 item.insert("item_type".to_string(), json!(item_type));
5669 }
5670 if let Some(content_type) = &self.content_type {
5671 item.insert("content_type".to_string(), json!(content_type));
5672 }
5673 if let Some(key) = derived_idempotency_key
5674 .as_ref()
5675 .or(self.idempotency_key.as_ref())
5676 {
5677 item.insert("idempotency_key".to_string(), json!(key));
5678 }
5679 Value::Object(item)
5680 }
5681}
5682
5683#[derive(Clone, Debug)]
5685pub struct WorkflowStreamItem {
5686 pub offset: u64,
5687 pub payload: Option<Value>,
5688 pub payload_envelope: Option<Value>,
5689 pub payload_reference: Option<String>,
5690 pub payload_codec: Option<String>,
5691 pub idempotency_key: Option<String>,
5692 pub item_type: Option<String>,
5693 pub content_type: Option<String>,
5694 pub origin: Option<String>,
5695 pub origin_reference: Option<String>,
5696 pub emitted_at: Option<String>,
5697 pub raw: Value,
5698}
5699
5700#[derive(Clone, Debug)]
5702pub struct WorkflowStreamPage {
5703 pub stream: WorkflowStreamDescription,
5704 pub items: Vec<WorkflowStreamItem>,
5705 pub next_offset: u64,
5706 pub terminal: bool,
5707}
5708
5709#[derive(Clone, Debug)]
5711pub struct WorkflowStreamAppendResult {
5712 pub stream: WorkflowStreamDescription,
5713 pub accepted_offsets: Vec<u64>,
5714 pub accepted: u64,
5715 pub deduped: u64,
5716}
5717
5718#[derive(Deserialize)]
5719struct WorkflowStreamListResponse {
5720 #[serde(default)]
5721 streams: Vec<WorkflowStreamDescription>,
5722}
5723
5724#[derive(Deserialize)]
5725struct WorkflowStreamDescriptionResponse {
5726 stream: WorkflowStreamDescription,
5727}
5728
5729#[derive(Deserialize)]
5730struct WorkflowStreamPageResponse {
5731 stream: WorkflowStreamDescription,
5732 #[serde(default)]
5733 items: Vec<Value>,
5734 next_offset: u64,
5735 terminal: bool,
5736}
5737
5738#[derive(Deserialize)]
5739struct WorkflowStreamAppendResponse {
5740 stream: WorkflowStreamDescription,
5741 #[serde(default)]
5742 accepted_offsets: Vec<u64>,
5743 accepted: u64,
5744 deduped: u64,
5745}
5746
5747impl WorkflowDescription {
5748 pub fn is_completed(&self) -> bool {
5749 matches!(self.status.as_deref(), Some("completed" | "Completed"))
5750 }
5751
5752 pub fn is_terminal(&self) -> bool {
5753 matches!(
5754 self.status.as_deref(),
5755 Some(
5756 "completed"
5757 | "Completed"
5758 | "failed"
5759 | "Failed"
5760 | "cancelled"
5761 | "Cancelled"
5762 | "terminated"
5763 | "Terminated"
5764 | "timed_out"
5765 | "TimedOut",
5766 )
5767 )
5768 }
5769
5770 fn decode_payloads(&mut self) -> Result<()> {
5771 if let Some(envelope) = &self.output_envelope {
5772 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5773 self.output = Some(value.clone().into_json()?);
5774 self.output_avro_value = Some(value);
5775 }
5776
5777 Ok(())
5778 }
5779
5780 fn raw_value(&self) -> Value {
5781 let mut data = self.raw.clone();
5782 data.insert(
5783 "workflow_id".to_string(),
5784 self.workflow_id
5785 .clone()
5786 .map(Value::String)
5787 .unwrap_or(Value::Null),
5788 );
5789 data.insert(
5790 "run_id".to_string(),
5791 self.run_id
5792 .clone()
5793 .map(Value::String)
5794 .unwrap_or(Value::Null),
5795 );
5796 data.insert(
5797 "workflow_type".to_string(),
5798 self.workflow_type
5799 .clone()
5800 .map(Value::String)
5801 .unwrap_or(Value::Null),
5802 );
5803 data.insert(
5804 "status".to_string(),
5805 self.status
5806 .clone()
5807 .map(Value::String)
5808 .unwrap_or(Value::Null),
5809 );
5810 data.insert(
5811 "closed_reason".to_string(),
5812 self.closed_reason
5813 .clone()
5814 .map(Value::String)
5815 .unwrap_or(Value::Null),
5816 );
5817 if let Some(failure) = &self.failure {
5818 data.insert("failure".to_string(), failure.clone());
5819 }
5820 if let Some(exception) = &self.exception {
5821 data.insert("exception".to_string(), exception.clone());
5822 }
5823 Value::Object(data.into_iter().collect())
5824 }
5825}
5826
5827fn workflow_terminal_outcome(
5828 description: &WorkflowDescription,
5829 workflow_id: &str,
5830 run_id: Option<&str>,
5831) -> WorkflowTerminalOutcome {
5832 let terminal_kind = description
5833 .closed_reason
5834 .as_deref()
5835 .or(description.status.as_deref())
5836 .unwrap_or("failed")
5837 .to_ascii_lowercase();
5838 let kind = match terminal_kind.as_str() {
5839 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5840 "terminated" => WorkflowTerminalKind::Terminated,
5841 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5842 _ => WorkflowTerminalKind::Failed,
5843 };
5844 let default_reason = match kind {
5845 WorkflowTerminalKind::Failed => "workflow_failed",
5846 WorkflowTerminalKind::Cancelled => "cancelled",
5847 WorkflowTerminalKind::Terminated => "terminated",
5848 WorkflowTerminalKind::TimedOut => "timed_out",
5849 };
5850 let failure = description
5851 .failure
5852 .as_ref()
5853 .filter(|value| value.is_object());
5854 let nested_failure = failure
5855 .and_then(|value| value.get("failures"))
5856 .and_then(Value::as_array)
5857 .and_then(|failures| failures.last())
5858 .or_else(|| description.failures.last());
5859 let exception = description
5860 .exception
5861 .clone()
5862 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5863 .or_else(|| {
5864 nested_failure
5865 .and_then(|value| value.get("exception_payload"))
5866 .cloned()
5867 });
5868 let string_field = |name: &str| {
5869 failure
5870 .and_then(|value| value.get(name))
5871 .and_then(Value::as_str)
5872 .or_else(|| {
5873 nested_failure
5874 .and_then(|value| value.get(name))
5875 .and_then(Value::as_str)
5876 })
5877 .map(str::to_string)
5878 };
5879 let exception_field = |name: &str| {
5880 exception
5881 .as_ref()
5882 .and_then(|value| value.get(name))
5883 .and_then(Value::as_str)
5884 .map(str::to_string)
5885 };
5886 let message = description
5887 .error
5888 .clone()
5889 .or_else(|| string_field("message"))
5890 .or_else(|| exception_field("message"));
5891 let reason = description
5892 .raw
5893 .get("reason")
5894 .and_then(Value::as_str)
5895 .map(str::to_string)
5896 .or_else(|| {
5897 failure
5898 .and_then(|value| value.get("reason"))
5899 .and_then(Value::as_str)
5900 .map(str::to_string)
5901 })
5902 .or_else(|| description.closed_reason.clone())
5903 .unwrap_or_else(|| default_reason.to_string());
5904 let failure_id = string_field("failure_id").or_else(|| {
5905 nested_failure
5906 .and_then(|value| value.get("id"))
5907 .and_then(Value::as_str)
5908 .map(str::to_string)
5909 });
5910
5911 WorkflowTerminalOutcome {
5912 kind,
5913 workflow_id: description
5914 .workflow_id
5915 .clone()
5916 .unwrap_or_else(|| workflow_id.to_string()),
5917 run_id: description
5918 .run_id
5919 .clone()
5920 .or_else(|| run_id.map(str::to_string)),
5921 reason,
5922 failure_category: string_field("failure_category")
5923 .or_else(|| Some(default_reason.to_string())),
5924 failure_id,
5925 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5926 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5927 non_retryable: failure
5928 .and_then(|value| value.get("non_retryable"))
5929 .and_then(Value::as_bool)
5930 .or_else(|| {
5931 nested_failure
5932 .and_then(|value| value.get("non_retryable"))
5933 .and_then(Value::as_bool)
5934 }),
5935 message,
5936 exception,
5937 raw: description.raw_value(),
5938 }
5939}
5940
5941#[derive(Clone, Debug, Deserialize)]
5942pub struct RegisterWorkerResponse {
5943 pub worker_id: String,
5944 pub registered: bool,
5945 #[serde(default)]
5946 pub heartbeat_interval_seconds: Option<u64>,
5947 #[serde(default)]
5948 pub protocol_version: Option<String>,
5949 #[serde(default)]
5950 pub server_capabilities: Option<Value>,
5951}
5952
5953#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5955pub struct WorkerDeregistrationEnvelope {
5956 pub worker_id: String,
5957 pub outcome: String,
5958 pub recovered_workflow_task_count: u64,
5959}
5960
5961#[derive(Clone, Debug, Deserialize)]
5962pub struct PollWorkflowTaskResponse {
5963 #[serde(default)]
5964 pub task: Option<WorkflowTask>,
5965 #[serde(default)]
5966 pub poll_status: Option<String>,
5967 #[serde(default)]
5968 pub reason: Option<String>,
5969 #[serde(default)]
5970 pub protocol_version: Option<String>,
5971 #[serde(default)]
5972 pub server_capabilities: Option<Value>,
5973}
5974
5975impl PollWorkflowTaskResponse {
5976 pub fn outcome(&self) -> WorkerPollOutcome {
5978 worker_poll_outcome(
5979 self.task.is_some(),
5980 self.poll_status.as_deref(),
5981 self.reason.as_deref(),
5982 )
5983 }
5984}
5985
5986fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5987 let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5988 return false;
5989 };
5990 let supported = capabilities
5991 .get("workflow_memo_updates")
5992 .and_then(Value::as_object)
5993 .and_then(|memo| memo.get("supported"))
5994 .and_then(Value::as_bool)
5995 == Some(true);
5996 let command_advertised = capabilities
5997 .get("supported_workflow_task_commands")
5998 .and_then(Value::as_array)
5999 .is_some_and(|commands| {
6000 commands
6001 .iter()
6002 .any(|command| command.as_str() == Some("upsert_memo"))
6003 });
6004 supported && command_advertised
6005}
6006
6007fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
6008 commands
6009 .iter()
6010 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
6011}
6012
6013#[derive(Clone, Debug, Deserialize)]
6014pub struct PollActivityTaskResponse {
6015 #[serde(default)]
6016 pub task: Option<ActivityTask>,
6017 #[serde(default)]
6018 pub poll_status: Option<String>,
6019 #[serde(default)]
6020 pub reason: Option<String>,
6021}
6022
6023impl PollActivityTaskResponse {
6024 pub fn outcome(&self) -> WorkerPollOutcome {
6026 worker_poll_outcome(
6027 self.task.is_some(),
6028 self.poll_status.as_deref(),
6029 self.reason.as_deref(),
6030 )
6031 }
6032}
6033
6034#[derive(Clone, Debug, Deserialize)]
6035pub struct PollQueryTaskResponse {
6036 #[serde(default)]
6037 pub task: Option<QueryTask>,
6038 #[serde(default)]
6039 pub poll_status: Option<String>,
6040 #[serde(default)]
6041 pub reason: Option<String>,
6042}
6043
6044impl PollQueryTaskResponse {
6045 pub fn outcome(&self) -> WorkerPollOutcome {
6047 worker_poll_outcome(
6048 self.task.is_some(),
6049 self.poll_status.as_deref(),
6050 self.reason.as_deref(),
6051 )
6052 }
6053}
6054
6055#[derive(Clone, Debug, PartialEq, Eq)]
6057pub enum WorkerPollOutcome {
6058 Task,
6060 Idle {
6062 poll_status: Option<String>,
6063 reason: Option<String>,
6064 },
6065 Stop {
6067 poll_status: Option<String>,
6068 reason: Option<String>,
6069 },
6070}
6071
6072impl WorkerPollOutcome {
6073 pub fn should_stop(&self) -> bool {
6074 matches!(self, Self::Stop { .. })
6075 }
6076}
6077
6078fn worker_poll_outcome(
6079 has_task: bool,
6080 poll_status: Option<&str>,
6081 reason: Option<&str>,
6082) -> WorkerPollOutcome {
6083 if worker_poll_is_stop(poll_status, reason) {
6084 return WorkerPollOutcome::Stop {
6085 poll_status: poll_status.map(str::to_string),
6086 reason: reason.map(str::to_string),
6087 };
6088 }
6089
6090 if has_task {
6091 WorkerPollOutcome::Task
6092 } else {
6093 WorkerPollOutcome::Idle {
6094 poll_status: poll_status.map(str::to_string),
6095 reason: reason.map(str::to_string),
6096 }
6097 }
6098}
6099
6100#[derive(Clone, Debug, Deserialize)]
6102pub struct QueryTask {
6103 pub query_task_id: String,
6104 #[serde(default = "default_workflow_task_attempt")]
6105 pub query_task_attempt: u64,
6106 #[serde(default)]
6107 pub lease_owner: Option<String>,
6108 #[serde(default)]
6109 pub workflow_id: Option<String>,
6110 #[serde(default)]
6111 pub run_id: Option<String>,
6112 pub workflow_type: String,
6113 pub query_name: String,
6114 #[serde(
6115 default = "missing_task_payload_codec",
6116 deserialize_with = "deserialize_task_payload_codec"
6117 )]
6118 pub payload_codec: String,
6119 #[serde(default)]
6120 pub workflow_arguments: Option<Value>,
6121 #[serde(default)]
6122 pub query_arguments: Option<Value>,
6123 #[serde(default)]
6124 pub history_events: Vec<HistoryEvent>,
6125 #[serde(default)]
6126 pub history_export: Option<Value>,
6127 #[serde(default)]
6128 pub run_status: Option<String>,
6129}
6130
6131#[derive(Clone, Debug, Deserialize)]
6132pub struct WorkflowTask {
6133 pub task_id: String,
6134 #[serde(default)]
6135 pub workflow_command_id: Option<String>,
6136 #[serde(default)]
6137 pub workflow_id: Option<String>,
6138 #[serde(default)]
6139 pub run_id: Option<String>,
6140 pub workflow_type: String,
6141 #[serde(default)]
6142 pub cancel_requested: bool,
6143 #[serde(
6144 default = "missing_task_payload_codec",
6145 deserialize_with = "deserialize_task_payload_codec"
6146 )]
6147 pub payload_codec: String,
6148 #[serde(default)]
6149 pub arguments: Option<Value>,
6150 #[serde(default)]
6151 pub history_events: Vec<HistoryEvent>,
6152 #[serde(default)]
6153 pub total_history_events: Option<u64>,
6154 #[serde(default)]
6155 pub history_size_bytes: Option<u64>,
6156 #[serde(default)]
6157 pub continue_as_new_recommended: Option<bool>,
6158 #[serde(default)]
6159 pub history_budget_pressure: Option<String>,
6160 #[serde(default)]
6161 pub next_history_page_token: Option<String>,
6162 #[serde(default = "default_workflow_task_attempt")]
6163 pub workflow_task_attempt: u64,
6164 #[serde(default)]
6165 pub workflow_signal_id: Option<String>,
6166 #[serde(default)]
6167 pub signal_name: Option<String>,
6168 #[serde(default)]
6169 pub signal_arguments: Option<Value>,
6170 #[serde(default)]
6171 pub workflow_update_id: Option<String>,
6172 #[serde(default)]
6173 pub update_name: Option<String>,
6174 #[serde(default)]
6175 pub lease_owner: Option<String>,
6176}
6177
6178impl WorkflowTask {
6179 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
6180 self.history_events.extend(page.history_events);
6181
6182 if page.total_history_events.is_some() {
6183 self.total_history_events = page.total_history_events;
6184 }
6185
6186 self.next_history_page_token = page
6187 .next_history_page_token
6188 .filter(|token| !token.is_empty());
6189 }
6190}
6191
6192#[derive(Clone, Debug, Deserialize)]
6193struct WorkflowTaskHistoryPage {
6194 #[serde(default)]
6195 history_events: Vec<HistoryEvent>,
6196 #[serde(default)]
6197 total_history_events: Option<u64>,
6198 #[serde(default)]
6199 next_history_page_token: Option<String>,
6200}
6201
6202#[derive(Clone, Debug, Deserialize)]
6203pub struct ActivityTask {
6204 pub task_id: String,
6205 #[serde(default)]
6206 pub activity_attempt_id: Option<String>,
6207 #[serde(default)]
6208 pub attempt_id: Option<String>,
6209 pub activity_type: String,
6210 #[serde(
6211 default = "missing_task_payload_codec",
6212 deserialize_with = "deserialize_task_payload_codec"
6213 )]
6214 pub payload_codec: String,
6215 #[serde(default)]
6216 pub arguments: Option<Value>,
6217 #[serde(default = "default_attempt_number")]
6218 pub attempt_number: u64,
6219 #[serde(default)]
6220 pub lease_owner: Option<String>,
6221}
6222
6223#[derive(Clone, Debug, Serialize, Deserialize)]
6224pub struct HistoryEvent {
6225 #[serde(alias = "type")]
6226 pub event_type: String,
6227 #[serde(default)]
6228 pub payload: Value,
6229 #[serde(flatten)]
6230 pub raw: HashMap<String, Value>,
6231}
6232
6233#[derive(Clone, Debug, PartialEq)]
6235pub struct QuerySignal {
6236 pub id: Option<String>,
6237 pub name: String,
6238 pub arguments: Vec<Value>,
6239 avro_arguments: Vec<AvroValue>,
6240 pub workflow_sequence: Option<u64>,
6241}
6242
6243impl QuerySignal {
6244 pub fn arguments_avro_value(&self) -> &[AvroValue] {
6246 &self.avro_arguments
6247 }
6248}
6249
6250#[derive(Clone, Debug)]
6256pub struct QueryContext {
6257 pub workflow_id: Option<String>,
6258 pub run_id: Option<String>,
6259 pub workflow_type: String,
6260 pub run_status: Option<String>,
6261 workflow_input: Value,
6262 workflow_input_avro_value: AvroValue,
6263 history_events: Arc<Vec<HistoryEvent>>,
6264 signal_events: Arc<Vec<QuerySignal>>,
6265}
6266
6267impl QueryContext {
6268 pub fn workflow_input(&self) -> &Value {
6270 &self.workflow_input
6271 }
6272
6273 pub fn workflow_input_avro_value(&self) -> &AvroValue {
6275 &self.workflow_input_avro_value
6276 }
6277
6278 pub fn history_events(&self) -> &[HistoryEvent] {
6280 self.history_events.as_slice()
6281 }
6282
6283 pub fn signal_events(&self) -> &[QuerySignal] {
6285 self.signal_events.as_slice()
6286 }
6287
6288 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
6290 self.signal_events
6291 .iter()
6292 .filter(|signal| signal.name == signal_name)
6293 .map(|signal| signal.arguments.clone())
6294 .collect()
6295 }
6296
6297 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
6299 self.signal_events
6300 .iter()
6301 .filter(|signal| signal.name == signal_name)
6302 .map(|signal| signal.avro_arguments.clone())
6303 .collect()
6304 }
6305}
6306
6307#[derive(Clone, Debug, Deserialize)]
6308pub struct ActivityHeartbeatResponse {
6309 #[serde(default)]
6310 pub cancel_requested: bool,
6311 #[serde(default)]
6312 pub heartbeat_recorded: bool,
6313 #[serde(default)]
6314 pub can_continue: Option<bool>,
6315 #[serde(default)]
6316 pub reason: Option<String>,
6317 #[serde(default)]
6318 pub run_closed_reason: Option<String>,
6319 #[serde(default)]
6320 pub run_closed_at: Option<String>,
6321 #[serde(default)]
6322 pub lease_expires_at: Option<String>,
6323 #[serde(default)]
6324 pub last_heartbeat_at: Option<String>,
6325}
6326
6327impl ActivityHeartbeatResponse {
6328 pub fn should_stop(&self) -> bool {
6330 self.cancel_requested || self.can_continue == Some(false)
6331 }
6332}
6333
6334fn missing_task_payload_codec() -> String {
6335 MISSING_TASK_PAYLOAD_CODEC.to_string()
6336}
6337
6338fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
6339where
6340 D: Deserializer<'de>,
6341{
6342 Ok(match Value::deserialize(deserializer)? {
6343 Value::String(codec) => codec,
6344 Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
6345 _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
6346 })
6347}
6348
6349fn default_workflow_task_attempt() -> u64 {
6350 1
6351}
6352
6353fn default_attempt_number() -> u64 {
6354 1
6355}
6356
6357type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
6358type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
6359type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
6360type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
6361type ReplayedWorkflowHandler =
6362 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
6363type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
6364type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
6365type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
6366type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
6367type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
6368type ReplayedQueryHandler = Arc<
6369 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
6370 + Send
6371 + Sync,
6372>;
6373type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
6374
6375struct ReplayedWorkflowInvocation {
6376 future: WorkflowFuture,
6377 snapshot: WorkflowStateSnapshot,
6378}
6379
6380#[derive(Clone)]
6381struct RegisteredWorkflow {
6382 execute: WorkflowHandler,
6383 replay: Option<ReplayedWorkflowHandler>,
6384 state_type: Option<TypeId>,
6385 definition_fingerprint: Option<String>,
6386}
6387
6388fn argument_vector_contracts(names: &[String]) -> Vec<Value> {
6389 names
6390 .iter()
6391 .map(|name| {
6392 json!({
6393 "name": name,
6394 "parameters": [{
6395 "name": "args",
6396 "position": 0,
6397 "required": false,
6398 "variadic": true,
6399 "type": null,
6400 "allows_null": true,
6401 "default_available": false,
6402 "default": null,
6403 }],
6404 })
6405 })
6406 .collect()
6407}
6408
6409#[derive(Debug)]
6410struct WorkflowTaskDecision {
6411 commands: Vec<Value>,
6412 message_stream_cursors: Vec<Value>,
6413 message_stream_waits: Vec<Value>,
6414 cancellation_delivery: Option<CancellationDelivery>,
6415 cancellation_scope_opening: Option<cancellation_scope::CancellationScopeOpening>,
6416 cancellation_scope_delivery: Option<cancellation_scope_replay::ScopeDeliveryIntent>,
6417}
6418
6419enum PreparedWorkflowTask {
6420 Decision(WorkflowTaskDecision),
6421 Execution {
6422 task: WorkflowTask,
6423 context: WorkflowContext,
6424 future: WorkflowFuture,
6425 },
6426}
6427
6428impl WorkflowTaskDecision {
6429 fn without_message_streams(commands: Vec<Value>) -> Self {
6430 Self {
6431 commands,
6432 message_stream_cursors: Vec::new(),
6433 message_stream_waits: Vec::new(),
6434 cancellation_delivery: None,
6435 cancellation_scope_opening: None,
6436 cancellation_scope_delivery: None,
6437 }
6438 }
6439}
6440
6441#[derive(Clone)]
6442enum RegisteredQuery {
6443 Snapshot(QueryHandler),
6444 Replayed {
6445 state_type: TypeId,
6446 handler: ReplayedQueryHandler,
6447 },
6448}
6449
6450#[derive(Clone, Debug)]
6451pub struct WorkerHeartbeatObservation {
6452 pub worker_id: String,
6453 pub task_queue: String,
6454 pub acknowledged_at_unix_millis: u64,
6455 pub acknowledgement: Value,
6456}
6457
6458#[derive(Clone, Copy, Debug)]
6473pub struct WorkerRetryPolicy {
6474 pub max_retries: usize,
6476 pub initial_backoff: Duration,
6478 pub max_backoff: Duration,
6480}
6481
6482impl Default for WorkerRetryPolicy {
6483 fn default() -> Self {
6484 Self {
6485 max_retries: 5,
6486 initial_backoff: Duration::from_millis(100),
6487 max_backoff: Duration::from_secs(5),
6488 }
6489 }
6490}
6491
6492#[derive(Clone, Debug)]
6493struct WorkerStorageAdmission {
6494 policy: WorkerRetryPolicy,
6495 stop: Arc<AtomicBool>,
6496}
6497
6498struct StopWorkerOnDrop(Arc<AtomicBool>);
6499
6500impl Drop for StopWorkerOnDrop {
6501 fn drop(&mut self) {
6502 self.0.store(true, Ordering::SeqCst);
6503 }
6504}
6505
6506async fn wait_for_worker_stop(stop: &AtomicBool) {
6507 while !stop.load(Ordering::SeqCst) {
6508 tokio::time::sleep(Duration::from_millis(100)).await;
6509 }
6510}
6511
6512#[derive(Clone, Copy, Debug, PartialEq, Eq)]
6513enum ManagedPollOutcome {
6514 Idle,
6515 Handled,
6516 Stop,
6517}
6518
6519#[derive(Clone)]
6520pub struct Worker {
6521 client: Client,
6522 worker_id: String,
6523 task_queue: String,
6524 workflows: HashMap<String, RegisteredWorkflow>,
6525 activities: HashMap<String, ActivityHandler>,
6526 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
6527 signals: HashMap<String, Vec<String>>,
6528 updates: HashMap<String, HashMap<String, UpdateHandler>>,
6529 registration_definitions:
6530 HashMap<(RegistrationKind, Option<String>, String), HandlerRegistration>,
6531 registration_error: Option<DuplicateRegistrationError>,
6532 max_concurrent_workflow_tasks: usize,
6533 max_concurrent_activity_tasks: usize,
6534 poll_timeout: Duration,
6535 heartbeat_interval: Duration,
6536 retry_policy: WorkerRetryPolicy,
6537 recover_transient_outages: bool,
6538 run_stop: Option<Arc<AtomicBool>>,
6539 heartbeat_observer: Option<WorkerHeartbeatObserver>,
6540 cooperative_cancellation_enabled: bool,
6541 allow_cancellation_scope_authoring: bool,
6542 allow_cancellation_scope_delivery: bool,
6543 cooperative_registration_confirmed: Arc<AtomicBool>,
6544 local_registration_confirmed: Arc<AtomicBool>,
6545 session_registration_confirmed: Arc<AtomicBool>,
6546 sticky_registration_confirmed: Arc<AtomicBool>,
6547 resource_capabilities: Vec<String>,
6548 sessions: Arc<Mutex<HashMap<String, WorkerSession>>>,
6549}
6550
6551impl Worker {
6552 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
6553 Self {
6554 client,
6555 worker_id: default_worker_id(),
6556 task_queue: task_queue.into(),
6557 workflows: HashMap::new(),
6558 activities: HashMap::new(),
6559 queries: HashMap::new(),
6560 signals: HashMap::new(),
6561 updates: HashMap::new(),
6562 registration_definitions: HashMap::new(),
6563 registration_error: None,
6564 max_concurrent_workflow_tasks: 10,
6565 max_concurrent_activity_tasks: 10,
6566 poll_timeout: Duration::from_secs(30),
6567 heartbeat_interval: Duration::from_secs(60),
6568 retry_policy: WorkerRetryPolicy::default(),
6569 recover_transient_outages: false,
6570 run_stop: None,
6571 heartbeat_observer: None,
6572 cooperative_cancellation_enabled: false,
6573 allow_cancellation_scope_authoring: false,
6574 allow_cancellation_scope_delivery: false,
6575 cooperative_registration_confirmed: Arc::new(AtomicBool::new(false)),
6576 local_registration_confirmed: Arc::new(AtomicBool::new(false)),
6577 session_registration_confirmed: Arc::new(AtomicBool::new(false)),
6578 sticky_registration_confirmed: Arc::new(AtomicBool::new(false)),
6579 resource_capabilities: Vec::new(),
6580 sessions: Arc::new(Mutex::new(HashMap::new())),
6581 }
6582 }
6583
6584 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
6585 self.worker_id = worker_id.into();
6586 self.cooperative_registration_confirmed = Arc::new(AtomicBool::new(false));
6587 self.local_registration_confirmed = Arc::new(AtomicBool::new(false));
6588 self.session_registration_confirmed = Arc::new(AtomicBool::new(false));
6589 self.sticky_registration_confirmed = Arc::new(AtomicBool::new(false));
6590 self.client.reset_sticky_cache();
6591 self.sessions = Arc::new(Mutex::new(HashMap::new()));
6592 self
6593 }
6594
6595 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
6596 self.poll_timeout = timeout;
6597 self
6598 }
6599
6600 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
6601 self.heartbeat_interval = interval;
6602 self
6603 }
6604
6605 pub fn local_activities(mut self, enabled: bool) -> Self {
6611 self.client.local_activities_enabled = enabled;
6612 self.local_registration_confirmed = Arc::new(AtomicBool::new(false));
6613 self
6614 }
6615
6616 pub fn cooperative_cancellation(mut self, enabled: bool) -> Self {
6626 self.cooperative_cancellation_enabled = enabled;
6627 self.client.cooperative_worker_protocol = enabled;
6628 self.cooperative_registration_confirmed = Arc::new(AtomicBool::new(false));
6629 self
6630 }
6631
6632 #[doc(hidden)]
6634 pub fn candidate_cancellation_scope_authoring(mut self, enabled: bool) -> Self {
6635 self.allow_cancellation_scope_authoring = enabled;
6636 self
6637 }
6638
6639 #[doc(hidden)]
6641 pub fn candidate_cancellation_scope_delivery(mut self, enabled: bool) -> Self {
6642 self.allow_cancellation_scope_delivery = enabled;
6643 self
6644 }
6645
6646 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
6648 self.retry_policy = policy;
6649 self
6650 }
6651
6652 pub fn recover_transient_outages(mut self, enabled: bool) -> Self {
6664 self.recover_transient_outages = enabled;
6665 self
6666 }
6667
6668 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
6669 where
6670 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
6671 {
6672 self.heartbeat_observer = Some(Arc::new(observer));
6673 self
6674 }
6675
6676 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
6677 self.max_concurrent_workflow_tasks = count.max(1);
6678 self
6679 }
6680
6681 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
6682 self.max_concurrent_activity_tasks = count.max(1);
6683 self
6684 }
6685
6686 pub fn validate_registration(&self) -> std::result::Result<(), DuplicateRegistrationError> {
6693 match &self.registration_error {
6694 Some(error) => Err(error.clone()),
6695 None => Ok(()),
6696 }
6697 }
6698
6699 #[track_caller]
6700 fn admit_registration<F>(
6701 &mut self,
6702 handler_kind: RegistrationKind,
6703 workflow_type: Option<&str>,
6704 handler_name: &str,
6705 method: &'static str,
6706 ) -> bool {
6707 let location = std::panic::Location::caller();
6708 let definition = HandlerRegistration {
6709 method,
6710 handler_type: type_name::<F>(),
6711 file: location.file(),
6712 line: location.line(),
6713 column: location.column(),
6714 };
6715 let key = (
6716 handler_kind,
6717 workflow_type.map(str::to_owned),
6718 handler_name.to_owned(),
6719 );
6720 if let Some(first_definition) = self.registration_definitions.get(&key) {
6721 if self.registration_error.is_none() {
6722 self.registration_error = Some(DuplicateRegistrationError {
6723 handler_kind,
6724 workflow_type: key.1.clone(),
6725 handler_name: key.2.clone(),
6726 first_definition: first_definition.clone(),
6727 second_definition: definition,
6728 });
6729 }
6730 return false;
6731 }
6732 self.registration_definitions.insert(key, definition);
6733 true
6734 }
6735
6736 #[track_caller]
6743 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
6744 where
6745 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
6746 Fut: Future<Output = Result<Value>> + Send + 'static,
6747 {
6748 let workflow_type = workflow_type.into();
6749 if !self.admit_registration::<F>(
6750 RegistrationKind::Workflow,
6751 None,
6752 &workflow_type,
6753 "register_workflow",
6754 ) {
6755 return;
6756 }
6757 let handler = Arc::new(handler);
6758 self.workflows.insert(
6759 workflow_type,
6760 RegisteredWorkflow {
6761 execute: Arc::new(move |ctx, input| {
6762 let handler = Arc::clone(&handler);
6763 Box::pin(async move {
6764 let result = handler(ctx, input.into_json()?).await?;
6765 AvroValue::from_serialize(&result)
6766 })
6767 }),
6768 replay: None,
6769 state_type: None,
6770 definition_fingerprint: None,
6771 },
6772 );
6773 }
6774
6775 #[track_caller]
6786 pub fn register_typed_workflow<I, O, F, Fut>(
6787 &mut self,
6788 workflow_type: impl Into<String>,
6789 handler: F,
6790 ) where
6791 I: DeserializeOwned + Send + 'static,
6792 O: Serialize + Send + 'static,
6793 F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
6794 Fut: Future<Output = Result<O>> + Send + 'static,
6795 {
6796 let workflow_type = workflow_type.into();
6797 if !self.admit_registration::<F>(
6798 RegistrationKind::Workflow,
6799 None,
6800 &workflow_type,
6801 "register_typed_workflow",
6802 ) {
6803 return;
6804 }
6805 let handler_name = workflow_type.clone();
6806 let handler = Arc::new(handler);
6807 self.workflows.insert(
6808 workflow_type,
6809 RegisteredWorkflow {
6810 execute: Arc::new(move |ctx, input| {
6811 let handler = Arc::clone(&handler);
6812 let handler_name = handler_name.clone();
6813 Box::pin(async move {
6814 let input =
6815 decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6816 let result = handler(ctx, input).await?;
6817 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6818 })
6819 }),
6820 replay: None,
6821 state_type: None,
6822 definition_fingerprint: None,
6823 },
6824 );
6825 }
6826
6827 #[track_caller]
6829 pub fn register_workflow_avro_value<F, Fut>(
6830 &mut self,
6831 workflow_type: impl Into<String>,
6832 handler: F,
6833 ) where
6834 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6835 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6836 {
6837 let workflow_type = workflow_type.into();
6838 if !self.admit_registration::<F>(
6839 RegistrationKind::Workflow,
6840 None,
6841 &workflow_type,
6842 "register_workflow_avro_value",
6843 ) {
6844 return;
6845 }
6846 self.workflows.insert(
6847 workflow_type,
6848 RegisteredWorkflow {
6849 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6850 replay: None,
6851 state_type: None,
6852 definition_fingerprint: None,
6853 },
6854 );
6855 }
6856
6857 #[track_caller]
6865 pub fn register_replayed_workflow<S, Factory, F, Fut>(
6866 &mut self,
6867 workflow_type: impl Into<String>,
6868 state_factory: Factory,
6869 handler: F,
6870 ) where
6871 S: Clone + Send + Sync + 'static,
6872 Factory: Fn() -> S + Send + Sync + 'static,
6873 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6874 Fut: Future<Output = Result<Value>> + Send + 'static,
6875 {
6876 let workflow_type = workflow_type.into();
6877 if !self.admit_registration::<F>(
6878 RegistrationKind::Workflow,
6879 None,
6880 &workflow_type,
6881 "register_replayed_workflow",
6882 ) {
6883 return;
6884 }
6885 let state_factory = Arc::new(state_factory);
6886 let handler = Arc::new(handler);
6887
6888 let execute_factory = Arc::clone(&state_factory);
6889 let execute_handler = Arc::clone(&handler);
6890 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6891 let state = WorkflowInstance::new(execute_factory());
6892 let handler = Arc::clone(&execute_handler);
6893 Box::pin(async move {
6894 let result = handler(ctx, input.into_json()?, state).await?;
6895 AvroValue::from_serialize(&result)
6896 }) as WorkflowFuture
6897 });
6898
6899 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6900 let state = WorkflowInstance::new(state_factory());
6901 let snapshot_state = state.clone();
6902 let snapshot: WorkflowStateSnapshot =
6903 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6904 let replay_handler = Arc::clone(&handler);
6905 let future = async move {
6906 let result = replay_handler(ctx, input.into_json()?, state).await?;
6907 AvroValue::from_serialize(&result)
6908 };
6909 ReplayedWorkflowInvocation {
6910 future: Box::pin(future),
6911 snapshot,
6912 }
6913 });
6914
6915 self.workflows.insert(
6916 workflow_type,
6917 RegisteredWorkflow {
6918 execute,
6919 replay: Some(replay),
6920 state_type: Some(TypeId::of::<S>()),
6921 definition_fingerprint: None,
6922 },
6923 );
6924 }
6925
6926 #[track_caller]
6932 pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6933 &mut self,
6934 workflow_type: impl Into<String>,
6935 state_factory: Factory,
6936 handler: F,
6937 ) where
6938 I: DeserializeOwned + Send + 'static,
6939 O: Serialize + Send + 'static,
6940 S: Clone + Send + Sync + 'static,
6941 Factory: Fn() -> S + Send + Sync + 'static,
6942 F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6943 Fut: Future<Output = Result<O>> + Send + 'static,
6944 {
6945 let workflow_type = workflow_type.into();
6946 if !self.admit_registration::<F>(
6947 RegistrationKind::Workflow,
6948 None,
6949 &workflow_type,
6950 "register_typed_replayed_workflow",
6951 ) {
6952 return;
6953 }
6954 let state_factory = Arc::new(state_factory);
6955 let handler = Arc::new(handler);
6956
6957 let execute_name = workflow_type.clone();
6958 let execute_factory = Arc::clone(&state_factory);
6959 let execute_handler = Arc::clone(&handler);
6960 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6961 let state = WorkflowInstance::new(execute_factory());
6962 let handler = Arc::clone(&execute_handler);
6963 let handler_name = execute_name.clone();
6964 Box::pin(async move {
6965 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6966 let result = handler(ctx, input, state).await?;
6967 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6968 }) as WorkflowFuture
6969 });
6970
6971 let replay_name = workflow_type.clone();
6972 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6973 let state = WorkflowInstance::new(state_factory());
6974 let snapshot_state = state.clone();
6975 let snapshot: WorkflowStateSnapshot =
6976 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6977 let handler = Arc::clone(&handler);
6978 let handler_name = replay_name.clone();
6979 let future = async move {
6980 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6981 let result = handler(ctx, input, state).await?;
6982 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6983 };
6984 ReplayedWorkflowInvocation {
6985 future: Box::pin(future),
6986 snapshot,
6987 }
6988 });
6989
6990 self.workflows.insert(
6991 workflow_type,
6992 RegisteredWorkflow {
6993 execute,
6994 replay: Some(replay),
6995 state_type: Some(TypeId::of::<S>()),
6996 definition_fingerprint: None,
6997 },
6998 );
6999 }
7000
7001 pub fn declare_workflow_signals(
7010 &mut self,
7011 workflow_type: &str,
7012 signal_names: &[&str],
7013 ) -> Result<()> {
7014 if !self.workflows.contains_key(workflow_type) {
7015 return Err(Error::Codec(format!(
7016 "workflow type {workflow_type:?} is not registered"
7017 )));
7018 }
7019 if signal_names
7020 .iter()
7021 .any(|name| name.is_empty() || *name != name.trim())
7022 {
7023 return Err(Error::Codec(
7024 "signal names must be non-empty and have no surrounding whitespace".into(),
7025 ));
7026 }
7027 let mut names = signal_names
7028 .iter()
7029 .map(|name| (*name).to_string())
7030 .collect::<Vec<_>>();
7031 names.sort();
7032 names.dedup();
7033 self.signals.insert(workflow_type.to_string(), names);
7034 Ok(())
7035 }
7036
7037 pub fn set_workflow_definition_sources(
7047 &mut self,
7048 workflow_type: &str,
7049 sources: &[&str],
7050 ) -> Result<()> {
7051 if sources.is_empty() || sources.iter().any(|source| source.is_empty()) {
7052 return Err(Error::Codec(
7053 "workflow definition sources must be non-empty".to_string(),
7054 ));
7055 }
7056 let workflow = self.workflows.get_mut(workflow_type).ok_or_else(|| {
7057 Error::Codec(format!("workflow type {workflow_type:?} is not registered"))
7058 })?;
7059 let mut hasher = Sha256::new();
7060 hasher.update(b"durable-workflow-rust.workflow-definition.v1\0");
7061 hasher.update((workflow_type.len() as u64).to_be_bytes());
7062 hasher.update(workflow_type.as_bytes());
7063 for source in sources {
7064 hasher.update((source.len() as u64).to_be_bytes());
7065 hasher.update(source.as_bytes());
7066 }
7067 workflow.definition_fingerprint = Some(format!("sha256:{:x}", hasher.finalize()));
7068 Ok(())
7069 }
7070
7071 #[track_caller]
7073 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
7074 &mut self,
7075 workflow_type: impl Into<String>,
7076 state_factory: Factory,
7077 handler: F,
7078 ) where
7079 S: Clone + Send + Sync + 'static,
7080 Factory: Fn() -> S + Send + Sync + 'static,
7081 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
7082 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
7083 {
7084 let workflow_type = workflow_type.into();
7085 if !self.admit_registration::<F>(
7086 RegistrationKind::Workflow,
7087 None,
7088 &workflow_type,
7089 "register_replayed_workflow_avro_value",
7090 ) {
7091 return;
7092 }
7093 let state_factory = Arc::new(state_factory);
7094 let handler = Arc::new(handler);
7095
7096 let execute_factory = Arc::clone(&state_factory);
7097 let execute_handler = Arc::clone(&handler);
7098 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
7099 let state = WorkflowInstance::new(execute_factory());
7100 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
7101 });
7102
7103 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
7104 let state = WorkflowInstance::new(state_factory());
7105 let snapshot_state = state.clone();
7106 let snapshot: WorkflowStateSnapshot =
7107 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
7108 ReplayedWorkflowInvocation {
7109 future: Box::pin(handler(ctx, input, state)),
7110 snapshot,
7111 }
7112 });
7113
7114 self.workflows.insert(
7115 workflow_type,
7116 RegisteredWorkflow {
7117 execute,
7118 replay: Some(replay),
7119 state_type: Some(TypeId::of::<S>()),
7120 definition_fingerprint: None,
7121 },
7122 );
7123 }
7124
7125 #[track_caller]
7126 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
7127 where
7128 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
7129 Fut: Future<Output = Result<Value>> + Send + 'static,
7130 {
7131 let activity_type = activity_type.into();
7132 if !self.admit_registration::<F>(
7133 RegistrationKind::Activity,
7134 None,
7135 &activity_type,
7136 "register_activity",
7137 ) {
7138 return;
7139 }
7140 let handler = Arc::new(handler);
7141 self.activities.insert(
7142 activity_type,
7143 Arc::new(move |ctx, args| {
7144 let handler = Arc::clone(&handler);
7145 Box::pin(async move {
7146 let result = handler(ctx, args.into_json()?).await?;
7147 AvroValue::from_serialize(&result)
7148 })
7149 }),
7150 );
7151 }
7152
7153 #[track_caller]
7159 pub fn register_typed_activity<I, O, F, Fut>(
7160 &mut self,
7161 activity_type: impl Into<String>,
7162 handler: F,
7163 ) where
7164 I: DeserializeOwned + Send + 'static,
7165 O: Serialize + Send + 'static,
7166 F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
7167 Fut: Future<Output = Result<O>> + Send + 'static,
7168 {
7169 let activity_type = activity_type.into();
7170 if !self.admit_registration::<F>(
7171 RegistrationKind::Activity,
7172 None,
7173 &activity_type,
7174 "register_typed_activity",
7175 ) {
7176 return;
7177 }
7178 let handler_name = activity_type.clone();
7179 let handler = Arc::new(handler);
7180 self.activities.insert(
7181 activity_type,
7182 Arc::new(move |ctx, input| {
7183 let handler = Arc::clone(&handler);
7184 let handler_name = handler_name.clone();
7185 Box::pin(async move {
7186 let input =
7187 decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
7188 let result = handler(ctx, input).await?;
7189 encode_handler_result(&result, HandlerKind::Activity, &handler_name)
7190 })
7191 }),
7192 );
7193 }
7194
7195 #[track_caller]
7197 pub fn register_activity_avro_value<F, Fut>(
7198 &mut self,
7199 activity_type: impl Into<String>,
7200 handler: F,
7201 ) where
7202 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
7203 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
7204 {
7205 let activity_type = activity_type.into();
7206 if !self.admit_registration::<F>(
7207 RegistrationKind::Activity,
7208 None,
7209 &activity_type,
7210 "register_activity_avro_value",
7211 ) {
7212 return;
7213 }
7214 self.activities.insert(
7215 activity_type,
7216 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
7217 );
7218 }
7219
7220 #[track_caller]
7226 pub fn register_query<F, Fut>(
7227 &mut self,
7228 workflow_type: impl Into<String>,
7229 query_name: impl Into<String>,
7230 handler: F,
7231 ) where
7232 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
7233 Fut: Future<Output = Result<Value>> + Send + 'static,
7234 {
7235 let workflow_type = workflow_type.into();
7236 let query_name = query_name.into();
7237 if !self.admit_registration::<F>(
7238 RegistrationKind::Query,
7239 Some(&workflow_type),
7240 &query_name,
7241 "register_query",
7242 ) {
7243 return;
7244 }
7245 let handler = Arc::new(handler);
7246 self.queries.entry(workflow_type).or_default().insert(
7247 query_name,
7248 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
7249 let handler = Arc::clone(&handler);
7250 Box::pin(async move {
7251 let result = handler(ctx, args.into_json()?).await?;
7252 AvroValue::from_serialize(&result)
7253 })
7254 })),
7255 );
7256 }
7257
7258 #[track_caller]
7260 pub fn register_query_avro_value<F, Fut>(
7261 &mut self,
7262 workflow_type: impl Into<String>,
7263 query_name: impl Into<String>,
7264 handler: F,
7265 ) where
7266 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
7267 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
7268 {
7269 let workflow_type = workflow_type.into();
7270 let query_name = query_name.into();
7271 if !self.admit_registration::<F>(
7272 RegistrationKind::Query,
7273 Some(&workflow_type),
7274 &query_name,
7275 "register_query_avro_value",
7276 ) {
7277 return;
7278 }
7279 self.queries.entry(workflow_type).or_default().insert(
7280 query_name,
7281 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
7282 );
7283 }
7284
7285 #[track_caller]
7292 pub fn register_replayed_query<S, F, Fut>(
7293 &mut self,
7294 workflow_type: impl Into<String>,
7295 query_name: impl Into<String>,
7296 handler: F,
7297 ) where
7298 S: Clone + Send + Sync + 'static,
7299 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
7300 Fut: Future<Output = Result<Value>> + Send + 'static,
7301 {
7302 let workflow_type = workflow_type.into();
7303 let query_name = query_name.into();
7304 if !self.admit_registration::<F>(
7305 RegistrationKind::Query,
7306 Some(&workflow_type),
7307 &query_name,
7308 "register_replayed_query",
7309 ) {
7310 return;
7311 }
7312 let handler = Arc::new(handler);
7313 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
7314 let state = state.downcast::<S>().map_err(|_| {
7315 "registered query state type does not match the replayed workflow state".to_string()
7316 })?;
7317 let handler = Arc::clone(&handler);
7318 Ok(Box::pin(async move {
7319 let result = handler(ctx, state, args.into_json()?).await?;
7320 AvroValue::from_serialize(&result)
7321 }))
7322 });
7323
7324 self.queries.entry(workflow_type).or_default().insert(
7325 query_name,
7326 RegisteredQuery::Replayed {
7327 state_type: TypeId::of::<S>(),
7328 handler: erased_handler,
7329 },
7330 );
7331 }
7332
7333 #[track_caller]
7335 pub fn register_replayed_query_avro_value<S, F, Fut>(
7336 &mut self,
7337 workflow_type: impl Into<String>,
7338 query_name: impl Into<String>,
7339 handler: F,
7340 ) where
7341 S: Clone + Send + Sync + 'static,
7342 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
7343 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
7344 {
7345 let workflow_type = workflow_type.into();
7346 let query_name = query_name.into();
7347 if !self.admit_registration::<F>(
7348 RegistrationKind::Query,
7349 Some(&workflow_type),
7350 &query_name,
7351 "register_replayed_query_avro_value",
7352 ) {
7353 return;
7354 }
7355 let handler = Arc::new(handler);
7356 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
7357 let state = state.downcast::<S>().map_err(|_| {
7358 "registered query state type does not match the replayed workflow state".to_string()
7359 })?;
7360 Ok(Box::pin(handler(ctx, state, args)))
7361 });
7362
7363 self.queries.entry(workflow_type).or_default().insert(
7364 query_name,
7365 RegisteredQuery::Replayed {
7366 state_type: TypeId::of::<S>(),
7367 handler: erased_handler,
7368 },
7369 );
7370 }
7371
7372 #[track_caller]
7374 pub fn register_update<F, Fut>(
7375 &mut self,
7376 workflow_type: impl Into<String>,
7377 update_name: impl Into<String>,
7378 handler: F,
7379 ) where
7380 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
7381 Fut: Future<Output = Result<Value>> + Send + 'static,
7382 {
7383 let workflow_type = workflow_type.into();
7384 let update_name = update_name.into();
7385 if !self.admit_registration::<F>(
7386 RegistrationKind::Update,
7387 Some(&workflow_type),
7388 &update_name,
7389 "register_update",
7390 ) {
7391 return;
7392 }
7393 let handler = Arc::new(handler);
7394 self.updates.entry(workflow_type).or_default().insert(
7395 update_name,
7396 Arc::new(move |ctx, args| {
7397 let handler = Arc::clone(&handler);
7398 Box::pin(async move {
7399 let result = handler(ctx, args.into_json()?).await?;
7400 AvroValue::from_serialize(&result)
7401 })
7402 }),
7403 );
7404 }
7405
7406 #[track_caller]
7408 pub fn register_update_avro_value<F, Fut>(
7409 &mut self,
7410 workflow_type: impl Into<String>,
7411 update_name: impl Into<String>,
7412 handler: F,
7413 ) where
7414 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
7415 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
7416 {
7417 let workflow_type = workflow_type.into();
7418 let update_name = update_name.into();
7419 if !self.admit_registration::<F>(
7420 RegistrationKind::Update,
7421 Some(&workflow_type),
7422 &update_name,
7423 "register_update_avro_value",
7424 ) {
7425 return;
7426 }
7427 self.updates.entry(workflow_type).or_default().insert(
7428 update_name,
7429 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
7430 );
7431 }
7432
7433 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
7434 self.validate_registration()?;
7435 self.sticky_registration_confirmed
7436 .store(false, Ordering::SeqCst);
7437 self.session_registration_confirmed
7438 .store(false, Ordering::SeqCst);
7439 for capability in &self.resource_capabilities {
7440 worker_session::validate_resource_capability(capability)?;
7441 }
7442 self.local_registration_confirmed
7443 .store(false, Ordering::SeqCst);
7444 if self.client.local_activities_enabled && self.cooperative_cancellation_enabled {
7445 return Err(Error::CooperativeCancellationUnavailable(
7446 "inline local activities cannot advertise prepared local supervision; use a separate ordinary worker for inline local callbacks".into(),
7447 ));
7448 }
7449 let mut command_contracts = serde_json::Map::new();
7450 for workflow_type in self.workflows.keys() {
7451 let mut queries = self
7452 .queries
7453 .get(workflow_type)
7454 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
7455 .unwrap_or_default();
7456 queries.sort();
7457 let mut updates = self
7458 .updates
7459 .get(workflow_type)
7460 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
7461 .unwrap_or_default();
7462 updates.sort();
7463 let signals = self.signals.get(workflow_type).cloned().unwrap_or_default();
7464 command_contracts.insert(
7465 workflow_type.clone(),
7466 json!({
7467 "queries": queries,
7468 "query_contracts": argument_vector_contracts(&queries),
7469 "signals": signals,
7470 "signal_contracts": argument_vector_contracts(&signals),
7471 "updates": updates,
7472 "update_contracts": argument_vector_contracts(&updates),
7473 "update_validators": [],
7474 }),
7475 );
7476 }
7477
7478 self.cooperative_registration_confirmed
7479 .store(false, Ordering::SeqCst);
7480 let response = self
7481 .client
7482 .register_worker_with_definition_fingerprints(
7483 &self.worker_id,
7484 &self.task_queue,
7485 self.workflows.keys().cloned().collect(),
7486 self.activities.keys().cloned().collect(),
7487 self.max_concurrent_workflow_tasks,
7488 self.max_concurrent_activity_tasks,
7489 [
7490 Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
7491 Some(DURABLE_SELECTION_CAPABILITY.to_string()),
7492 Some(MEMO_UPSERTS_CAPABILITY.to_string()),
7493 Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
7494 self.client
7495 .local_activities_enabled
7496 .then(|| "local_activities".to_string()),
7497 self.client
7498 .worker_sessions_enabled
7499 .then(|| "worker_sessions".to_string()),
7500 self.client
7501 .sticky_cache
7502 .is_some()
7503 .then(|| "sticky_execution".to_string()),
7504 self.cooperative_cancellation_enabled
7505 .then(|| "cooperative_cancellation".to_string()),
7506 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
7507 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
7508 worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
7509 .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
7510 ]
7511 .into_iter()
7512 .flatten()
7513 .chain(self.resource_capabilities.iter().cloned())
7514 .collect(),
7515 Value::Object(command_contracts),
7516 Some(
7517 self.workflows
7518 .iter()
7519 .filter_map(|(workflow_type, workflow)| {
7520 workflow
7521 .definition_fingerprint
7522 .as_ref()
7523 .map(|fingerprint| (workflow_type.clone(), fingerprint.clone()))
7524 })
7525 .collect(),
7526 ),
7527 )
7528 .await?;
7529 if self.cooperative_cancellation_enabled && response.registered {
7530 self.cooperative_registration_confirmed
7531 .store(true, Ordering::SeqCst);
7532 }
7533 if self.client.local_activities_enabled && response.registered {
7534 self.local_registration_confirmed
7535 .store(true, Ordering::SeqCst);
7536 }
7537 if self.client.worker_sessions_enabled && response.registered {
7538 self.session_registration_confirmed
7539 .store(true, Ordering::SeqCst);
7540 }
7541 if self.client.sticky_cache.is_some() && response.registered {
7542 self.sticky_registration_confirmed
7543 .store(true, Ordering::SeqCst);
7544 }
7545 Ok(response)
7546 }
7547
7548 pub async fn run(&self) -> Result<()> {
7556 self.run_until(std::future::pending::<()>()).await
7557 }
7558
7559 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
7563 where
7564 F: Future<Output = ()>,
7565 {
7566 let _clear_cache_on_drop =
7567 sticky_worker::ClearCacheOnDrop(self.client.sticky_cache.clone());
7568 let stop = Arc::new(AtomicBool::new(false));
7569 let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
7570 let mut worker = self.with_storage_admission(Arc::clone(&stop));
7571 worker.run_stop = Some(Arc::clone(&stop));
7572 let run = worker.run_with_storage_admission(Arc::clone(&stop));
7573 tokio::pin!(run);
7574 tokio::pin!(shutdown);
7575 tokio::select! {
7576 result = &mut run => result,
7577 _ = &mut shutdown => {
7578 stop.store(true, Ordering::SeqCst);
7579 run.await
7580 }
7581 }
7582 }
7583
7584 fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
7585 let mut worker = self.clone();
7586 worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
7587 policy: self.retry_policy,
7588 stop,
7589 });
7590 worker
7591 }
7592
7593 async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
7594 let registration = self.register().await?;
7595 if !registration.registered {
7596 return Err(Error::WorkerLoop(format!(
7597 "worker registration for {:?} was not accepted",
7598 self.worker_id
7599 )));
7600 }
7601 let registered_worker_id = registration.worker_id.clone();
7602 let primary = self.run_registered_until(stop, registration).await;
7603 let primary = match (primary, self.close_worker_sessions().await) {
7604 (Ok(()), result) => result,
7605 (Err(error), Ok(())) => Err(error),
7606 (Err(primary), Err(close)) => Err(Error::WorkerShutdown {
7607 primary: Box::new(primary),
7608 deregistration: Box::new(close),
7609 }),
7610 };
7611 self.session_registration_confirmed
7612 .store(false, Ordering::SeqCst);
7613 self.cooperative_registration_confirmed
7614 .store(false, Ordering::SeqCst);
7615 self.local_registration_confirmed
7616 .store(false, Ordering::SeqCst);
7617 let deregistration = self
7618 .client
7619 .deregister_worker_registration(®istered_worker_id)
7620 .await;
7621 self.sticky_registration_confirmed
7622 .store(false, Ordering::SeqCst);
7623 self.client.clear_sticky_cache()?;
7624
7625 match (primary, deregistration) {
7626 (Ok(()), Ok(_)) => Ok(()),
7627 (Ok(()), Err(deregistration)) => Err(deregistration),
7628 (Err(primary), Ok(_)) => Err(primary),
7629 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
7630 primary: Box::new(primary),
7631 deregistration: Box::new(deregistration),
7632 }),
7633 }
7634 }
7635
7636 async fn run_registered_until(
7637 &self,
7638 stop: Arc<AtomicBool>,
7639 registration: RegisterWorkerResponse,
7640 ) -> Result<()> {
7641 let heartbeat_interval = Duration::from_secs(
7642 registration
7643 .heartbeat_interval_seconds
7644 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
7645 );
7646 let heartbeat = tokio::time::sleep(Duration::ZERO);
7652 tokio::pin!(heartbeat);
7653 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
7657 let worker = self.clone();
7658 let stop = Arc::clone(&stop);
7659 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
7660 });
7661 let mut activity_poller = (!self.activities.is_empty()).then(|| {
7662 let worker = self.clone();
7663 let stop = Arc::clone(&stop);
7664 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
7665 });
7666 let mut query_poller = (!self.queries.is_empty()).then(|| {
7667 let worker = self.clone();
7668 let stop = Arc::clone(&stop);
7669 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
7670 });
7671
7672 loop {
7673 tokio::select! {
7674 _ = wait_for_worker_stop(&stop) => {
7675 stop.store(true, Ordering::SeqCst);
7676 break;
7677 }
7678 _ = &mut heartbeat => {
7679 let result = self.retry_worker_operation(|| {
7680 self.client.heartbeat_worker_with_sessions(
7681 &self.worker_id,
7682 self.max_concurrent_workflow_tasks,
7683 self.max_concurrent_activity_tasks,
7684 self.client.worker_sessions_enabled.then(|| self.session_available()),
7685 )
7686 }).await;
7687 heartbeat
7688 .as_mut()
7689 .reset(tokio::time::Instant::now() + heartbeat_interval);
7690 match result {
7691 Ok(Some(acknowledgement)) => {
7692 if let Some(observer) = &self.heartbeat_observer {
7693 observer(&WorkerHeartbeatObservation {
7694 worker_id: self.worker_id.clone(),
7695 task_queue: self.task_queue.clone(),
7696 acknowledged_at_unix_millis: SystemTime::now()
7697 .duration_since(UNIX_EPOCH)
7698 .unwrap_or_default()
7699 .as_millis()
7700 .min(u64::MAX as u128)
7701 as u64,
7702 acknowledgement,
7703 });
7704 }
7705 }
7706 Ok(None) => break,
7707 Err(error) => {
7708 stop.store(true, Ordering::SeqCst);
7709 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
7710 return Err(error);
7711 }
7712 }
7713 }
7714 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
7715 workflow_poller = None;
7716 let stopped_by_server = stop.load(Ordering::SeqCst);
7717 stop.store(true, Ordering::SeqCst);
7718 let poller_result = optional_poller_result("workflow", result);
7719 let join_result =
7720 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
7721 poller_result?;
7722 join_result?;
7723 if stopped_by_server {
7724 return Ok(());
7725 }
7726 return Err(Error::WorkerLoop(
7727 "workflow poller stopped unexpectedly".to_string(),
7728 ));
7729 }
7730 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
7731 activity_poller = None;
7732 let stopped_by_server = stop.load(Ordering::SeqCst);
7733 stop.store(true, Ordering::SeqCst);
7734 let poller_result = optional_poller_result("activity", result);
7735 let join_result =
7736 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
7737 poller_result?;
7738 join_result?;
7739 if stopped_by_server {
7740 return Ok(());
7741 }
7742 return Err(Error::WorkerLoop(
7743 "activity poller stopped unexpectedly".to_string(),
7744 ));
7745 }
7746 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
7747 query_poller = None;
7748 let stopped_by_server = stop.load(Ordering::SeqCst);
7749 stop.store(true, Ordering::SeqCst);
7750 let poller_result = optional_poller_result("query", result);
7751 let join_result =
7752 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
7753 poller_result?;
7754 join_result?;
7755 if stopped_by_server {
7756 return Ok(());
7757 }
7758 return Err(Error::WorkerLoop(
7759 "query poller stopped unexpectedly".to_string(),
7760 ));
7761 }
7762 }
7763 }
7764
7765 join_pollers(
7766 workflow_poller.take(),
7767 activity_poller.take(),
7768 query_poller.take(),
7769 )
7770 .await
7771 }
7772
7773 pub async fn run_once(&self) -> Result<usize> {
7787 self.validate_registration()?;
7788 if self.client.worker_sessions_enabled {
7789 self.require_session_registration()?;
7790 }
7791 if self.cooperative_cancellation_enabled
7792 && !self
7793 .cooperative_registration_confirmed
7794 .load(Ordering::SeqCst)
7795 {
7796 return Err(Error::CooperativeCancellationUnavailable(
7797 "register this cooperative worker before polling tasks".into(),
7798 ));
7799 }
7800 let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
7801 let mut handled = 0;
7802 match worker.poll_workflow_once().await? {
7803 ManagedPollOutcome::Handled => handled += 1,
7804 ManagedPollOutcome::Stop => return Ok(handled),
7805 ManagedPollOutcome::Idle => {}
7806 }
7807 match worker.poll_activity_once().await? {
7808 ManagedPollOutcome::Handled => handled += 1,
7809 ManagedPollOutcome::Stop => return Ok(handled),
7810 ManagedPollOutcome::Idle => {}
7811 }
7812 if !self.queries.is_empty() {
7813 match worker.poll_query_once().await? {
7814 ManagedPollOutcome::Handled => handled += 1,
7815 ManagedPollOutcome::Stop => return Ok(handled),
7816 ManagedPollOutcome::Idle => {}
7817 }
7818 }
7819 Ok(handled)
7820 }
7821
7822 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
7823 if self.client.sticky_cache.is_some()
7824 && !self.sticky_registration_confirmed.load(Ordering::SeqCst)
7825 {
7826 return Err(Error::WorkerLoop("sticky_registration_unconfirmed: register the sticky-capable worker before polling".into()));
7827 }
7828 if self.client.local_activities_enabled
7829 && !self.local_registration_confirmed.load(Ordering::SeqCst)
7830 {
7831 return Err(Error::WorkerLoop("local_activity_registration_unconfirmed: register this local-capable worker before polling workflow tasks".into()));
7832 }
7833 if self.cooperative_cancellation_enabled {
7834 return self.poll_cooperative_workflow_once().await;
7835 }
7836 let poll_request_id = unique_request_id("rust-workflow-poll");
7837 let response = self
7838 .retry_worker_operation(|| self.poll_workflow_with_sticky_cache(&poll_request_id))
7839 .await;
7840 let Some((response, snapshot)) = self.settle_worker_poll_response(response).await? else {
7841 return Ok(ManagedPollOutcome::Idle);
7842 };
7843 if response.outcome().should_stop() {
7844 return Ok(ManagedPollOutcome::Stop);
7845 }
7846 let memo_updates_supported =
7847 runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
7848 let Some(task) = response.task else {
7849 return Ok(ManagedPollOutcome::Idle);
7850 };
7851
7852 let task_id = task.task_id.clone();
7853 let attempt = task.workflow_task_attempt;
7854 let run_id = task.run_id.clone();
7855 let lease_owner = task
7856 .lease_owner
7857 .clone()
7858 .unwrap_or_else(|| self.worker_id.clone());
7859
7860 let decision = match self.execute_workflow_with_local_activities(task).await {
7861 Ok(Some(decision)) => Ok(decision),
7862 Ok(None) => return Ok(ManagedPollOutcome::Handled),
7863 Err(error) => Err(error),
7864 };
7865 self.settle_workflow_task_decision(
7866 &task_id,
7867 &lease_owner,
7868 attempt,
7869 run_id.as_deref(),
7870 decision,
7871 memo_updates_supported,
7872 snapshot,
7873 )
7874 .await
7875 }
7876
7877 #[allow(clippy::too_many_arguments)]
7878 async fn settle_workflow_task_decision(
7879 &self,
7880 task_id: &str,
7881 lease_owner: &str,
7882 attempt: u64,
7883 run_id: Option<&str>,
7884 decision: Result<WorkflowTaskDecision>,
7885 memo_updates_supported: bool,
7886 snapshot: Option<sticky_worker::StickySnapshot>,
7887 ) -> Result<ManagedPollOutcome> {
7888 match decision {
7889 Ok(decision)
7890 if decision.cancellation_delivery.is_some()
7891 || decision.cancellation_scope_delivery.is_some()
7892 || decision.cancellation_scope_opening.is_some() =>
7893 {
7894 return Err(Error::CooperativeCancellationUnavailable(
7895 "worker cancellation delivery has not been negotiated".into(),
7896 ));
7897 }
7898 Ok(decision)
7899 if commands_use_workflow_memo_updates(&decision.commands)
7900 && !memo_updates_supported =>
7901 {
7902 self.client
7903 .fail_workflow_task(
7904 task_id,
7905 lease_owner,
7906 attempt,
7907 Error::WorkflowMemoUpdatesUnavailable.to_string(),
7908 )
7909 .await?;
7910 }
7911 Ok(decision) if decision.commands.is_empty() => {
7912 self.client
7918 .fail_workflow_task_with_type(
7919 task_id,
7920 lease_owner,
7921 attempt,
7922 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
7923 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
7924 )
7925 .await?;
7926 }
7927 Ok(decision) => {
7928 let sticky_claim = self.sticky_claim(snapshot.as_ref(), &decision.commands)?;
7929 let completion = self
7930 .client
7931 .complete_workflow_task_with_message_streams(
7932 task_id,
7933 lease_owner,
7934 attempt,
7935 decision.commands,
7936 decision.message_stream_cursors,
7937 decision.message_stream_waits,
7938 sticky_claim,
7939 )
7940 .await;
7941 if let Err(error) = completion {
7942 self.discard_sticky_snapshot(snapshot.as_ref())?;
7943 if !workflow_task_completion_is_terminal_timeout(
7944 &error, task_id, attempt, run_id,
7945 ) {
7946 return Err(error);
7947 }
7948 }
7949 }
7950 Err(error @ Error::CancellationScopeExecutionUnavailable) => return Err(error),
7951 Err(error) => {
7952 self.discard_sticky_snapshot(snapshot.as_ref())?;
7953 self.client
7954 .fail_workflow_task(task_id, lease_owner, attempt, error.to_string())
7955 .await?;
7956 }
7957 }
7958
7959 Ok(ManagedPollOutcome::Handled)
7960 }
7961
7962 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7963 while !stop.load(Ordering::SeqCst) {
7964 match self.poll_workflow_once().await {
7965 Ok(ManagedPollOutcome::Stop) => {
7966 stop.store(true, Ordering::SeqCst);
7967 break;
7968 }
7969 Err(error) => {
7970 stop.store(true, Ordering::SeqCst);
7971 return Err(error);
7972 }
7973 _ => {}
7974 }
7975 }
7976
7977 Ok(())
7978 }
7979
7980 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
7981 if self.client.worker_sessions_enabled {
7982 self.require_session_registration()?;
7983 }
7984 if self.cooperative_cancellation_enabled {
7985 return self.poll_cooperative_activity_once().await;
7986 }
7987 let poll_request_id = unique_request_id("rust-activity-poll");
7988 let response = self
7989 .retry_worker_operation(|| {
7990 self.client.poll_activity_task_response_with_request_id(
7991 &self.worker_id,
7992 &self.task_queue,
7993 self.poll_timeout,
7994 &poll_request_id,
7995 0,
7996 )
7997 })
7998 .await;
7999 let Some(response) = self.settle_worker_poll_response(response).await? else {
8000 return Ok(ManagedPollOutcome::Idle);
8001 };
8002 if response.outcome().should_stop() {
8003 return Ok(ManagedPollOutcome::Stop);
8004 }
8005 let Some(task) = response.task else {
8006 return Ok(ManagedPollOutcome::Idle);
8007 };
8008
8009 let session = self.track_session_task(task.worker_session.as_ref())?;
8010 let task = task.task;
8011
8012 let task_id = task.task_id.clone();
8013 let attempt_id = task
8014 .activity_attempt_id
8015 .clone()
8016 .or(task.attempt_id.clone())
8017 .unwrap_or_default();
8018 let lease_owner = task
8019 .lease_owner
8020 .clone()
8021 .unwrap_or_else(|| self.worker_id.clone());
8022 let codec = task.payload_codec.clone();
8023 let result = self.execute_session_activity_task(task, session).await;
8024 match result {
8025 Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
8026 Ok(value) => {
8027 let completion = self
8028 .client
8029 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
8030 .await;
8031 if let Err(error) = completion {
8032 if !activity_task_rejection_is_final(&error) {
8033 return Err(error);
8034 }
8035 }
8036 }
8037 Err(error) => {
8038 let failure = self
8039 .client
8040 .fail_activity_task(
8041 &task_id,
8042 &attempt_id,
8043 &lease_owner,
8044 error.to_string(),
8045 false,
8046 )
8047 .await;
8048 if let Err(error) = failure {
8049 if !activity_task_rejection_is_final(&error) {
8050 return Err(error);
8051 }
8052 }
8053 }
8054 }
8055
8056 Ok(ManagedPollOutcome::Handled)
8057 }
8058
8059 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
8060 let lane_count = if self.cooperative_cancellation_enabled {
8064 self.max_concurrent_activity_tasks
8065 } else {
8066 1
8067 };
8068 let mut lanes = FuturesUnordered::new();
8069 for _ in 0..lane_count {
8070 let worker = self.clone();
8071 let stop = Arc::clone(&stop);
8072 lanes.push(async move {
8073 while !stop.load(Ordering::SeqCst) {
8074 match worker.poll_activity_once().await {
8075 Ok(ManagedPollOutcome::Stop) => {
8076 stop.store(true, Ordering::SeqCst);
8077 break;
8078 }
8079 Err(error) => {
8080 stop.store(true, Ordering::SeqCst);
8081 return Err(error);
8082 }
8083 _ => {}
8084 }
8085 }
8086 Ok(())
8087 });
8088 }
8089 let mut first_error = None;
8091 while let Some(result) = lanes.next().await {
8092 if let Err(error) = result {
8093 first_error.get_or_insert(error);
8094 }
8095 }
8096 first_error.map_or(Ok(()), Err)
8097 }
8098
8099 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
8100 let poll_request_id = unique_request_id("rust-query-poll");
8101 let response = self
8102 .retry_worker_operation(|| {
8103 self.client.poll_query_task_response_with_request_id(
8104 &self.worker_id,
8105 &self.task_queue,
8106 self.poll_timeout,
8107 &poll_request_id,
8108 0,
8109 )
8110 })
8111 .await;
8112 let Some(response) = self.settle_worker_poll_response(response).await? else {
8113 return Ok(ManagedPollOutcome::Idle);
8114 };
8115 if response.outcome().should_stop() {
8116 return Ok(ManagedPollOutcome::Stop);
8117 }
8118 let Some(task) = response.task else {
8119 return Ok(ManagedPollOutcome::Idle);
8120 };
8121
8122 let query_task_id = task.query_task_id.clone();
8123 let attempt = task.query_task_attempt;
8124 let lease_owner = task
8125 .lease_owner
8126 .clone()
8127 .unwrap_or_else(|| self.worker_id.clone());
8128 let codec = task.payload_codec.clone();
8129
8130 match self.execute_query_task(task).await {
8131 Ok(value) => {
8132 let result_envelope = match encode_typed_envelope(&value, &codec) {
8133 Ok(result_envelope) => result_envelope,
8134 Err(error) => {
8135 let failure = self
8136 .client
8137 .fail_query_task(
8138 &query_task_id,
8139 &lease_owner,
8140 attempt,
8141 error.to_string(),
8142 "query_result_encode_failed",
8143 "QueryResultEncodeFailed",
8144 )
8145 .await;
8146 if let Err(error) = failure {
8147 if !query_task_rejection_is_final(&error) {
8148 return Err(error);
8149 }
8150 }
8151 return Ok(ManagedPollOutcome::Handled);
8152 }
8153 };
8154
8155 if let Err(error) = self
8156 .client
8157 .complete_query_task_with_envelope(
8158 &query_task_id,
8159 &lease_owner,
8160 attempt,
8161 value.clone().into_json()?,
8162 result_envelope,
8163 )
8164 .await
8165 {
8166 if !query_task_rejection_is_final(&error) {
8167 return Err(error);
8168 }
8169 }
8170 }
8171 Err(failure) => {
8172 let result = self
8173 .client
8174 .fail_query_task(
8175 &query_task_id,
8176 &lease_owner,
8177 attempt,
8178 failure.message,
8179 failure.reason,
8180 failure.failure_type,
8181 )
8182 .await;
8183 if let Err(error) = result {
8184 if !query_task_rejection_is_final(&error) {
8185 return Err(error);
8186 }
8187 }
8188 }
8189 }
8190
8191 Ok(ManagedPollOutcome::Handled)
8192 }
8193
8194 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
8195 while !stop.load(Ordering::SeqCst) {
8196 match self.poll_query_once().await {
8197 Ok(ManagedPollOutcome::Stop) => {
8198 stop.store(true, Ordering::SeqCst);
8199 break;
8200 }
8201 Err(error) => {
8202 stop.store(true, Ordering::SeqCst);
8203 return Err(error);
8204 }
8205 _ => {}
8206 }
8207 }
8208
8209 Ok(())
8210 }
8211
8212 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<Option<T>>
8213 where
8214 F: FnMut() -> Fut,
8215 Fut: Future<Output = Result<T>>,
8216 {
8217 let mut retries = 0;
8218
8219 loop {
8220 if self
8221 .run_stop
8222 .as_ref()
8223 .is_some_and(|stop| stop.load(Ordering::SeqCst))
8224 {
8225 return Ok(None);
8226 }
8227 match operation().await {
8228 Err(error)
8229 if worker_operation_is_retryable(&error)
8230 && (retries < self.retry_policy.max_retries
8231 || (self.recover_transient_outages
8232 && self.run_stop.is_some()
8233 && self.retry_policy.max_retries > 0)) =>
8234 {
8235 retries = retries.saturating_add(1);
8236 let delay = worker_retry_delay(self.retry_policy, retries)
8237 .max(Duration::from_millis(1));
8238 if let Some(stop) = &self.run_stop {
8239 tokio::select! {
8240 _ = tokio::time::sleep(delay) => {}
8241 _ = wait_for_worker_stop(stop) => return Ok(None),
8242 }
8243 } else {
8244 tokio::time::sleep(delay).await;
8245 }
8246 }
8247 result => return result.map(Some),
8248 }
8249 }
8250 }
8251
8252 async fn settle_worker_poll_response<T>(
8253 &self,
8254 response: Result<Option<T>>,
8255 ) -> Result<Option<T>> {
8256 match response {
8257 Ok(response) => Ok(response),
8258 Err(error) => {
8259 let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
8260 return Err(error);
8261 };
8262 let minimum_delay = self
8263 .retry_policy
8264 .initial_backoff
8265 .max(Duration::from_millis(1));
8266 let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
8267 tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
8268 Ok(None)
8269 }
8270 }
8271 }
8272
8273 async fn execute_query_task(
8274 &self,
8275 mut task: QueryTask,
8276 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
8277 validate_query_task_payloads(&task).map_err(|error| {
8278 QueryTaskExecutionFailure::new(
8279 "query_payload_decode_failed",
8280 error.to_string(),
8281 "QueryPayloadDecodeFailed",
8282 )
8283 })?;
8284
8285 if !self.workflows.contains_key(&task.workflow_type) {
8286 return Err(QueryTaskExecutionFailure::new(
8287 "query_workflow_type_not_registered",
8288 format!("no workflow registered for type {:?}", task.workflow_type),
8289 "WorkflowTypeNotRegistered",
8290 ));
8291 }
8292
8293 let Some(handlers) = self.queries.get(&task.workflow_type) else {
8294 return Err(QueryTaskExecutionFailure::new(
8295 "query_handler_unavailable",
8296 format!(
8297 "query handlers are unavailable for workflow type {:?}",
8298 task.workflow_type
8299 ),
8300 "QueryHandlerUnavailable",
8301 ));
8302 };
8303 let Some(query) = handlers.get(&task.query_name) else {
8304 return Err(QueryTaskExecutionFailure::new(
8305 "rejected_unknown_query",
8306 format!("unknown query {:?}", task.query_name),
8307 "QueryFailed",
8308 ));
8309 };
8310
8311 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
8312 .map_err(|error| {
8313 QueryTaskExecutionFailure::new(
8314 "query_payload_decode_failed",
8315 format!("cannot decode query arguments: {error}"),
8316 "QueryPayloadDecodeFailed",
8317 )
8318 })?;
8319 let workflow_input_typed =
8320 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
8321 .map_err(|error| {
8322 QueryTaskExecutionFailure::new(
8323 "query_workflow_state_unavailable",
8324 format!("cannot decode workflow start input: {error}"),
8325 "QueryWorkflowStateUnavailable",
8326 )
8327 })?;
8328 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
8329 QueryTaskExecutionFailure::new(
8330 "query_workflow_state_unavailable",
8331 format!("cannot project workflow start input: {error}"),
8332 "QueryWorkflowStateUnavailable",
8333 )
8334 })?;
8335 hydrate_query_history_from_export(&mut task).map_err(|error| {
8336 QueryTaskExecutionFailure::new(
8337 "query_workflow_state_unavailable",
8338 format!("cannot restore query history snapshot: {error}"),
8339 "QueryWorkflowStateUnavailable",
8340 )
8341 })?;
8342 enrich_query_history_from_export(&mut task).map_err(|error| {
8343 QueryTaskExecutionFailure::new(
8344 "query_workflow_state_unavailable",
8345 format!("cannot restore compact query history payloads: {error}"),
8346 "QueryWorkflowStateUnavailable",
8347 )
8348 })?;
8349 let signal_events = query_signal_events(&task).map_err(|error| {
8350 QueryTaskExecutionFailure::new(
8351 "query_workflow_state_unavailable",
8352 format!("cannot decode committed workflow signals: {error}"),
8353 "QueryWorkflowStateUnavailable",
8354 )
8355 })?;
8356 let history_events = Arc::new(std::mem::take(&mut task.history_events));
8357 let context = QueryContext {
8358 workflow_id: task.workflow_id,
8359 run_id: task.run_id,
8360 workflow_type: task.workflow_type.clone(),
8361 run_status: task.run_status,
8362 workflow_input,
8363 workflow_input_avro_value: workflow_input_typed.clone(),
8364 history_events: Arc::clone(&history_events),
8365 signal_events: Arc::new(signal_events),
8366 };
8367
8368 let future = match query {
8369 RegisteredQuery::Snapshot(handler) => handler(context, args),
8370 RegisteredQuery::Replayed {
8371 state_type,
8372 handler,
8373 } => {
8374 let workflow = self
8375 .workflows
8376 .get(&task.workflow_type)
8377 .expect("workflow registration was checked above");
8378 if workflow.state_type != Some(*state_type) {
8379 return Err(QueryTaskExecutionFailure::new(
8380 "query_workflow_state_unavailable",
8381 "replayed query state type does not match its workflow registration",
8382 "QueryWorkflowStateUnavailable",
8383 ));
8384 }
8385 let replay = workflow.replay.as_ref().ok_or_else(|| {
8386 QueryTaskExecutionFailure::new(
8387 "query_workflow_state_unavailable",
8388 format!(
8389 "workflow type {:?} is not registered for instance-state replay",
8390 task.workflow_type
8391 ),
8392 "QueryWorkflowStateUnavailable",
8393 )
8394 })?;
8395 let workflow_state = Arc::new(Mutex::new(
8396 WorkflowState::new_with_identity(
8397 history_events.as_ref().clone(),
8398 context.workflow_id.clone(),
8399 context.run_id.clone(),
8400 self.task_queue.clone(),
8401 task.payload_codec,
8402 None,
8403 )
8404 .map_err(|error| {
8405 QueryTaskExecutionFailure::new(
8406 "query_workflow_state_unavailable",
8407 format!("workflow replay failed before query: {error}"),
8408 "QueryWorkflowStateUnavailable",
8409 )
8410 })?,
8411 ));
8412 let workflow_context = WorkflowContext {
8413 state: workflow_state,
8414 cancellation_scope_id: "root".into(),
8415 };
8416 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
8417 let mut cx = TaskContext::from_waker(noop_waker_ref());
8418 let outcome = {
8419 let _replay = cancellation_replay_clock::ReplayGuard::enter(&workflow_context)
8420 .map_err(|error| {
8421 QueryTaskExecutionFailure::new(
8422 "query_workflow_state_unavailable",
8423 error.to_string(),
8424 "QueryWorkflowStateUnavailable",
8425 )
8426 })?;
8427 invocation.future.as_mut().poll(&mut cx)
8428 };
8429 match outcome {
8430 Poll::Ready(Ok(_)) => {
8431 workflow_context
8432 .ensure_history_consumed()
8433 .map_err(|error| {
8434 QueryTaskExecutionFailure::new(
8435 "query_workflow_state_unavailable",
8436 format!("workflow replay failed before query: {error}"),
8437 "QueryWorkflowStateUnavailable",
8438 )
8439 })?;
8440 }
8441 Poll::Ready(Err(error)) => {
8442 return Err(QueryTaskExecutionFailure::new(
8443 "query_workflow_state_unavailable",
8444 format!("workflow replay failed before query: {error}"),
8445 "QueryWorkflowStateUnavailable",
8446 ));
8447 }
8448 Poll::Pending => {
8449 let commands = workflow_context.take_commands().map_err(|error| {
8450 QueryTaskExecutionFailure::new(
8451 "query_workflow_state_unavailable",
8452 format!("workflow replay failed before query: {error}"),
8453 "QueryWorkflowStateUnavailable",
8454 )
8455 })?;
8456 if commands.is_empty()
8457 && !workflow_context
8458 .matched_recorded_pending()
8459 .map_err(|error| {
8460 QueryTaskExecutionFailure::new(
8461 "query_workflow_state_unavailable",
8462 format!("workflow replay failed before query: {error}"),
8463 "QueryWorkflowStateUnavailable",
8464 )
8465 })?
8466 {
8467 return Err(QueryTaskExecutionFailure::new(
8468 "query_workflow_state_unavailable",
8469 "workflow replay yielded without a durable command",
8470 "QueryWorkflowStateUnavailable",
8471 ));
8472 }
8473 }
8474 }
8475 let state = (invocation.snapshot)().map_err(|error| {
8476 QueryTaskExecutionFailure::new(
8477 "query_workflow_state_unavailable",
8478 format!("cannot snapshot replayed workflow state: {error}"),
8479 "QueryWorkflowStateUnavailable",
8480 )
8481 })?;
8482 handler(context, state, args).map_err(|message| {
8483 QueryTaskExecutionFailure::new(
8484 "query_workflow_state_unavailable",
8485 message,
8486 "QueryWorkflowStateUnavailable",
8487 )
8488 })?
8489 }
8490 };
8491
8492 future.await.map_err(|error| {
8493 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
8494 })
8495 }
8496
8497 #[cfg(test)]
8498 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
8499 Ok(self.execute_workflow_task_decision(task)?.commands)
8500 }
8501
8502 fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
8503 self.execute_workflow_task_decision_with_cancellation(task, None)
8504 }
8505
8506 fn execute_workflow_task_decision_with_cancellation(
8507 &self,
8508 task: WorkflowTask,
8509 observation: Option<&CancellationRequest>,
8510 ) -> Result<WorkflowTaskDecision> {
8511 match self.prepare_workflow_task_execution(task, observation)? {
8512 PreparedWorkflowTask::Decision(decision) => Ok(decision),
8513 PreparedWorkflowTask::Execution {
8514 task,
8515 context,
8516 mut future,
8517 } => {
8518 let outcome = Self::poll_workflow_future(&context, &mut future)?;
8519 self.finish_workflow_task_execution(&task, &context, outcome)
8520 }
8521 }
8522 }
8523
8524 fn prepare_workflow_task_execution(
8525 &self,
8526 task: WorkflowTask,
8527 observation: Option<&CancellationRequest>,
8528 ) -> Result<PreparedWorkflowTask> {
8529 if (self.allow_cancellation_scope_authoring || self.allow_cancellation_scope_delivery)
8530 && !self.cooperative_cancellation_enabled
8531 {
8532 return Err(Error::CancellationScopeExecutionUnavailable);
8533 }
8534 validate_workflow_task_payloads(&task)?;
8535 cooperative_cancellation::assert_cancellation_scope_replay_supported(
8536 &task.history_events,
8537 self.allow_cancellation_scope_authoring,
8538 self.allow_cancellation_scope_delivery,
8539 )?;
8540 let cancellation_history = if let Some(observation) = observation {
8541 observation.validate_observation()?;
8542 cooperative_cancellation::cancellation_claim(&task)?;
8543 let canonical = CancellationHistory::from_events(
8544 &task.history_events,
8545 task.run_id.as_deref().unwrap_or_default(),
8546 Some(observation),
8547 )?;
8548 if canonical.request_index >= task.history_events.len() {
8549 return Err(Error::InvalidCooperativeCancellation(
8550 "workflow replay requires the observed canonical request".into(),
8551 ));
8552 }
8553 Some(canonical)
8554 } else {
8555 None
8556 };
8557 if let Some(update_id) = task
8558 .workflow_update_id
8559 .as_deref()
8560 .filter(|update_id| !update_id.is_empty())
8561 {
8562 if self.allow_cancellation_scope_delivery {
8563 return Err(Error::CancellationScopeExecutionUnavailable);
8564 }
8565 CancellationHistory::from_events(
8566 &task.history_events,
8567 task.run_id.as_deref().unwrap_or_default(),
8568 None,
8569 )?;
8570 return self
8571 .execute_update_task(&task, update_id)
8572 .map(WorkflowTaskDecision::without_message_streams)
8573 .map(PreparedWorkflowTask::Decision);
8574 }
8575
8576 let workflow = self
8577 .workflows
8578 .get(&task.workflow_type)
8579 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
8580 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
8581 let resume_signal = decode_resume_signal(&task)?;
8582 let history_budget = WorkflowHistoryBudget {
8583 event_count: task
8584 .total_history_events
8585 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
8586 size_bytes: task.history_size_bytes,
8587 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
8588 pressure: task.history_budget_pressure.clone(),
8589 };
8590 let workflow_command_identity = task
8591 .workflow_command_id
8592 .clone()
8593 .filter(|identity| !identity.is_empty())
8594 .or_else(|| (!task.task_id.is_empty()).then(|| task.task_id.clone()))
8595 .unwrap_or_default();
8596 let mut workflow_state = WorkflowState::new_with_identity_and_scope_delivery(
8597 task.history_events.clone(),
8598 task.workflow_id.clone(),
8599 task.run_id.clone(),
8600 self.task_queue.clone(),
8601 task.payload_codec.clone(),
8602 resume_signal,
8603 self.allow_cancellation_scope_authoring,
8604 self.allow_cancellation_scope_delivery,
8605 )?;
8606 workflow_state.history_budget = history_budget;
8607 workflow_state.workflow_command_identity = workflow_command_identity;
8608 if let Some(canonical) = cancellation_history {
8609 workflow_state.cancellation_history = canonical;
8610 workflow_state.cancellation_delivery_enabled = true;
8611 }
8612 if workflow_state.cancellation_history.request.is_none() {
8613 workflow_state.cancel_requested = task.cancel_requested;
8614 }
8615 let state = Arc::new(Mutex::new(workflow_state));
8616 let ctx = WorkflowContext {
8617 state,
8618 cancellation_scope_id: "root".into(),
8619 };
8620 let future = (workflow.execute)(ctx.clone(), input);
8621 Ok(PreparedWorkflowTask::Execution {
8622 task,
8623 context: ctx,
8624 future,
8625 })
8626 }
8627
8628 fn poll_workflow_future(
8629 ctx: &WorkflowContext,
8630 future: &mut WorkflowFuture,
8631 ) -> Result<Poll<Result<AvroValue>>> {
8632 let mut cx = TaskContext::from_waker(noop_waker_ref());
8633 let _replay = cancellation_replay_clock::ReplayGuard::enter(ctx)?;
8634 Ok(future.as_mut().poll(&mut cx))
8635 }
8636
8637 fn finish_workflow_task_execution(
8638 &self,
8639 task: &WorkflowTask,
8640 ctx: &WorkflowContext,
8641 outcome: Poll<Result<AvroValue>>,
8642 ) -> Result<WorkflowTaskDecision> {
8643 if !ctx
8644 .state
8645 .lock()
8646 .map_err(|_| Error::WorkflowStatePoisoned)?
8647 .local_activity_requests
8648 .is_empty()
8649 {
8650 return Err(Error::CooperativeCancellationUnavailable(
8651 "local_activity_execution_unavailable: unresolved inline local calls require an ordinary workflow worker executor; cooperative execution requires prepared local supervision".into(),
8652 ));
8653 }
8654 let awaiting_canonical_boundary = {
8655 let state = ctx.state.lock().map_err(|_| Error::WorkflowStatePoisoned)?;
8656 state.cancellation_delivery_intent.is_some()
8657 || state.cancellation_scope_opening.is_some()
8658 || state
8659 .scope_delivery
8660 .as_ref()
8661 .is_some_and(|replay| replay.intent.is_some())
8662 };
8663 if awaiting_canonical_boundary {
8664 if let Poll::Ready(Err(error)) = outcome {
8668 if workflow_task_integrity_error(&error) {
8669 return Err(error);
8670 }
8671 }
8672 return self.message_stream_decision(&ctx, ctx.take_commands()?);
8673 }
8674
8675 match outcome {
8676 Poll::Ready(Ok(result)) => {
8677 ctx.ensure_history_consumed()?;
8678 let result = encode_typed_envelope(&result, &task.payload_codec)?;
8679 let mut commands = ctx.take_commands()?;
8680 commands.push(json!({
8681 "type": "complete_workflow",
8682 "result": result
8683 }));
8684 self.message_stream_decision(&ctx, commands)
8685 }
8686 Poll::Ready(Err(error)) => {
8687 if let Error::ContinueAsNew(request) = error {
8688 let mut commands = ctx.take_commands()?;
8689 if let Some(command) = ctx.continue_as_new_command(request)? {
8690 commands.push(command);
8691 }
8692 ctx.ensure_history_consumed()?;
8693 return self.message_stream_decision(&ctx, commands);
8694 }
8695 if workflow_task_integrity_error(&error) {
8696 return Err(error);
8701 }
8702 ctx.ensure_history_consumed()?;
8705 let mut commands = ctx.take_commands()?;
8706 commands.push(workflow_failure_command(
8707 &error,
8708 recorded_activity_failure_boundary(&ctx, &error),
8709 ));
8710 self.message_stream_decision(&ctx, commands)
8711 }
8712 Poll::Pending => {
8713 let commands = ctx.take_commands()?;
8714 if commands.is_empty() && !ctx.matched_recorded_pending()? {
8715 Err(Error::WorkflowYieldedWithoutCommand)
8716 } else {
8717 self.message_stream_decision(&ctx, commands)
8718 }
8719 }
8720 }
8721 }
8722
8723 fn message_stream_decision(
8724 &self,
8725 ctx: &WorkflowContext,
8726 commands: Vec<Value>,
8727 ) -> Result<WorkflowTaskDecision> {
8728 if !self.cooperative_cancellation_enabled
8729 && commands.iter().any(|command| {
8730 command["type"] == "schedule_activity"
8731 && command.get("cancellation_policy").is_some()
8732 })
8733 {
8734 return Err(Error::CooperativeCancellationUnavailable(format!(
8735 "activity_cancellation_policy_not_supported: Rust worker {} must enable cooperative cancellation with worker protocol 1.20 and a compatible Server/Native backend",
8736 self.worker_id,
8737 )));
8738 }
8739 if !self.cooperative_cancellation_enabled
8740 && commands.iter().any(|command| {
8741 command["type"] == "start_child_workflow"
8742 && (command["parent_close_policy"] == "request_cancellation"
8743 || matches!(
8744 command["cancellation_policy"].as_str(),
8745 Some("try_cancel" | "wait_cancellation_completed")
8746 ))
8747 })
8748 {
8749 return Err(Error::CooperativeCancellationUnavailable(format!(
8750 "child_cancellation_policy_not_supported: Rust worker {} must enable cooperative cancellation with worker protocol 1.20 and a compatible Server/Native backend",
8751 self.worker_id,
8752 )));
8753 }
8754 let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
8755 let state = ctx.state.lock().map_err(|_| Error::WorkflowStatePoisoned)?;
8756 let scope_delivery = state
8757 .scope_delivery
8758 .as_ref()
8759 .and_then(|replay| replay.intent.clone());
8760 if scope_delivery
8761 .as_ref()
8762 .is_some_and(|intent| commands.len() != intent.command_count)
8763 {
8764 return Err(invalid_recorded_history(
8765 "cancellation_scope_pending_call_escaped",
8766 scope_delivery.as_ref().unwrap().boundary.sequence,
8767 "commands preceding the pending scoped call",
8768 "commands after suspension",
8769 "workflow cannot publish beyond an uncommitted scope delivery",
8770 ));
8771 }
8772 if let Some(opening) = &state.cancellation_scope_opening {
8773 if commands.len() != opening.command_count {
8774 return Err(invalid_recorded_history(
8775 "cancellation_scope_pending_call_escaped",
8776 opening.sequence,
8777 "commands before original scope opening",
8778 "commands after suspended opening",
8779 "workflow cannot publish beyond an uncommitted scope opening",
8780 ));
8781 }
8782 }
8783 if state.cancellation_delivery_intent.is_some()
8784 && commands.len() != state.cancellation_delivery_command_count
8785 {
8786 return Err(invalid_recorded_history(
8787 "cooperative_cancellation_pending_call_escaped",
8788 state
8789 .cancellation_delivery_intent
8790 .as_ref()
8791 .expect("pending delivery")
8792 .sequence,
8793 "only commands preceding the pending cancellation call",
8794 "commands authored after a suspended call",
8795 "workflow code cannot publish work beyond an uncommitted cancellation boundary",
8796 ));
8797 }
8798 Ok(WorkflowTaskDecision {
8799 commands,
8800 message_stream_cursors,
8801 message_stream_waits,
8802 cancellation_delivery: state.cancellation_delivery_intent.clone(),
8803 cancellation_scope_opening: state.cancellation_scope_opening.clone(),
8804 cancellation_scope_delivery: scope_delivery,
8805 })
8806 }
8807
8808 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
8809 if !self.workflows.contains_key(&task.workflow_type) {
8810 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
8811 }
8812
8813 let accepted = task.history_events.iter().rev().find_map(|event| {
8814 (event.event_type == "UpdateAccepted"
8815 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
8816 .then_some(&event.payload)
8817 });
8818 let update_name = accepted
8819 .and_then(|payload| payload.get("update_name"))
8820 .and_then(Value::as_str)
8821 .or(task.update_name.as_deref())
8822 .unwrap_or_default();
8823 let Some(handler) = self
8824 .updates
8825 .get(&task.workflow_type)
8826 .and_then(|handlers| handlers.get(update_name))
8827 else {
8828 return Ok(vec![json!({
8829 "type": "fail_update",
8830 "update_id": update_id,
8831 "message": format!(
8832 "no update handler is registered for {}.{update_name}",
8833 task.workflow_type
8834 ),
8835 "exception_type": "UnknownUpdate",
8836 "non_retryable": true,
8837 })]);
8838 };
8839 let arguments = accepted
8840 .and_then(|payload| payload.get("arguments"))
8841 .or(task.arguments.as_ref());
8842 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
8843 let workflow_input_avro_value =
8844 decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
8845 let workflow_input = workflow_input_avro_value.clone().into_json()?;
8846 let signal_events = history_signal_events(&task.history_events, None, &task.payload_codec)?;
8847 let context = QueryContext {
8848 workflow_id: task.workflow_id.clone(),
8849 run_id: task.run_id.clone(),
8850 workflow_type: task.workflow_type.clone(),
8851 run_status: Some("running".to_string()),
8852 workflow_input,
8853 workflow_input_avro_value,
8854 history_events: Arc::new(task.history_events.clone()),
8855 signal_events: Arc::new(signal_events),
8856 };
8857 let mut future = handler(context, arguments);
8858 let mut cx = TaskContext::from_waker(noop_waker_ref());
8859
8860 match future.as_mut().poll(&mut cx) {
8861 Poll::Ready(Ok(result)) => Ok(vec![json!({
8862 "type": "complete_update",
8863 "update_id": update_id,
8864 "result": encode_typed_envelope(&result, &task.payload_codec)?,
8865 })]),
8866 Poll::Ready(Err(error)) => Ok(vec![json!({
8867 "type": "fail_update",
8868 "update_id": update_id,
8869 "message": error.to_string(),
8870 "exception_type": "UpdateFailed",
8871 "non_retryable": true,
8872 })]),
8873 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
8874 }
8875 }
8876
8877 #[cfg(test)]
8878 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
8879 self.execute_session_activity_task(task, None).await
8880 }
8881
8882 async fn execute_session_activity_task(
8883 &self,
8884 task: ActivityTask,
8885 worker_session: Option<WorkerSession>,
8886 ) -> Result<AvroValue> {
8887 validate_activity_task_payloads(&task)?;
8888 if worker_session.is_some() {
8889 if task.lease_owner.as_deref() != Some(self.worker_id.as_str())
8890 || task
8891 .activity_attempt_id
8892 .as_deref()
8893 .or(task.attempt_id.as_deref())
8894 .is_none_or(str::is_empty)
8895 || matches!((&task.activity_attempt_id,&task.attempt_id),(Some(a),Some(b)) if a != b)
8896 {
8897 return Err(Error::ActivityExecutionAbandoned(
8898 "session activity requires this worker's exact immutable claim".into(),
8899 ));
8900 }
8901 }
8902
8903 let handler = self
8904 .activities
8905 .get(&task.activity_type)
8906 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
8907 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
8908 let attempt_id = task
8909 .activity_attempt_id
8910 .clone()
8911 .or(task.attempt_id.clone())
8912 .unwrap_or_default();
8913 let lease_owner = task
8914 .lease_owner
8915 .clone()
8916 .unwrap_or_else(|| self.worker_id.clone());
8917 let session = worker_session.clone();
8918 let ctx = ActivityContext {
8919 client: self.client.clone(),
8920 task_id: task.task_id,
8921 activity_attempt_id: attempt_id,
8922 lease_owner,
8923 activity_type: task.activity_type,
8924 attempt_number: task.attempt_number,
8925 task_queue: self.task_queue.clone(),
8926 worker_id: self.worker_id.clone(),
8927 claim_guard: None,
8928 local_heartbeats: None,
8929 worker_session,
8930 };
8931
8932 let callback = handler(ctx, args);
8933 if let Some(session) = session {
8934 tokio::select! {
8935 biased;
8936 _ = session.wait_until_unavailable() => Err(Error::ActivityExecutionAbandoned("worker-session lease or TTL expired, closed or became uncertain".into())),
8937 result = callback => if session.active() { result } else {
8938 Err(Error::ActivityExecutionAbandoned("worker-session authority ended before callback settlement".into()))
8939 },
8940 }
8941 } else {
8942 callback.await
8943 }
8944 }
8945}
8946
8947fn poller_result(
8948 kind: &str,
8949 result: std::result::Result<Result<()>, tokio::task::JoinError>,
8950) -> Result<()> {
8951 match result {
8952 Ok(result) => result,
8953 Err(error) => Err(Error::WorkerLoop(format!(
8954 "{kind} poller join error: {error}"
8955 ))),
8956 }
8957}
8958
8959fn optional_poller_result(
8960 kind: &str,
8961 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
8962) -> Result<()> {
8963 match result {
8964 Some(result) => poller_result(kind, result),
8965 None => Ok(()),
8966 }
8967}
8968
8969async fn join_pollers(
8970 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
8971 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
8972 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
8973) -> Result<()> {
8974 let mut first_error = None;
8975
8976 if let Some(handle) = workflow_poller {
8977 if let Err(error) = poller_result("workflow", handle.await) {
8978 first_error.get_or_insert(error);
8979 }
8980 }
8981
8982 if let Some(handle) = activity_poller {
8983 if let Err(error) = poller_result("activity", handle.await) {
8984 first_error.get_or_insert(error);
8985 }
8986 }
8987
8988 if let Some(handle) = query_poller {
8989 if let Err(error) = poller_result("query", handle.await) {
8990 first_error.get_or_insert(error);
8991 }
8992 }
8993
8994 if let Some(error) = first_error {
8995 Err(error)
8996 } else {
8997 Ok(())
8998 }
8999}
9000
9001fn default_worker_id() -> String {
9002 let millis = SystemTime::now()
9003 .duration_since(UNIX_EPOCH)
9004 .unwrap_or_default()
9005 .as_millis();
9006 format!("rust-worker-{}-{millis}", std::process::id())
9007}
9008
9009fn percent_encode_path_segment(segment: &str) -> String {
9010 const HEX: &[u8; 16] = b"0123456789ABCDEF";
9011 let mut encoded = String::with_capacity(segment.len());
9012
9013 for byte in segment.bytes() {
9014 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
9015 encoded.push(char::from(byte));
9016 } else {
9017 encoded.push('%');
9018 encoded.push(char::from(HEX[(byte >> 4) as usize]));
9019 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
9020 }
9021 }
9022
9023 encoded
9024}
9025
9026fn unique_request_id(prefix: &str) -> String {
9027 let nanos = SystemTime::now()
9028 .duration_since(UNIX_EPOCH)
9029 .unwrap_or_default()
9030 .as_nanos();
9031 format!("{prefix}-{}-{nanos}", std::process::id())
9032}
9033
9034#[derive(Debug)]
9035struct QueryTaskExecutionFailure {
9036 reason: String,
9037 message: String,
9038 failure_type: String,
9039}
9040
9041impl QueryTaskExecutionFailure {
9042 fn new(
9043 reason: impl Into<String>,
9044 message: impl Into<String>,
9045 failure_type: impl Into<String>,
9046 ) -> Self {
9047 Self {
9048 reason: reason.into(),
9049 message: message.into(),
9050 failure_type: failure_type.into(),
9051 }
9052 }
9053}
9054
9055#[derive(Clone, Debug)]
9061pub struct WorkflowInstance<S> {
9062 state: Arc<Mutex<S>>,
9063}
9064
9065impl<S> WorkflowInstance<S> {
9066 fn new(state: S) -> Self {
9067 Self {
9068 state: Arc::new(Mutex::new(state)),
9069 }
9070 }
9071
9072 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
9074 let state = self
9075 .state
9076 .lock()
9077 .map_err(|_| Error::WorkflowStatePoisoned)?;
9078 Ok(reader(&state))
9079 }
9080
9081 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
9083 let mut state = self
9084 .state
9085 .lock()
9086 .map_err(|_| Error::WorkflowStatePoisoned)?;
9087 Ok(transition(&mut state))
9088 }
9089}
9090
9091impl<S: Clone> WorkflowInstance<S> {
9092 fn snapshot(&self) -> Result<S> {
9093 self.read(Clone::clone)
9094 }
9095}
9096
9097#[derive(Clone, Debug, PartialEq)]
9098pub struct MessageStreamMessage {
9099 pub stream_name: String,
9100 pub message_id: String,
9101 pub position: u64,
9102 pub arguments: Vec<AvroValue>,
9103}
9104
9105#[derive(Clone, Debug)]
9106pub struct MessageStream {
9107 ctx: WorkflowContext,
9108 name: String,
9109}
9110
9111impl MessageStream {
9112 pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
9114 if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
9115 return Err(Error::Codec(format!(
9116 "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
9117 )));
9118 }
9119 loop {
9120 if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
9121 return Ok(batch);
9122 }
9123
9124 self.ctx.record_message_stream_wait(&self.name)?;
9125 let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
9126 let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
9127 self.ctx.buffer_message_stream_delivery(arguments)?;
9128 if let Some(sequence) = replay_wait_sequence {
9129 self.ctx.buffer_message_stream_history_for_wait(sequence)?;
9130 }
9131 }
9132 }
9133
9134 pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
9135 self.receive(1)
9136 .await?
9137 .into_iter()
9138 .next()
9139 .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
9140 }
9141}
9142
9143#[derive(Clone, Debug)]
9144pub struct WorkflowContext {
9145 state: Arc<Mutex<WorkflowState>>,
9146 cancellation_scope_id: String,
9147}
9148
9149fn valid_memo_key(key: &str) -> bool {
9150 let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
9151
9152 !key.is_empty()
9153 && key.len() <= 64
9154 && (numeric_candidate.is_empty()
9155 || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
9156 && key
9157 .bytes()
9158 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
9159}
9160
9161fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
9162 BASE64
9163 .decode(encode_avro_value(value)?.blob)
9164 .map(|bytes| bytes.len())
9165 .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
9166}
9167
9168fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
9169 let AvroValue::Map(entries) = value else {
9170 return Err(Error::InvalidMemoUpdate(
9171 "entries must serialize to an Avro string-keyed map".to_string(),
9172 ));
9173 };
9174 if require_entries && entries.is_empty() {
9175 return Err(Error::InvalidMemoUpdate(
9176 "at least one entry is required".to_string(),
9177 ));
9178 }
9179 if entries.len() > MAX_MEMO_ENTRIES {
9180 return Err(Error::InvalidMemoUpdate(format!(
9181 "at most {MAX_MEMO_ENTRIES} entries are allowed"
9182 )));
9183 }
9184
9185 for (key, value) in &entries {
9186 if !valid_memo_key(&key) {
9187 return Err(Error::InvalidMemoUpdate(
9188 "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
9189 ));
9190 }
9191 if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
9192 return Err(Error::InvalidMemoUpdate(format!(
9193 "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
9194 )));
9195 }
9196 }
9197
9198 let value = AvroValue::Map(entries);
9199 if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
9200 return Err(Error::InvalidMemoUpdate(format!(
9201 "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
9202 )));
9203 }
9204 Ok(value)
9205}
9206
9207fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
9208 let object = envelope.as_object().ok_or_else(|| {
9209 Error::InvalidMemoUpdate(
9210 "history field must use the public {codec, blob} payload envelope".to_string(),
9211 )
9212 })?;
9213 if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
9214 return Err(Error::InvalidMemoUpdate(
9215 "history field must use exactly the public {codec, blob} payload envelope".to_string(),
9216 ));
9217 }
9218
9219 canonical_memo_entries(
9220 decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
9221 require_entries,
9222 )
9223}
9224
9225impl WorkflowContext {
9226 pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
9227 let name = name.into();
9228 if name.is_empty()
9229 || name.len() > 128
9230 || !name.bytes().all(|byte| {
9231 byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
9232 })
9233 {
9234 return Err(Error::Codec(
9235 "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
9236 .to_string(),
9237 ));
9238 }
9239 Ok(MessageStream {
9240 ctx: self.clone(),
9241 name,
9242 })
9243 }
9244
9245 fn record_message_stream_wait(&self, name: &str) -> Result<()> {
9246 let mut state = self
9247 .state
9248 .lock()
9249 .map_err(|_| Error::WorkflowStatePoisoned)?;
9250 let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
9251 state
9252 .message_stream_waits
9253 .insert(name.to_string(), position);
9254 Ok(())
9255 }
9256
9257 fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
9258 let mut state = self
9259 .state
9260 .lock()
9261 .map_err(|_| Error::WorkflowStatePoisoned)?;
9262 let cursor = state
9263 .message_stream_cursors
9264 .get(&message.stream_name)
9265 .copied()
9266 .unwrap_or(0);
9267 if message.position <= cursor {
9268 return Ok(());
9269 }
9270 let pending = state
9271 .message_stream_messages
9272 .entry(message.stream_name.clone())
9273 .or_default();
9274 if pending.iter().any(|candidate| {
9275 candidate.position == message.position || candidate.message_id == message.message_id
9276 }) {
9277 return Ok(());
9278 }
9279 pending.push(message);
9280 pending.sort_by_key(|candidate| candidate.position);
9281 Ok(())
9282 }
9283
9284 fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
9285 if let Some(delivery) = decode_message_stream_delivery(arguments)? {
9286 match delivery {
9287 MessageStreamDelivery::Message(message) => {
9288 let stream_name = message.stream_name.clone();
9289 self.buffer_message_stream(message)?;
9290 return Ok(Some(stream_name));
9291 }
9292 MessageStreamDelivery::Cursor {
9293 stream_name,
9294 through_position,
9295 } => self.apply_message_stream_cursor(&stream_name, through_position)?,
9296 }
9297 }
9298 Ok(None)
9299 }
9300
9301 fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
9302 let state = self
9303 .state
9304 .lock()
9305 .map_err(|_| Error::WorkflowStatePoisoned)?;
9306 Ok(match state.recorded_commands.get(state.command_cursor) {
9307 Some(RecordedCommand::SignalWait {
9308 sequence,
9309 signal_name,
9310 ..
9311 }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
9312 _ => None,
9313 })
9314 }
9315
9316 fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
9317 let (history, payload_codec) = {
9318 let state = self
9319 .state
9320 .lock()
9321 .map_err(|_| Error::WorkflowStatePoisoned)?;
9322 (
9323 Arc::clone(&state.history_events),
9324 state.payload_codec.clone(),
9325 )
9326 };
9327
9328 let Some(opened_index) = history.iter().position(|event| {
9329 event.event_type == "SignalWaitOpened"
9330 && durable_event_sequence(event) == Some(wait_sequence)
9331 && event.payload.get("signal_name").and_then(Value::as_str)
9332 == Some(MESSAGE_STREAM_SIGNAL)
9333 }) else {
9334 return Ok(());
9335 };
9336 let boundary_index = history
9337 .iter()
9338 .enumerate()
9339 .skip(opened_index + 1)
9340 .find_map(|(index, event)| {
9341 (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
9342 && is_authored_command_open_event(event))
9343 .then_some(index)
9344 })
9345 .unwrap_or(history.len());
9346
9347 for event in history[opened_index + 1..boundary_index]
9348 .iter()
9349 .filter(|event| {
9350 event.event_type == "SignalReceived"
9351 && event.payload.get("signal_name").and_then(Value::as_str)
9352 == Some(MESSAGE_STREAM_SIGNAL)
9353 })
9354 {
9355 let arguments = decode_signal_event_arguments(event, &payload_codec)?
9356 .into_iter()
9357 .map(AvroValue::into_json)
9358 .collect::<Result<Vec<_>>>()?;
9359 self.buffer_message_stream_delivery(arguments)?;
9360 }
9361 Ok(())
9362 }
9363
9364 fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
9365 let mut state = self
9366 .state
9367 .lock()
9368 .map_err(|_| Error::WorkflowStatePoisoned)?;
9369 let cursor = state
9370 .message_stream_cursors
9371 .entry(name.to_string())
9372 .or_default();
9373 *cursor = (*cursor).max(through_position);
9374 if let Some(pending) = state.message_stream_messages.get_mut(name) {
9375 pending.retain(|message| message.position > through_position);
9376 }
9377 Ok(())
9378 }
9379
9380 fn take_message_stream_batch(
9381 &self,
9382 name: &str,
9383 max_items: usize,
9384 ) -> Result<Option<Vec<MessageStreamMessage>>> {
9385 let mut state = self
9386 .state
9387 .lock()
9388 .map_err(|_| Error::WorkflowStatePoisoned)?;
9389 let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
9390 let pending = state
9391 .message_stream_messages
9392 .entry(name.to_string())
9393 .or_default();
9394 let count = contiguous_message_stream_count(pending, cursor, max_items);
9395 if count == 0 {
9396 return Ok(None);
9397 }
9398 let batch = pending.drain(..count).collect::<Vec<_>>();
9399 let position = batch.last().map(|message| message.position).unwrap_or(0);
9400 state
9401 .message_stream_cursors
9402 .insert(name.to_string(), position);
9403 state.message_stream_waits.remove(name);
9404 Ok(Some(batch))
9405 }
9406
9407 fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
9408 let state = self
9409 .state
9410 .lock()
9411 .map_err(|_| Error::WorkflowStatePoisoned)?;
9412 let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
9413 cursors.sort_by_key(|(name, _)| *name);
9414 let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
9415 waits.sort_by_key(|(name, _)| *name);
9416 Ok((
9417 cursors
9418 .into_iter()
9419 .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
9420 .collect(),
9421 waits
9422 .into_iter()
9423 .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
9424 .collect(),
9425 ))
9426 }
9427 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
9429 let state = self
9430 .state
9431 .lock()
9432 .map_err(|_| Error::WorkflowStatePoisoned)?;
9433 Ok(WorkflowIdentity {
9434 workflow_id: state.workflow_id.clone(),
9435 run_id: state.run_id.clone(),
9436 })
9437 }
9438
9439 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
9441 let state = self
9442 .state
9443 .lock()
9444 .map_err(|_| Error::WorkflowStatePoisoned)?;
9445 Ok(state.history_budget.clone())
9446 }
9447
9448 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
9454 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
9455 }
9456
9457 pub fn continue_as_new_with_options<T: Serialize>(
9459 &self,
9460 options: ContinueAsNewOptions,
9461 args: T,
9462 ) -> Result<Value> {
9463 options.validate()?;
9464 Err(Error::ContinueAsNew(ContinueAsNewRequest {
9465 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
9466 options,
9467 }))
9468 }
9469
9470 pub fn activity<T: Serialize>(
9471 &self,
9472 activity_type: impl Into<String>,
9473 args: T,
9474 ) -> ActivityCall {
9475 self.activity_with_options(activity_type, ActivityOptions::new(), args)
9476 }
9477
9478 pub fn activity_on_queue<T, Q>(
9479 &self,
9480 activity_type: impl Into<String>,
9481 task_queue: Option<Q>,
9482 args: T,
9483 ) -> ActivityCall
9484 where
9485 T: Serialize,
9486 Q: Into<String>,
9487 {
9488 let mut options = ActivityOptions::new();
9489 options.task_queue = task_queue.map(Into::into);
9490 self.activity_with_options(activity_type, options, args)
9491 }
9492
9493 pub fn activity_with_options<T: Serialize>(
9531 &self,
9532 activity_type: impl Into<String>,
9533 options: ActivityOptions,
9534 args: T,
9535 ) -> ActivityCall {
9536 ActivityCall {
9537 ctx: self.clone(),
9538 activity_type: activity_type.into(),
9539 options,
9540 worker_session: None,
9541 args: Some(AvroValue::from_serialize(&args)),
9542 scheduled: false,
9543 local: false,
9544 local_result: None,
9545 parallel_group_path: Vec::new(),
9546 }
9547 }
9548
9549 pub async fn activity_avro_value<T: Serialize>(
9550 &self,
9551 activity_type: impl Into<String>,
9552 args: T,
9553 ) -> Result<AvroValue> {
9554 let mut call = self.activity(activity_type, args);
9555 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
9556 }
9557
9558 pub async fn activity_avro_value_with_options<T: Serialize>(
9559 &self,
9560 activity_type: impl Into<String>,
9561 options: ActivityOptions,
9562 args: T,
9563 ) -> Result<AvroValue> {
9564 let mut call = self.activity_with_options(activity_type, options, args);
9565 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
9566 }
9567
9568 pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
9570 where
9571 I: Serialize,
9572 O: DeserializeOwned,
9573 {
9574 self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
9575 .await
9576 }
9577
9578 pub async fn activity_typed_with_options<I, O>(
9584 &self,
9585 activity_type: impl Into<String>,
9586 options: ActivityOptions,
9587 args: I,
9588 ) -> Result<O>
9589 where
9590 I: Serialize,
9591 O: DeserializeOwned,
9592 {
9593 let activity_type = activity_type.into();
9594 let encoded = AvroValue::from_serialize(&args).map_err(|error| {
9595 handler_type_error::<I>(
9596 HandlerKind::Activity,
9597 &activity_type,
9598 HandlerValueKind::Input,
9599 error.to_string(),
9600 )
9601 });
9602 let mut call = ActivityCall {
9603 ctx: self.clone(),
9604 activity_type: activity_type.clone(),
9605 options,
9606 worker_session: None,
9607 args: Some(encoded),
9608 scheduled: false,
9609 local: false,
9610 local_result: None,
9611 parallel_group_path: Vec::new(),
9612 };
9613 let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
9614 decode_handler_result(result, HandlerKind::Activity, &activity_type)
9615 }
9616
9617 pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
9624 ParallelCall::new(self.clone(), operations)
9625 }
9626
9627 pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
9629 self.parallel(operations)
9630 }
9631
9632 pub async fn parallel_avro_value(
9634 &self,
9635 operations: Vec<ParallelOperation>,
9636 ) -> Result<Vec<ParallelAvroResult>> {
9637 let mut call = self.parallel(operations);
9638 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
9639 }
9640
9641 pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
9644 let operations = operations
9645 .into_iter()
9646 .enumerate()
9647 .map(|(index, operation)| (SelectionKey::Index(index), operation))
9648 .collect();
9649 SelectCall::new(self.clone(), operations)
9650 }
9651
9652 pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
9654 where
9655 K: Into<SelectionKey>,
9656 {
9657 SelectCall::new(
9658 self.clone(),
9659 operations
9660 .into_iter()
9661 .map(|(key, operation)| (key.into(), operation))
9662 .collect(),
9663 )
9664 }
9665
9666 pub fn saga(&self) -> Saga {
9668 Saga::new(self.clone())
9669 }
9670
9671 pub fn is_cancellation_requested(&self) -> Result<bool> {
9677 let state = self
9678 .state
9679 .lock()
9680 .map_err(|_| Error::WorkflowStatePoisoned)?;
9681 if self.cancellation_scope_id != "root" && state.scope_delivery.is_some() {
9682 return Ok(state
9683 .scope_delivery
9684 .as_ref()
9685 .unwrap()
9686 .contexts
9687 .contains_key(&self.cancellation_scope_id));
9688 }
9689 Ok(state.cancel_requested
9690 || state
9691 .scope_delivery
9692 .as_ref()
9693 .is_some_and(|replay| replay.contexts.contains_key(&self.cancellation_scope_id)))
9694 }
9695
9696 pub fn throw_if_cancellation_requested(&self) -> Result<()> {
9701 let mut state = self
9702 .state
9703 .lock()
9704 .map_err(|_| Error::WorkflowStatePoisoned)?;
9705 if state.cancellation_shield_depth == 0 {
9706 if let Some(replay) = &state.scope_delivery {
9707 if let Some((context, _)) = replay.contexts.get(&self.cancellation_scope_id) {
9708 let delivered = replay
9709 .consumed_delivery_for_scope(&self.cancellation_scope_id)
9710 .ok_or_else(|| {
9711 Error::InvalidCooperativeCancellation(
9712 "scoped request lacks its consumed original delivery".into(),
9713 )
9714 })?;
9715 return Err(Error::CancellationScopeRequested(
9716 CancellationScopeRequested {
9717 context: context
9718 .clone()
9719 .with_replay(Some(Arc::downgrade(&self.state))),
9720 delivery: delivered.boundary.clone(),
9721 },
9722 ));
9723 }
9724 }
9725 }
9726 if state.cancel_requested && state.cancellation_shield_depth == 0 {
9727 if self.cancellation_scope_id != "root" && state.scope_delivery.is_some() {
9728 return Ok(());
9729 }
9730 return Err(state.cancellation_error());
9731 }
9732 Ok(())
9733 }
9734
9735 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
9736 SignalCall {
9737 ctx: self.clone(),
9738 signal_name: signal_name.into(),
9739 runtime_reserved_allowed: false,
9740 opened_wait: false,
9741 matched_pending: false,
9742 parallel_group_path: Vec::new(),
9743 }
9744 }
9745
9746 fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
9747 SignalCall {
9748 ctx: self.clone(),
9749 signal_name: signal_name.into(),
9750 runtime_reserved_allowed: true,
9751 opened_wait: false,
9752 matched_pending: false,
9753 parallel_group_path: Vec::new(),
9754 }
9755 }
9756
9757 pub async fn wait_signal_avro_value(
9758 &self,
9759 signal_name: impl Into<String>,
9760 ) -> Result<Vec<AvroValue>> {
9761 let mut call = self.wait_signal(signal_name);
9762 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
9763 }
9764
9765 pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
9771 self.signals_avro_value(signal_name)?
9772 .into_iter()
9773 .map(|arguments| {
9774 arguments
9775 .into_iter()
9776 .map(AvroValue::into_json)
9777 .collect::<Result<Vec<_>>>()
9778 })
9779 .collect()
9780 }
9781
9782 pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
9784 let state = self
9785 .state
9786 .lock()
9787 .map_err(|_| Error::WorkflowStatePoisoned)?;
9788 state
9789 .history_events
9790 .iter()
9791 .filter(|event| {
9792 event.event_type == "SignalReceived"
9793 && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
9794 })
9795 .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
9796 .collect()
9797 }
9798
9799 pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
9805 self.updates_avro_value(update_name)?
9806 .into_iter()
9807 .map(|arguments| {
9808 arguments
9809 .into_iter()
9810 .map(AvroValue::into_json)
9811 .collect::<Result<Vec<_>>>()
9812 })
9813 .collect()
9814 }
9815
9816 pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
9818 let state = self
9819 .state
9820 .lock()
9821 .map_err(|_| Error::WorkflowStatePoisoned)?;
9822 let mut seen = Vec::new();
9823 let mut updates = Vec::new();
9824 for event in state.history_events.iter() {
9825 if !matches!(
9826 event.event_type.as_str(),
9827 "UpdateAccepted" | "UpdateApplied"
9828 ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
9829 || event.payload.get("arguments").is_none()
9830 {
9831 continue;
9832 }
9833 if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
9834 if seen.iter().any(|recorded| recorded == update_id) {
9835 continue;
9836 }
9837 seen.push(update_id.to_string());
9838 }
9839 updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
9840 }
9841 Ok(updates)
9842 }
9843
9844 pub fn wait_condition<F>(
9852 &self,
9853 options: ConditionWaitOptions,
9854 predicate: F,
9855 ) -> ConditionWaitCall
9856 where
9857 F: Fn() -> Result<bool> + Send + 'static,
9858 {
9859 ConditionWaitCall {
9860 ctx: self.clone(),
9861 options,
9862 predicate: Box::new(predicate),
9863 occurrence_id: None,
9864 opened_wait: false,
9865 parallel_group_path: Vec::new(),
9866 }
9867 }
9868
9869 pub fn sleep(&self, duration: Duration) -> TimerCall {
9890 let delay_seconds = duration
9891 .as_secs()
9892 .checked_add(u64::from(duration.subsec_nanos() > 0));
9893 TimerCall {
9894 ctx: self.clone(),
9895 delay_seconds,
9896 scheduled: false,
9897 matched_pending: false,
9898 parallel_group_path: Vec::new(),
9899 }
9900 }
9901
9902 pub fn start_timer(&self, duration: Duration) -> TimerCall {
9904 self.sleep(duration)
9905 }
9906
9907 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
9914 where
9915 T: Serialize + DeserializeOwned,
9916 F: FnOnce() -> T,
9917 {
9918 {
9919 let mut state = self
9920 .state
9921 .lock()
9922 .map_err(|_| Error::WorkflowStatePoisoned)?;
9923 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9924 return match recorded {
9925 RecordedCommand::SideEffect { sequence, value } => {
9926 state.command_cursor += 1;
9927 value.deserialize().map_err(|error| {
9928 Error::NonDeterministicReplay(ReplayFailure::new(
9929 "side_effect_type_mismatch",
9930 Some(sequence),
9931 Some(std::any::type_name::<T>().to_string()),
9932 Some(error.to_string()),
9933 "recorded side-effect value is incompatible with the requested Rust type",
9934 ))
9935 })
9936 }
9937 other => Err(command_mismatch(&other, "side effect")),
9938 };
9939 }
9940 }
9941
9942 let value = callback();
9943 let avro_value = AvroValue::from_serialize(&value)?;
9944 let mut state = self
9945 .state
9946 .lock()
9947 .map_err(|_| Error::WorkflowStatePoisoned)?;
9948 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
9949 state.commands.push(json!({
9950 "type": "record_side_effect",
9951 "result": result,
9952 }));
9953 Ok(value)
9954 }
9955
9956 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
9958 where
9959 F: FnOnce() -> AvroValue,
9960 {
9961 {
9962 let mut state = self
9963 .state
9964 .lock()
9965 .map_err(|_| Error::WorkflowStatePoisoned)?;
9966 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9967 return match recorded {
9968 RecordedCommand::SideEffect { value, .. } => {
9969 state.command_cursor += 1;
9970 Ok(value)
9971 }
9972 other => Err(command_mismatch(&other, "side effect")),
9973 };
9974 }
9975 }
9976
9977 let value = callback();
9978 let mut state = self
9979 .state
9980 .lock()
9981 .map_err(|_| Error::WorkflowStatePoisoned)?;
9982 let result = encode_typed_envelope(&value, &state.payload_codec)?;
9983 state.commands.push(json!({
9984 "type": "record_side_effect",
9985 "result": result,
9986 }));
9987 Ok(value)
9988 }
9989
9990 pub fn append_workflow_stream(
9996 &self,
9997 stream_name: impl Into<String>,
9998 items: &[WorkflowStreamAppendItem],
9999 max_pending_items: Option<u64>,
10000 ) -> Result<()> {
10001 if items.is_empty() {
10002 return Err(Error::Codec(
10003 "workflow_stream_items_empty: append requires at least one item".to_string(),
10004 ));
10005 }
10006 if max_pending_items == Some(0) {
10007 return Err(Error::Codec(
10008 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
10009 .to_string(),
10010 ));
10011 }
10012 let stream_name = stream_name.into();
10013 if stream_name.is_empty() {
10014 return Err(Error::Codec(
10015 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
10016 ));
10017 }
10018
10019 let mut state = self
10020 .state
10021 .lock()
10022 .map_err(|_| Error::WorkflowStatePoisoned)?;
10023 let command_ordinal = state.workflow_stream_command_counter;
10024 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10025 state.workflow_stream_command_counter += 1;
10026 return match recorded {
10027 RecordedCommand::SideEffect { .. } => {
10028 state.command_cursor += 1;
10029 Ok(())
10030 }
10031 other => Err(command_mismatch(&other, "workflow stream append")),
10032 };
10033 }
10034
10035 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
10036 state.workflow_stream_command_counter += 1;
10037 let wire_items = items
10038 .iter()
10039 .enumerate()
10040 .map(|(item_index, item)| {
10041 item.wire_value(Some(format!(
10042 "dw-stream:{identity}:{command_ordinal}:{item_index}"
10043 )))
10044 })
10045 .collect::<Vec<_>>();
10046 let mut directive = json!({
10047 "operation": "append",
10048 "stream_name": stream_name,
10049 "command_identity": identity,
10050 "command_ordinal": command_ordinal,
10051 "items": wire_items,
10052 });
10053 if let Some(max_pending_items) = max_pending_items {
10054 directive["max_pending_items"] = json!(max_pending_items);
10055 }
10056 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
10057 state.commands.push(json!({
10058 "type": "record_side_effect",
10059 "result": result,
10060 "workflow_stream": directive,
10061 }));
10062 Ok(())
10063 }
10064
10065 pub fn close_workflow_stream(
10067 &self,
10068 stream_name: impl Into<String>,
10069 retention_seconds: Option<u64>,
10070 ) -> Result<()> {
10071 self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
10072 }
10073
10074 pub fn error_workflow_stream(
10076 &self,
10077 stream_name: impl Into<String>,
10078 error_reason: impl Into<String>,
10079 retention_seconds: Option<u64>,
10080 ) -> Result<()> {
10081 let error_reason = error_reason.into();
10082 if error_reason.is_empty() {
10083 return Err(Error::Codec(
10084 "workflow_stream_error_invalid: error reason must not be empty".to_string(),
10085 ));
10086 }
10087 self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
10088 }
10089
10090 fn finish_workflow_stream(
10091 &self,
10092 stream_name: String,
10093 error_reason: Option<String>,
10094 retention_seconds: Option<u64>,
10095 ) -> Result<()> {
10096 if stream_name.is_empty() {
10097 return Err(Error::Codec(
10098 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
10099 ));
10100 }
10101 if retention_seconds == Some(0) {
10102 return Err(Error::Codec(
10103 "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
10104 ));
10105 }
10106 let mut state = self
10107 .state
10108 .lock()
10109 .map_err(|_| Error::WorkflowStatePoisoned)?;
10110 let command_ordinal = state.workflow_stream_command_counter;
10111 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10112 state.workflow_stream_command_counter += 1;
10113 return match recorded {
10114 RecordedCommand::SideEffect { .. } => {
10115 state.command_cursor += 1;
10116 Ok(())
10117 }
10118 other => Err(command_mismatch(&other, "workflow stream close")),
10119 };
10120 }
10121 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
10122 state.workflow_stream_command_counter += 1;
10123 let mut directive = json!({
10124 "operation": if error_reason.is_some() { "error" } else { "close" },
10125 "stream_name": stream_name,
10126 "command_identity": identity,
10127 "command_ordinal": command_ordinal,
10128 });
10129 if let Some(error_reason) = error_reason {
10130 directive["error_reason"] = json!(error_reason);
10131 }
10132 if let Some(retention_seconds) = retention_seconds {
10133 directive["retention_seconds"] = json!(retention_seconds);
10134 }
10135 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
10136 state.commands.push(json!({
10137 "type": "record_side_effect",
10138 "result": result,
10139 "workflow_stream": directive,
10140 }));
10141 Ok(())
10142 }
10143
10144 fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
10145 let identity = state.workflow_command_identity.as_str();
10146 if identity.is_empty() {
10147 return Err(Error::MissingWorkflowCommandIdentity);
10148 }
10149 Ok(identity)
10150 }
10151
10152 pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
10158 update.validate()?;
10159 let (attributes, attribute_types) = update.into_wire_parts();
10160 let mut state = self
10161 .state
10162 .lock()
10163 .map_err(|_| Error::WorkflowStatePoisoned)?;
10164
10165 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10166 return match recorded {
10167 RecordedCommand::SearchAttributes {
10168 sequence,
10169 attributes: recorded_attributes,
10170 attribute_types: recorded_attribute_types,
10171 } => {
10172 if recorded_attributes != attributes {
10173 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10174 "search_attribute_value_mismatch",
10175 Some(sequence),
10176 Some(recorded_attributes.to_string()),
10177 Some(attributes.to_string()),
10178 "search-attribute values differ from the recorded durable command",
10179 )));
10180 }
10181 if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
10182 if recorded_types != attribute_types {
10183 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10184 "search_attribute_type_mismatch",
10185 Some(sequence),
10186 Some(json!(recorded_types).to_string()),
10187 Some(json!(attribute_types).to_string()),
10188 "search-attribute declared types differ from the recorded durable command",
10189 )));
10190 }
10191 }
10192 state.command_cursor += 1;
10193 Ok(())
10194 }
10195 other => Err(command_mismatch(&other, "search-attribute update")),
10196 };
10197 }
10198
10199 let mut command = serde_json::Map::from_iter([
10200 ("type".to_string(), json!("upsert_search_attributes")),
10201 ("attributes".to_string(), attributes),
10202 ]);
10203 if !attribute_types.is_empty() {
10204 command.insert("attribute_types".to_string(), json!(attribute_types));
10205 }
10206 state.commands.push(Value::Object(command));
10207 Ok(())
10208 }
10209
10210 pub fn uuid_v4(&self) -> Result<Uuid> {
10212 self.side_effect(Uuid::new_v4)
10213 }
10214
10215 pub fn get_version(
10225 &self,
10226 change_id: impl Into<String>,
10227 min_supported: i32,
10228 max_supported: i32,
10229 ) -> Result<i32> {
10230 let change_id = change_id.into();
10231 if change_id.trim().is_empty() {
10232 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10233 "version_change_id_invalid",
10234 None,
10235 Some("non-empty change ID".to_string()),
10236 Some(change_id),
10237 "version markers require a stable non-empty change ID",
10238 )));
10239 }
10240 if min_supported > max_supported {
10241 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10242 "version_range_invalid",
10243 None,
10244 Some("min_supported <= max_supported".to_string()),
10245 Some(format!("{min_supported}..={max_supported}")),
10246 "version marker supported range is invalid",
10247 )));
10248 }
10249
10250 let mut state = self
10251 .state
10252 .lock()
10253 .map_err(|_| Error::WorkflowStatePoisoned)?;
10254 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
10255 if let Some(RecordedCommand::VersionMarker {
10256 sequence: alias_sequence,
10257 change_id: alias_change_id,
10258 version: alias_version,
10259 }) = state.recorded_commands.get(state.command_cursor)
10260 {
10261 if alias_change_id == &change_id {
10262 if *alias_version != version {
10263 return Err(invalid_recorded_history(
10264 "version_marker_conflict",
10265 *alias_sequence,
10266 &format!("selected version {version} for change ID {change_id:?}"),
10267 &format!("recorded version {alias_version}"),
10268 "recorded marker conflicts with the decision already selected during replay",
10269 ));
10270 }
10271 ensure_version_supported(
10272 &change_id,
10273 version,
10274 min_supported,
10275 max_supported,
10276 *alias_sequence,
10277 )?;
10278 state.command_cursor += 1;
10281 return Ok(version);
10282 }
10283 }
10284 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
10285 return Ok(version);
10286 }
10287
10288 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10289 return match recorded {
10290 RecordedCommand::VersionMarker {
10291 sequence,
10292 change_id: recorded_change_id,
10293 version,
10294 ..
10295 } => {
10296 if recorded_change_id != change_id {
10297 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10298 "version_change_id_mismatch",
10299 Some(sequence),
10300 Some(recorded_change_id),
10301 Some(change_id),
10302 "recorded version marker change ID differs from current workflow code",
10303 )));
10304 }
10305 ensure_version_supported(
10306 &change_id,
10307 version,
10308 min_supported,
10309 max_supported,
10310 sequence,
10311 )?;
10312 state.command_cursor += 1;
10313 state.version_markers.insert(change_id, (version, sequence));
10314 Ok(version)
10315 }
10316 other => {
10317 let version = -1;
10318 let sequence = other.sequence();
10319 ensure_version_supported(
10320 &change_id,
10321 version,
10322 min_supported,
10323 max_supported,
10324 sequence,
10325 )?;
10326 state.version_markers.insert(change_id, (version, sequence));
10329 Ok(version)
10330 }
10331 };
10332 }
10333
10334 if state
10335 .history_events
10336 .iter()
10337 .any(|event| event.event_type == "WorkflowCompleted")
10338 {
10339 let version = -1;
10340 ensure_version_supported(&change_id, version, min_supported, max_supported, 0)?;
10341 state.version_markers.insert(change_id, (version, 0));
10342 return Ok(version);
10343 }
10344
10345 let version = max_supported;
10346 state.commands.push(json!({
10347 "type": "record_version_marker",
10348 "change_id": change_id,
10349 "version": version,
10350 "min_supported": min_supported,
10351 "max_supported": max_supported,
10352 }));
10353 state.version_markers.insert(change_id, (version, 0));
10356 Ok(version)
10357 }
10358
10359 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
10361 Ok(self.get_version(change_id, -1, 1)? == 1)
10362 }
10363
10364 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
10366 self.get_version(change_id, -1, 1).map(|_| ())
10367 }
10368
10369 pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
10374 let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
10375 let mut state = self
10376 .state
10377 .lock()
10378 .map_err(|_| Error::WorkflowStatePoisoned)?;
10379
10380 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10381 return match recorded {
10382 RecordedCommand::Memo {
10383 sequence,
10384 entries: recorded_entries,
10385 } => {
10386 if recorded_entries != entries {
10387 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10388 "memo_update_mismatch",
10389 Some(sequence),
10390 Some(format!("{recorded_entries:?}")),
10391 Some(format!("{entries:?}")),
10392 "recorded memo entries differ from the current workflow update",
10393 )));
10394 }
10395 state.command_cursor += 1;
10396 Ok(())
10397 }
10398 other => Err(command_mismatch(&other, "memo upsert")),
10399 };
10400 }
10401
10402 let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
10403 state.commands.push(json!({
10404 "type": "upsert_memo",
10405 "entries": entries_envelope,
10406 }));
10407 Ok(())
10408 }
10409
10410 pub fn start_child_workflow<T: Serialize>(
10437 &self,
10438 workflow_type: impl Into<String>,
10439 options: ChildWorkflowOptions,
10440 args: T,
10441 ) -> ChildWorkflowCall {
10442 ChildWorkflowCall {
10443 ctx: self.clone(),
10444 workflow_type: workflow_type.into(),
10445 options,
10446 args: Some(AvroValue::from_serialize(&args)),
10447 scheduled: false,
10448 matched_pending: false,
10449 parallel_group_path: Vec::new(),
10450 }
10451 }
10452
10453 pub async fn start_child_workflow_avro_value<T: Serialize>(
10454 &self,
10455 workflow_type: impl Into<String>,
10456 options: ChildWorkflowOptions,
10457 args: T,
10458 ) -> Result<ChildWorkflowAvroResult> {
10459 let mut call = self.start_child_workflow(workflow_type, options, args);
10460 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
10461 }
10462
10463 fn take_commands(&self) -> Result<Vec<Value>> {
10464 let mut state = self
10465 .state
10466 .lock()
10467 .map_err(|_| Error::WorkflowStatePoisoned)?;
10468 Ok(std::mem::take(&mut state.commands))
10469 }
10470
10471 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
10472 let mut state = self
10473 .state
10474 .lock()
10475 .map_err(|_| Error::WorkflowStatePoisoned)?;
10476
10477 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10478 return Err(command_mismatch(&recorded, "continue as new"));
10479 }
10480 if state.recorded_continue_as_new_sequence.is_some() {
10481 state.continue_as_new_consumed = true;
10482 return Ok(None);
10483 }
10484
10485 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
10486 let mut command = serde_json::Map::from_iter([
10487 ("type".to_string(), json!("continue_as_new")),
10488 ("arguments".to_string(), arguments),
10489 ("queue".to_string(), json!(state.task_queue.clone())),
10490 ]);
10491 if let Some(workflow_type) = request.options.workflow_type {
10492 command.insert("workflow_type".to_string(), json!(workflow_type));
10493 }
10494 if let Some(task_queue) = request.options.task_queue {
10495 command.insert("queue".to_string(), json!(task_queue));
10496 }
10497 Ok(Some(Value::Object(command)))
10498 }
10499
10500 fn matched_recorded_pending(&self) -> Result<bool> {
10501 let state = self
10502 .state
10503 .lock()
10504 .map_err(|_| Error::WorkflowStatePoisoned)?;
10505 Ok(state.matched_recorded_pending)
10506 }
10507
10508 fn ensure_history_consumed(&self) -> Result<()> {
10509 let state = self
10510 .state
10511 .lock()
10512 .map_err(|_| Error::WorkflowStatePoisoned)?;
10513 if let Some(replay) = &state.scope_delivery {
10514 if let Some((&sequence, _)) = replay
10515 .canonical
10516 .deliveries
10517 .iter()
10518 .find(|(sequence, _)| !replay.consumed.contains(sequence))
10519 {
10520 return Err(invalid_recorded_history(
10521 "cancellation_scope_delivery_unconsumed",
10522 sequence,
10523 "original scoped delivery",
10524 "workflow completion",
10525 "workflow completed without consuming its committed scope delivery",
10526 ));
10527 }
10528 }
10529 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
10530 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10531 "recorded_commands_unconsumed",
10532 Some(command.sequence()),
10533 Some(command.shape().to_string()),
10534 Some("workflow completion".to_string()),
10535 "workflow completed before consuming all recorded durable commands",
10536 )));
10537 }
10538 if let Some(delivery) = state
10539 .cancellation_history
10540 .delivery
10541 .as_ref()
10542 .filter(|_| !state.cancellation_consumed)
10543 {
10544 return Err(invalid_recorded_history(
10545 "cooperative_cancellation_unconsumed",
10546 delivery.sequence,
10547 "committed cancellation call",
10548 "workflow completion",
10549 "workflow completed without reaching its committed cancellation boundary",
10550 ));
10551 }
10552 if let Some(sequence) = state
10553 .recorded_continue_as_new_sequence
10554 .filter(|_| !state.continue_as_new_consumed)
10555 {
10556 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10557 "recorded_continue_as_new_unconsumed",
10558 Some(sequence),
10559 Some("continue as new".to_string()),
10560 Some("workflow completion".to_string()),
10561 "workflow completed without consuming its recorded continue-as-new transition",
10562 )));
10563 }
10564 Ok(())
10565 }
10566}
10567
10568fn contiguous_message_stream_count(
10569 pending: &[MessageStreamMessage],
10570 cursor: u64,
10571 max_items: usize,
10572) -> usize {
10573 pending
10574 .iter()
10575 .take(max_items)
10576 .enumerate()
10577 .take_while(|(offset, message)| {
10578 u64::try_from(*offset)
10579 .ok()
10580 .and_then(|offset| cursor.checked_add(offset + 1))
10581 == Some(message.position)
10582 })
10583 .count()
10584}
10585
10586fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
10587 matches!(
10588 event.event_type.as_str(),
10589 "ActivityScheduled"
10590 | "TimerScheduled"
10591 | "ChildWorkflowScheduled"
10592 | "SignalWaitOpened"
10593 | "ConditionWaitOpened"
10594 | "SearchAttributesUpserted"
10595 | "SideEffectRecorded"
10596 | "VersionMarkerRecorded"
10597 | "MemoUpserted"
10598 | "WorkflowContinuedAsNew"
10599 )
10600}
10601
10602#[derive(Debug)]
10603struct WorkflowState {
10604 workflow_id: Option<String>,
10605 run_id: Option<String>,
10606 task_queue: String,
10607 payload_codec: String,
10608 history_events: Arc<Vec<HistoryEvent>>,
10609 history_budget: WorkflowHistoryBudget,
10610 cancel_requested: bool,
10611 cancellation_history: CancellationHistory,
10612 cancellation_consumed: bool,
10613 cancellation_clock: Option<cancellation_replay_clock::ReplayClock>,
10614 cancellation_shield_depth: u64,
10615 cancellation_delivery_enabled: bool,
10616 cancellation_delivery_intent: Option<CancellationDelivery>,
10617 cancellation_delivery_command_count: usize,
10618 allow_cancellation_scope_authoring: bool,
10619 cancellation_scope_opening: Option<cancellation_scope::CancellationScopeOpening>,
10620 cancellation_scope_memberships: BTreeMap<u64, String>,
10621 scope_delivery: Option<cancellation_scope_replay::ScopeReplay>,
10622 resume_signal: Option<ResumeSignal>,
10623 recorded_commands: Vec<RecordedCommand>,
10624 selection_markers: Vec<SelectionMarker>,
10625 selection_marker_cursor: usize,
10626 cancelled_selection_members: Vec<SelectionCancellation>,
10627 recorded_continue_as_new_sequence: Option<u64>,
10628 continue_as_new_consumed: bool,
10629 command_cursor: usize,
10630 condition_wait_occurrence_counter: u64,
10631 matched_recorded_pending: bool,
10632 version_markers: HashMap<String, (i32, u64)>,
10633 workflow_command_identity: String,
10634 workflow_stream_command_counter: u64,
10635 commands: Vec<Value>,
10636 local_activity_requests: Vec<local_activity::Request>,
10637 message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
10638 message_stream_cursors: HashMap<String, u64>,
10639 message_stream_waits: HashMap<String, u64>,
10640}
10641
10642impl WorkflowState {
10643 #[cfg(test)]
10644 fn new(
10645 history: Vec<HistoryEvent>,
10646 task_queue: String,
10647 payload_codec: String,
10648 resume_signal: Option<ResumeSignal>,
10649 ) -> Result<Self> {
10650 Self::new_with_identity(
10651 history,
10652 None,
10653 None,
10654 task_queue,
10655 payload_codec,
10656 resume_signal,
10657 )
10658 }
10659
10660 fn new_with_identity(
10661 history: Vec<HistoryEvent>,
10662 workflow_id: Option<String>,
10663 run_id: Option<String>,
10664 task_queue: String,
10665 payload_codec: String,
10666 resume_signal: Option<ResumeSignal>,
10667 ) -> Result<Self> {
10668 Self::new_with_identity_and_scopes(
10669 history,
10670 workflow_id,
10671 run_id,
10672 task_queue,
10673 payload_codec,
10674 resume_signal,
10675 false,
10676 )
10677 }
10678
10679 fn new_with_identity_and_scopes(
10680 history: Vec<HistoryEvent>,
10681 workflow_id: Option<String>,
10682 run_id: Option<String>,
10683 task_queue: String,
10684 payload_codec: String,
10685 resume_signal: Option<ResumeSignal>,
10686 allow_cancellation_scope_authoring: bool,
10687 ) -> Result<Self> {
10688 Self::new_with_identity_and_scope_delivery(
10689 history,
10690 workflow_id,
10691 run_id,
10692 task_queue,
10693 payload_codec,
10694 resume_signal,
10695 allow_cancellation_scope_authoring,
10696 false,
10697 )
10698 }
10699
10700 fn new_with_identity_and_scope_delivery(
10701 history: Vec<HistoryEvent>,
10702 workflow_id: Option<String>,
10703 run_id: Option<String>,
10704 task_queue: String,
10705 payload_codec: String,
10706 resume_signal: Option<ResumeSignal>,
10707 allow_cancellation_scope_authoring: bool,
10708 allow_cancellation_scope_delivery: bool,
10709 ) -> Result<Self> {
10710 cooperative_cancellation::assert_cancellation_scope_replay_supported(
10711 &history,
10712 allow_cancellation_scope_authoring,
10713 allow_cancellation_scope_delivery,
10714 )?;
10715 let scopes = if allow_cancellation_scope_authoring {
10716 cancellation_scope::CancellationScopeHistory::read(
10717 &history,
10718 run_id.as_deref().unwrap_or_default(),
10719 )?
10720 } else {
10721 cancellation_scope::CancellationScopeHistory::default()
10722 };
10723 let cancellation_history = CancellationHistory::from_events(
10724 &history,
10725 run_id.as_deref().unwrap_or_default(),
10726 None,
10727 )?;
10728 if allow_cancellation_scope_authoring
10729 && cancellation_history.request.is_some()
10730 && !allow_cancellation_scope_delivery
10731 {
10732 return Err(Error::CancellationScopeExecutionUnavailable);
10733 }
10734 let scope_delivery = if allow_cancellation_scope_delivery {
10735 Some(cancellation_scope_replay::ScopeReplay::read(
10736 &history,
10737 &scopes,
10738 run_id.as_deref().unwrap_or_default(),
10739 workflow_id.as_deref().unwrap_or_default(),
10740 )?)
10741 } else {
10742 None
10743 };
10744 let mut recorded_commands = cancellation_history.bind_commands(recorded_commands(
10745 &history,
10746 &payload_codec,
10747 WorkflowIdentity {
10748 workflow_id: workflow_id.clone(),
10749 run_id: run_id.clone(),
10750 },
10751 )?)?;
10752 for (sequence, opening) in &scopes.openings {
10753 if recorded_commands
10754 .iter()
10755 .any(|command| command.sequence() == *sequence)
10756 {
10757 return Err(invalid_recorded_history(
10758 "invalid_cancellation_scope_history",
10759 *sequence,
10760 "distinct authored scope opening",
10761 "operation collision",
10762 "scope opening collides with a command",
10763 ));
10764 }
10765 recorded_commands.push(RecordedCommand::CancellationScope {
10766 sequence: *sequence,
10767 scope_id: opening.scope_id.clone(),
10768 parent_scope_id: opening.parent_scope_id.clone(),
10769 shield_parent: opening.shield_parent,
10770 });
10771 }
10772 recorded_commands.sort_by_key(RecordedCommand::sequence);
10773 if let Some(replay) = &scope_delivery {
10774 replay.bind_commands(&mut recorded_commands)?;
10775 }
10776 let selection_markers = recorded_selection_markers(&history)?;
10777 let cancelled_selection_members = recorded_selection_cancellations(&history)?;
10778 let recorded_continue_as_new = history
10779 .iter()
10780 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
10781 .collect::<Vec<_>>();
10782 if recorded_continue_as_new.len() > 1 {
10783 return Err(invalid_recorded_history(
10784 "duplicate_continue_as_new_transition",
10785 recorded_continue_as_new
10786 .last()
10787 .and_then(|event| durable_event_sequence(event))
10788 .unwrap_or(0),
10789 "one WorkflowContinuedAsNew event",
10790 &format!(
10791 "{} WorkflowContinuedAsNew events",
10792 recorded_continue_as_new.len()
10793 ),
10794 "workflow history records one continue-as-new transition more than once",
10795 ));
10796 }
10797 let recorded_continue_as_new_sequence = recorded_continue_as_new
10798 .first()
10799 .map(|event| {
10800 durable_event_sequence(event).ok_or_else(|| {
10801 Error::NonDeterministicReplay(ReplayFailure::new(
10802 "continue_as_new_sequence_missing",
10803 None,
10804 Some("recorded transition sequence".to_string()),
10805 Some("missing sequence".to_string()),
10806 "WorkflowContinuedAsNew history is missing its recorded sequence",
10807 ))
10808 })
10809 })
10810 .transpose()?;
10811 let mut message_stream_cursors = HashMap::new();
10812 for event in &history {
10813 if !matches!(
10814 event.event_type.as_str(),
10815 "SignalReceived" | "SignalApplied"
10816 ) || event.payload.get("signal_name").and_then(Value::as_str)
10817 != Some(MESSAGE_STREAM_SIGNAL)
10818 {
10819 continue;
10820 }
10821 let arguments = decode_signal_event_arguments(event, &payload_codec)?;
10822 if arguments.len() != 1 {
10823 continue;
10824 }
10825 let envelope = arguments[0].clone().into_json()?;
10826 let Some(envelope) = envelope.as_object() else {
10827 continue;
10828 };
10829 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
10830 {
10831 continue;
10832 }
10833 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
10834 continue;
10835 };
10836 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
10837 else {
10838 continue;
10839 };
10840 let cursor = message_stream_cursors
10841 .entry(stream_name.to_string())
10842 .or_insert(0);
10843 *cursor = (*cursor).max(through_position);
10844 }
10845 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
10846 let cancel_requested = history.iter().any(|event| {
10847 matches!(
10848 event.event_type.as_str(),
10849 "WorkflowCancellationRequested" | "WorkflowCancelRequested"
10850 )
10851 });
10852 let cancellation_clock = cancellation_history
10853 .request
10854 .as_ref()
10855 .and_then(|request| request.context.as_ref())
10856 .map(|_| {
10857 cancellation_replay_clock::ReplayClock::new(
10858 &history,
10859 &recorded_commands,
10860 cancellation_history.delivery_index,
10861 )
10862 })
10863 .or_else(|| {
10864 scope_delivery.as_ref().map(|_| {
10865 cancellation_replay_clock::ReplayClock::new(&history, &recorded_commands, None)
10866 })
10867 });
10868 Ok(Self {
10869 workflow_command_identity: String::new(),
10870 workflow_stream_command_counter: 0,
10871 workflow_id,
10872 run_id,
10873 task_queue,
10874 payload_codec,
10875 history_events: Arc::new(history),
10876 history_budget: WorkflowHistoryBudget {
10877 event_count,
10878 ..WorkflowHistoryBudget::default()
10879 },
10880 cancel_requested,
10881 cancellation_history,
10882 cancellation_consumed: false,
10883 cancellation_clock,
10884 cancellation_shield_depth: 0,
10885 cancellation_delivery_enabled: false,
10886 cancellation_delivery_intent: None,
10887 cancellation_delivery_command_count: 0,
10888 allow_cancellation_scope_authoring,
10889 cancellation_scope_opening: None,
10890 cancellation_scope_memberships: scopes.memberships,
10891 scope_delivery,
10892 resume_signal,
10893 recorded_commands,
10894 selection_markers,
10895 selection_marker_cursor: 0,
10896 cancelled_selection_members,
10897 recorded_continue_as_new_sequence,
10898 continue_as_new_consumed: false,
10899 command_cursor: 0,
10900 condition_wait_occurrence_counter: 0,
10901 matched_recorded_pending: false,
10902 version_markers: HashMap::new(),
10903 commands: Vec::new(),
10904 local_activity_requests: Vec::new(),
10905 message_stream_messages: HashMap::new(),
10906 message_stream_cursors,
10907 message_stream_waits: HashMap::new(),
10908 })
10909 }
10910}
10911
10912enum MessageStreamDelivery {
10913 Message(MessageStreamMessage),
10914 Cursor {
10915 stream_name: String,
10916 through_position: u64,
10917 },
10918}
10919
10920fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
10921 if arguments.len() != 1 {
10922 return Ok(None);
10923 }
10924 let envelope = arguments
10925 .into_iter()
10926 .next()
10927 .expect("one argument was checked");
10928 let Some(envelope) = envelope.as_object() else {
10929 return Ok(None);
10930 };
10931 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
10932 return Ok(None);
10933 };
10934 if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
10935 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
10936 else {
10937 return Ok(None);
10938 };
10939 return Ok(Some(MessageStreamDelivery::Cursor {
10940 stream_name: stream_name.to_string(),
10941 through_position,
10942 }));
10943 }
10944 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
10945 return Ok(None);
10946 }
10947 let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
10948 return Ok(None);
10949 };
10950 let Some(position) = envelope
10951 .get("position")
10952 .and_then(Value::as_u64)
10953 .filter(|value| *value > 0)
10954 else {
10955 return Ok(None);
10956 };
10957 let Some(payload_envelope) = envelope.get("payload_envelope") else {
10958 return Ok(None);
10959 };
10960 let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
10961 else {
10962 return Ok(None);
10963 };
10964 let decoded = decode_avro_value(&payload_envelope)?;
10965 let AvroValue::Array(values) = decoded else {
10966 return Ok(None);
10967 };
10968 Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
10969 stream_name: stream_name.to_string(),
10970 message_id: message_id.to_string(),
10971 position,
10972 arguments: values,
10973 })))
10974}
10975
10976#[derive(Clone, Debug)]
10977enum RecordedCommand {
10978 CancellationScope {
10979 sequence: u64,
10980 scope_id: String,
10981 parent_scope_id: String,
10982 shield_parent: bool,
10983 },
10984 CancellationGroup {
10985 sequence: u64,
10986 span: u64,
10987 original: Vec<RecordedCommand>,
10988 },
10989 CancellationBoundary {
10990 sequence: u64,
10991 call_kind: CancellationCallKind,
10992 original: Option<Box<RecordedCommand>>,
10993 },
10994 Activity {
10995 sequence: u64,
10996 activity_type: Option<String>,
10997 cancellation_policy: String,
10998 options: Option<RecordedActivityOptions>,
10999 outcome: Option<ActivityOutcome>,
11000 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
11001 },
11002 Timer {
11003 sequence: u64,
11004 delay_seconds: u64,
11005 fired: bool,
11006 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
11007 },
11008 ChildWorkflow {
11009 sequence: u64,
11010 workflow_type: Option<String>,
11011 policies: RecordedChildPolicies,
11012 outcome: Option<ChildWorkflowOutcome>,
11013 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
11014 },
11015 SignalWait {
11016 sequence: u64,
11017 signal_name: String,
11018 value: Option<Vec<AvroValue>>,
11019 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
11020 },
11021 ConditionWait {
11022 sequence: u64,
11023 occurrence_id: String,
11024 condition_key: Option<String>,
11025 predicate_identity: String,
11026 timeout_seconds: Option<u64>,
11027 result: Option<ConditionWaitResult>,
11028 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
11029 },
11030 SearchAttributes {
11031 sequence: u64,
11032 attributes: Value,
11033 attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
11034 },
11035 SideEffect {
11036 sequence: u64,
11037 value: AvroValue,
11038 },
11039 VersionMarker {
11040 sequence: u64,
11041 change_id: String,
11042 version: i32,
11043 },
11044 Memo {
11045 sequence: u64,
11046 entries: AvroValue,
11047 },
11048}
11049
11050#[derive(Clone, Debug, PartialEq, Eq)]
11051struct SelectionMarker {
11052 selection_group_id: String,
11053 selection_group_base_sequence: u64,
11054 selection_group_size: usize,
11055 member_key: SelectionKey,
11056 member_index: usize,
11057 member_base_sequence: u64,
11058 member_size: usize,
11059 operation_kind: String,
11060 operation_identity: String,
11061 outcome: String,
11062 resolution_event_id: String,
11063 resolution_event_type: String,
11064}
11065
11066#[derive(Clone, Debug, PartialEq, Eq)]
11067struct SelectionCancellation {
11068 selection_group_id: String,
11069 member_key: SelectionKey,
11070 member_index: usize,
11071 member_base_sequence: u64,
11072 member_size: usize,
11073 operation_kind: String,
11074 operation_identity: String,
11075}
11076
11077fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
11078 let mut markers: Vec<SelectionMarker> = Vec::new();
11079 for event in events
11080 .iter()
11081 .filter(|event| event.event_type == "SelectionResolved")
11082 {
11083 let payload = &event.payload;
11084 let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
11085 let group_size = required_selection_usize(payload, "selection_group_size")?;
11086 let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
11087 let member_size = required_selection_usize(payload, "member_size")?;
11088 let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
11089 let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
11090 invalid_recorded_history(
11091 "selection_marker_invalid",
11092 base_sequence,
11093 "non-empty selection_group_id",
11094 &payload.to_string(),
11095 "selection winner history is missing its durable group identity",
11096 )
11097 })?;
11098 let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
11099 if group_id != expected_group_id {
11100 return Err(invalid_recorded_history(
11101 "selection_marker_invalid",
11102 base_sequence,
11103 &expected_group_id,
11104 &group_id,
11105 "selection winner history contains an incompatible group identity",
11106 ));
11107 }
11108 let group_end = base_sequence
11109 .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
11110 .unwrap_or(u64::MAX);
11111 let member_end = member_base_sequence
11112 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
11113 .unwrap_or(u64::MAX);
11114 if member_index >= group_size
11115 || member_base_sequence < base_sequence
11116 || member_end > group_end
11117 {
11118 return Err(invalid_recorded_history(
11119 "selection_marker_invalid",
11120 base_sequence,
11121 "winner member within selection group bounds",
11122 &payload.to_string(),
11123 "selection winner history contains an invalid member range",
11124 ));
11125 }
11126 let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
11127 invalid_recorded_history(
11128 "selection_marker_invalid",
11129 base_sequence,
11130 "selection operation kind",
11131 &payload.to_string(),
11132 "selection winner history is missing its operation kind",
11133 )
11134 })?;
11135 if !matches!(
11136 operation_kind.as_str(),
11137 "activity" | "child" | "timer" | "signal" | "condition" | "group"
11138 ) {
11139 return Err(invalid_recorded_history(
11140 "selection_marker_invalid",
11141 base_sequence,
11142 "activity, child, timer, signal, condition, or group",
11143 &operation_kind,
11144 "selection winner history contains an unsupported operation kind",
11145 ));
11146 }
11147 let operation_identity =
11148 payload_string(payload, "operation_identity").ok_or_else(|| {
11149 invalid_recorded_history(
11150 "selection_marker_invalid",
11151 base_sequence,
11152 "non-empty operation identity",
11153 &payload.to_string(),
11154 "selection winner history is missing its durable operation identity",
11155 )
11156 })?;
11157 let outcome = payload_string(payload, "outcome").ok_or_else(|| {
11158 invalid_recorded_history(
11159 "selection_marker_invalid",
11160 base_sequence,
11161 "completed or failed selection outcome",
11162 &payload.to_string(),
11163 "selection winner history is missing its outcome",
11164 )
11165 })?;
11166 if !matches!(outcome.as_str(), "completed" | "failed") {
11167 return Err(invalid_recorded_history(
11168 "selection_marker_invalid",
11169 base_sequence,
11170 "completed or failed selection outcome",
11171 &outcome,
11172 "selection winner history contains an unsupported outcome",
11173 ));
11174 }
11175 let marker = SelectionMarker {
11176 selection_group_id: group_id,
11177 selection_group_base_sequence: base_sequence,
11178 selection_group_size: group_size,
11179 member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
11180 member_index,
11181 member_base_sequence,
11182 member_size,
11183 operation_kind,
11184 operation_identity,
11185 outcome,
11186 resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
11187 || {
11188 invalid_recorded_history(
11189 "selection_marker_invalid",
11190 base_sequence,
11191 "durable resolution_event_id",
11192 &payload.to_string(),
11193 "selection winner history is missing its terminal event identity",
11194 )
11195 },
11196 )?,
11197 resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
11198 || {
11199 invalid_recorded_history(
11200 "selection_marker_invalid",
11201 base_sequence,
11202 "durable resolution_event_type",
11203 &payload.to_string(),
11204 "selection winner history is missing its terminal event type",
11205 )
11206 },
11207 )?,
11208 };
11209 if let Some(existing) = markers
11210 .iter()
11211 .find(|existing| existing.selection_group_id == marker.selection_group_id)
11212 {
11213 if existing != &marker {
11214 return Err(invalid_recorded_history(
11215 "selection_marker_conflict",
11216 base_sequence,
11217 &format!("one winner for {}", marker.selection_group_id),
11218 &payload.to_string(),
11219 "selection history records conflicting winners for one durable group",
11220 ));
11221 }
11222 continue;
11223 }
11224 markers.push(marker);
11225 }
11226 Ok(markers)
11227}
11228
11229fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
11230 let mut cancelled: Vec<SelectionCancellation> = Vec::new();
11231 for event in events
11232 .iter()
11233 .filter(|event| event.event_type == "SelectionOperationCancelled")
11234 {
11235 let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
11236 invalid_recorded_history(
11237 "selection_cancellation_invalid",
11238 0,
11239 "non-empty selection_group_id",
11240 &event.payload.to_string(),
11241 "selection cancellation history is missing its group identity",
11242 )
11243 })?;
11244 let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
11245 let marker = SelectionCancellation {
11246 selection_group_id: group_id,
11247 member_key: selection_key_from_value(
11248 event.payload.get("member_key"),
11249 member_base_sequence,
11250 )?,
11251 member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
11252 member_base_sequence,
11253 member_size: required_selection_usize(&event.payload, "member_size")?,
11254 operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
11255 invalid_recorded_history(
11256 "selection_cancellation_invalid",
11257 member_base_sequence,
11258 "selection operation kind",
11259 &event.payload.to_string(),
11260 "selection cancellation is missing its operation kind",
11261 )
11262 })?,
11263 operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
11264 || {
11265 invalid_recorded_history(
11266 "selection_cancellation_invalid",
11267 member_base_sequence,
11268 "selection operation identity",
11269 &event.payload.to_string(),
11270 "selection cancellation is missing its operation identity",
11271 )
11272 },
11273 )?,
11274 };
11275 if let Some(existing) = cancelled.iter().find(|recorded| {
11276 recorded.selection_group_id == marker.selection_group_id
11277 && recorded.member_base_sequence == marker.member_base_sequence
11278 }) {
11279 if existing != &marker {
11280 return Err(invalid_recorded_history(
11281 "selection_cancellation_conflict",
11282 member_base_sequence,
11283 "one stable SelectionOperationCancelled marker",
11284 &event.payload.to_string(),
11285 "selection cancellation history contains conflicting member metadata",
11286 ));
11287 }
11288 } else {
11289 cancelled.push(marker);
11290 }
11291 }
11292 Ok(cancelled)
11293}
11294
11295fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
11296 payload
11297 .get(field)
11298 .and_then(value_as_u64)
11299 .filter(|value| *value > 0)
11300 .ok_or_else(|| {
11301 invalid_recorded_history(
11302 "selection_marker_invalid",
11303 0,
11304 &format!("positive integer {field}"),
11305 &payload.to_string(),
11306 "selection history contains invalid durable identity metadata",
11307 )
11308 })
11309}
11310
11311fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
11312 required_selection_usize_allow_zero(payload, field).and_then(|value| {
11313 if value > 0 {
11314 Ok(value)
11315 } else {
11316 Err(invalid_recorded_history(
11317 "selection_marker_invalid",
11318 0,
11319 &format!("positive integer {field}"),
11320 &payload.to_string(),
11321 "selection history contains invalid durable identity metadata",
11322 ))
11323 }
11324 })
11325}
11326
11327fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
11328 payload
11329 .get(field)
11330 .and_then(value_as_u64)
11331 .and_then(|value| usize::try_from(value).ok())
11332 .ok_or_else(|| {
11333 invalid_recorded_history(
11334 "selection_marker_invalid",
11335 0,
11336 &format!("non-negative integer {field}"),
11337 &payload.to_string(),
11338 "selection history contains invalid durable identity metadata",
11339 )
11340 })
11341}
11342
11343#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
11344struct RecordedActivityOptions {
11345 task_queue: RecordedSnapshotValue<Option<String>>,
11346 execution_mode: RecordedSnapshotValue<Option<String>>,
11347 retry_policy: ActivityRetrySnapshot,
11348 worker_session: Option<Value>,
11349}
11350
11351#[derive(Clone, Debug)]
11352struct RecordedChildPolicies {
11353 parent_close_policy: String,
11354 cancellation_policy: String,
11355}
11356
11357fn recorded_activity_cancellation_policy(
11358 events: &[&HistoryEvent],
11359 sequence: u64,
11360) -> Result<String> {
11361 let mut policy: Option<String> = None;
11362 for event in events.iter().filter(|event| {
11363 matches!(
11364 event.event_type.as_str(),
11365 "ActivityScheduled"
11366 | "ActivityStarted"
11367 | "ActivityCompleted"
11368 | "ActivityFailed"
11369 | "ActivityTimedOut"
11370 | "ActivityCancelled"
11371 )
11372 }) {
11373 for source in [Some(&event.payload), event.payload.get("activity")]
11374 .into_iter()
11375 .flatten()
11376 {
11377 let Some(value) = source.get("cancellation_policy") else {
11378 continue;
11379 };
11380 let Some(incoming @ ("try_cancel" | "wait_cancellation_completed" | "abandon")) =
11381 value.as_str()
11382 else {
11383 return Err(invalid_recorded_history(
11384 "invalid_activity_cancellation_policy_history",
11385 sequence,
11386 "supported Activity cancellation policy",
11387 &value.to_string(),
11388 "Activity history contains an invalid cancellation policy",
11389 ));
11390 };
11391 if let Some(previous) = policy.as_deref() {
11392 if previous != incoming {
11393 return Err(invalid_recorded_history(
11394 "activity_cancellation_policy_history_conflict",
11395 sequence,
11396 previous,
11397 incoming,
11398 "Activity cancellation policy changed between history events",
11399 ));
11400 }
11401 }
11402 policy = Some(incoming.to_string());
11403 }
11404 policy.get_or_insert_with(|| "try_cancel".to_string());
11405 }
11406 Ok(policy.unwrap_or_else(|| "try_cancel".to_string()))
11407}
11408
11409fn recorded_child_policy_value<'a>(
11410 payload: &'a Value,
11411 field: &str,
11412 sequence: u64,
11413) -> Result<Option<&'a str>> {
11414 let Some(value) = payload.get(field).filter(|value| !value.is_null()) else {
11415 return Ok(None);
11416 };
11417 let valid = match (field, value.as_str()) {
11418 (
11419 "parent_close_policy",
11420 Some("abandon" | "request_cancel" | "request_cancellation" | "terminate"),
11421 ) => true,
11422 ("cancellation_policy", Some("abandon" | "try_cancel" | "wait_cancellation_completed")) => {
11423 true
11424 }
11425 _ => false,
11426 };
11427 if !valid {
11428 return Err(invalid_recorded_history(
11429 "invalid_child_workflow_policy_history",
11430 sequence,
11431 "supported child workflow policy",
11432 &value.to_string(),
11433 "child workflow history contains an invalid policy",
11434 ));
11435 }
11436 Ok(value.as_str())
11437}
11438
11439fn recorded_child_policies(
11440 events: &[&HistoryEvent],
11441 scheduled: &HistoryEvent,
11442 sequence: u64,
11443) -> Result<RecordedChildPolicies> {
11444 let parent_close_policy =
11445 recorded_child_policy_value(&scheduled.payload, "parent_close_policy", sequence)?
11446 .unwrap_or("abandon")
11447 .to_string();
11448 let cancellation_policy =
11449 recorded_child_policy_value(&scheduled.payload, "cancellation_policy", sequence)?
11450 .unwrap_or("abandon")
11451 .to_string();
11452 for event in events {
11453 for (field, expected) in [
11454 ("parent_close_policy", parent_close_policy.as_str()),
11455 ("cancellation_policy", cancellation_policy.as_str()),
11456 ] {
11457 if let Some(actual) = recorded_child_policy_value(&event.payload, field, sequence)? {
11458 if actual != expected {
11459 return Err(invalid_recorded_history(
11460 "child_workflow_policy_history_conflict",
11461 sequence,
11462 expected,
11463 actual,
11464 "child workflow policy changed between history events",
11465 ));
11466 }
11467 }
11468 }
11469 }
11470 Ok(RecordedChildPolicies {
11471 parent_close_policy,
11472 cancellation_policy,
11473 })
11474}
11475
11476fn ensure_child_policies_match(
11477 sequence: u64,
11478 recorded: &RecordedChildPolicies,
11479 current: &ChildWorkflowOptions,
11480) -> Result<()> {
11481 for (field, expected, actual) in [
11482 (
11483 "parent_close_policy",
11484 recorded.parent_close_policy.as_str(),
11485 current.parent_close_policy.as_str(),
11486 ),
11487 (
11488 "cancellation_policy",
11489 recorded.cancellation_policy.as_str(),
11490 current.cancellation_policy.as_str(),
11491 ),
11492 ] {
11493 if expected != actual {
11494 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11495 "child_workflow_policy_changed",
11496 Some(sequence),
11497 Some(expected.to_string()),
11498 Some(actual.to_string()),
11499 format!("child workflow {field} changed during replay"),
11500 )));
11501 }
11502 }
11503 Ok(())
11504}
11505
11506#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
11507enum RecordedSnapshotValue<T> {
11508 Unknown,
11510 Known(T),
11511}
11512
11513impl<T: PartialEq> RecordedSnapshotValue<T> {
11514 fn matches_current(&self, current: &Self) -> bool {
11515 match self {
11516 Self::Unknown => true,
11517 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
11518 }
11519 }
11520}
11521
11522#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
11523struct ActivityRetrySnapshot {
11524 snapshot_version: RecordedSnapshotValue<Option<u64>>,
11525 max_attempts: RecordedSnapshotValue<Option<u64>>,
11526 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
11527 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
11528 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
11529 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
11530 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
11531 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
11532}
11533
11534impl ActivityRetrySnapshot {
11535 fn matches_current(&self, current: &Self) -> bool {
11536 self.snapshot_version
11537 .matches_current(¤t.snapshot_version)
11538 && self.max_attempts.matches_current(¤t.max_attempts)
11539 && self
11540 .backoff_seconds
11541 .matches_current(¤t.backoff_seconds)
11542 && self
11543 .start_to_close_timeout
11544 .matches_current(¤t.start_to_close_timeout)
11545 && self
11546 .schedule_to_start_timeout
11547 .matches_current(¤t.schedule_to_start_timeout)
11548 && self
11549 .schedule_to_close_timeout
11550 .matches_current(¤t.schedule_to_close_timeout)
11551 && self
11552 .heartbeat_timeout
11553 .matches_current(¤t.heartbeat_timeout)
11554 && self
11555 .non_retryable_error_types
11556 .matches_current(¤t.non_retryable_error_types)
11557 }
11558}
11559
11560fn recorded_optional_u64(
11561 object: Option<&serde_json::Map<String, Value>>,
11562 field: &str,
11563) -> RecordedSnapshotValue<Option<u64>> {
11564 match object.and_then(|object| object.get(field)) {
11565 None => RecordedSnapshotValue::Unknown,
11566 Some(Value::Null) => RecordedSnapshotValue::Known(None),
11567 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
11568 }
11569}
11570
11571fn recorded_optional_string(
11572 object: &serde_json::Map<String, Value>,
11573 field: &str,
11574) -> RecordedSnapshotValue<Option<String>> {
11575 match object.get(field) {
11576 None => RecordedSnapshotValue::Unknown,
11577 Some(Value::Null) => RecordedSnapshotValue::Known(None),
11578 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
11579 }
11580}
11581
11582fn recorded_activity_execution_mode(
11583 events: &[&HistoryEvent],
11584 sequence: u64,
11585) -> Result<RecordedSnapshotValue<Option<String>>> {
11586 let mut recorded = RecordedSnapshotValue::Unknown;
11587 for event in events {
11588 for object in [
11589 event.payload.as_object(),
11590 event.payload.get("activity").and_then(Value::as_object),
11591 ]
11592 .into_iter()
11593 .flatten()
11594 {
11595 let mode = match object.get("execution_mode") {
11596 None => None,
11597 Some(Value::Null) => Some(None),
11598 Some(Value::String(mode)) if !mode.is_empty() => Some(Some(mode.clone())),
11599 Some(_) => {
11600 return Err(invalid_recorded_history(
11601 "activity_execution_mode_invalid",
11602 sequence,
11603 "null or a non-empty execution mode",
11604 "invalid execution_mode",
11605 "activity history contains an invalid execution mode",
11606 ))
11607 }
11608 };
11609 let legacy_mode = match object.get("local_activity") {
11610 None => None,
11611 Some(Value::Bool(local)) => Some(local.then(|| "local".to_string())),
11612 Some(_) => {
11613 return Err(invalid_recorded_history(
11614 "activity_execution_mode_invalid",
11615 sequence,
11616 "boolean local_activity",
11617 "invalid local_activity",
11618 "activity history contains an invalid local-activity marker",
11619 ))
11620 }
11621 };
11622 for mode in [mode, legacy_mode].into_iter().flatten() {
11623 if matches!(&recorded, RecordedSnapshotValue::Known(previous) if previous != &mode)
11624 {
11625 return Err(invalid_recorded_history(
11626 "activity_execution_mode_conflict", sequence, "one activity execution mode", "conflicting execution modes",
11627 "activity lifecycle events at one workflow sequence disagree on execution mode",
11628 ));
11629 }
11630 recorded = RecordedSnapshotValue::Known(mode);
11631 }
11632 }
11633 }
11634 Ok(recorded)
11635}
11636
11637fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
11638 let policy = policy.and_then(Value::as_object);
11639 let backoff_seconds = policy
11640 .and_then(|policy| policy.get("backoff_seconds"))
11641 .and_then(Value::as_array)
11642 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
11643 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
11644 let mut non_retryable_error_types = Vec::new();
11645 for error_type in policy
11646 .and_then(|policy| policy.get("non_retryable_error_types"))
11647 .and_then(Value::as_array)
11648 .into_iter()
11649 .flatten()
11650 .filter_map(Value::as_str)
11651 .map(str::trim)
11652 .filter(|error_type| !error_type.is_empty())
11653 {
11654 if !non_retryable_error_types
11655 .iter()
11656 .any(|recorded| recorded == error_type)
11657 {
11658 non_retryable_error_types.push(error_type.to_string());
11659 }
11660 }
11661
11662 ActivityRetrySnapshot {
11663 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
11664 max_attempts: recorded_optional_u64(policy, "max_attempts"),
11665 backoff_seconds,
11666 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
11667 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
11668 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
11669 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
11670 non_retryable_error_types: if policy
11671 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
11672 {
11673 RecordedSnapshotValue::Known(non_retryable_error_types)
11674 } else {
11675 RecordedSnapshotValue::Unknown
11676 },
11677 }
11678}
11679
11680fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
11681 let policy = options.retry_policy.as_ref();
11682 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
11683 Some(Value::Null) => None,
11684 Some(value) => value_as_u64(value),
11685 None => Some(1),
11686 };
11687 let backoff_seconds = policy
11688 .and_then(|policy| policy.get("backoff_seconds"))
11689 .and_then(Value::as_array)
11690 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
11691 .unwrap_or_default();
11692 let non_retryable_error_types = policy
11693 .and_then(|policy| policy.get("non_retryable_error_types"))
11694 .and_then(Value::as_array)
11695 .into_iter()
11696 .flatten()
11697 .filter_map(Value::as_str)
11698 .map(str::to_string)
11699 .collect();
11700
11701 ActivityRetrySnapshot {
11702 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
11703 max_attempts: RecordedSnapshotValue::Known(max_attempts),
11704 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
11705 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
11706 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
11707 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
11708 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
11709 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
11710 }
11711}
11712
11713fn activity_options_description(options: &RecordedActivityOptions) -> String {
11714 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
11715}
11716
11717impl RecordedCommand {
11718 fn sequence(&self) -> u64 {
11719 match self {
11720 Self::CancellationGroup { sequence, .. }
11721 | Self::CancellationScope { sequence, .. }
11722 | Self::CancellationBoundary { sequence, .. }
11723 | Self::Activity { sequence, .. }
11724 | Self::Timer { sequence, .. }
11725 | Self::ChildWorkflow { sequence, .. }
11726 | Self::SignalWait { sequence, .. }
11727 | Self::ConditionWait { sequence, .. }
11728 | Self::SearchAttributes { sequence, .. }
11729 | Self::SideEffect { sequence, .. }
11730 | Self::VersionMarker { sequence, .. }
11731 | Self::Memo { sequence, .. } => *sequence,
11732 }
11733 }
11734
11735 fn shape(&self) -> &'static str {
11736 match self {
11737 Self::CancellationScope { .. } => "cancellation scope",
11738 Self::CancellationBoundary { .. } | Self::CancellationGroup { .. } => {
11739 "cooperative cancellation"
11740 }
11741 Self::Activity { .. } => "activity",
11742 Self::Timer { .. } => "timer",
11743 Self::ChildWorkflow { .. } => "child workflow",
11744 Self::SignalWait { .. } => "signal wait",
11745 Self::ConditionWait { .. } => "condition wait",
11746 Self::SearchAttributes { .. } => "search-attribute update",
11747 Self::SideEffect { .. } => "side effect",
11748 Self::VersionMarker { .. } => "version marker",
11749 Self::Memo { .. } => "memo upsert",
11750 }
11751 }
11752}
11753
11754fn ensure_version_supported(
11755 change_id: &str,
11756 version: i32,
11757 min_supported: i32,
11758 max_supported: i32,
11759 sequence: u64,
11760) -> Result<()> {
11761 if (min_supported..=max_supported).contains(&version) {
11762 return Ok(());
11763 }
11764 Err(Error::NonDeterministicReplay(ReplayFailure::new(
11765 "version_marker_incompatible_range",
11766 (sequence != 0).then_some(sequence),
11767 Some(format!("{min_supported}..={max_supported}")),
11768 Some(format!("{change_id}:{version}")),
11769 "recorded workflow version is outside the range supported by current code",
11770 )))
11771}
11772
11773#[derive(Clone, Debug)]
11774struct ResumeSignal {
11775 signal_name: String,
11776 arguments: Vec<AvroValue>,
11777}
11778
11779const MAX_PARALLEL_OPERATIONS: usize = 1000;
11780
11781fn parallel_group_prefix(kind: &str) -> &'static str {
11782 match kind {
11783 "activity" => "parallel-activities",
11784 "child" => "parallel-children",
11785 "timer" => "parallel-timers",
11786 _ => "parallel-calls",
11787 }
11788}
11789
11790fn parallel_group_entry(
11791 base_sequence: u64,
11792 size: usize,
11793 index: usize,
11794 kind: &str,
11795) -> ParallelGroupMetadata {
11796 ParallelGroupMetadata {
11797 parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
11798 parallel_group_kind: kind.to_string(),
11799 parallel_group_base_sequence: base_sequence,
11800 parallel_group_size: size,
11801 parallel_group_index: index,
11802 parallel_group_mode: None,
11803 selection_member_key: None,
11804 selection_member_index: None,
11805 selection_member_base_sequence: None,
11806 selection_member_size: None,
11807 selection_member_kind: None,
11808 }
11809}
11810
11811struct SelectionMemberMetadata {
11812 key: SelectionKey,
11813 index: usize,
11814 base_sequence: u64,
11815 size: usize,
11816 kind: String,
11817}
11818
11819fn selection_group_entry(
11820 base_sequence: u64,
11821 size: usize,
11822 index: usize,
11823 kind: &str,
11824 member: &SelectionMemberMetadata,
11825) -> ParallelGroupMetadata {
11826 ParallelGroupMetadata {
11827 parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
11828 parallel_group_kind: kind.to_string(),
11829 parallel_group_base_sequence: base_sequence,
11830 parallel_group_size: size,
11831 parallel_group_index: index,
11832 parallel_group_mode: Some("select".to_string()),
11833 selection_member_key: Some(member.key.clone()),
11834 selection_member_index: Some(member.index),
11835 selection_member_base_sequence: Some(member.base_sequence),
11836 selection_member_size: Some(member.size),
11837 selection_member_kind: Some(member.kind.clone()),
11838 }
11839}
11840
11841fn apply_parallel_group_path(
11842 command: &mut serde_json::Map<String, Value>,
11843 path: &[ParallelGroupMetadata],
11844) {
11845 let Some(inner) = path.last() else {
11846 return;
11847 };
11848 command.insert(
11849 "parallel_group_id".to_string(),
11850 json!(inner.parallel_group_id),
11851 );
11852 command.insert(
11853 "parallel_group_kind".to_string(),
11854 json!(inner.parallel_group_kind),
11855 );
11856 command.insert(
11857 "parallel_group_base_sequence".to_string(),
11858 json!(inner.parallel_group_base_sequence),
11859 );
11860 command.insert(
11861 "parallel_group_size".to_string(),
11862 json!(inner.parallel_group_size),
11863 );
11864 command.insert(
11865 "parallel_group_index".to_string(),
11866 json!(inner.parallel_group_index),
11867 );
11868 if let Some(mode) = &inner.parallel_group_mode {
11869 command.insert("parallel_group_mode".to_string(), json!(mode));
11870 }
11871 if let Some(key) = &inner.selection_member_key {
11872 command.insert("selection_member_key".to_string(), json!(key));
11873 }
11874 if let Some(index) = inner.selection_member_index {
11875 command.insert("selection_member_index".to_string(), json!(index));
11876 }
11877 if let Some(base_sequence) = inner.selection_member_base_sequence {
11878 command.insert(
11879 "selection_member_base_sequence".to_string(),
11880 json!(base_sequence),
11881 );
11882 }
11883 if let Some(size) = inner.selection_member_size {
11884 command.insert("selection_member_size".to_string(), json!(size));
11885 }
11886 if let Some(kind) = &inner.selection_member_kind {
11887 command.insert("selection_member_kind".to_string(), json!(kind));
11888 }
11889 command.insert("parallel_group_path".to_string(), json!(path));
11890}
11891
11892fn ensure_parallel_path_matches(
11893 sequence: u64,
11894 recorded: Option<&[ParallelGroupMetadata]>,
11895 expected: &[ParallelGroupMetadata],
11896) -> Result<()> {
11897 match (recorded, expected.is_empty()) {
11898 (None, true) => Ok(()),
11899 (Some(recorded), false) if recorded == expected => Ok(()),
11900 (None, false) => Err(invalid_recorded_history(
11901 "parallel_group_metadata_missing",
11902 sequence,
11903 &serde_json::to_string(expected).unwrap_or_default(),
11904 "<missing>",
11905 "recorded parallel member is missing its durable group path",
11906 )),
11907 (Some(recorded), true) => Err(invalid_recorded_history(
11908 "parallel_group_shape_mismatch",
11909 sequence,
11910 "sequential command",
11911 &serde_json::to_string(recorded).unwrap_or_default(),
11912 "recorded command belonged to a parallel group but current code schedules it sequentially",
11913 )),
11914 (Some(recorded), false) => Err(invalid_recorded_history(
11915 "parallel_group_shape_mismatch",
11916 sequence,
11917 &serde_json::to_string(recorded).unwrap_or_default(),
11918 &serde_json::to_string(expected).unwrap_or_default(),
11919 "recorded parallel-group identity or path changed during replay",
11920 )),
11921 }
11922}
11923
11924#[derive(Clone, Debug)]
11925enum ParallelShape {
11926 Leaf,
11927 Group(Vec<ParallelShape>),
11928}
11929
11930struct ParallelDescriptor {
11931 operation: ParallelOperation,
11932 offset: usize,
11933 member_path: Vec<usize>,
11934 group_path: Vec<ParallelGroupMetadata>,
11935}
11936
11937fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
11938 operations
11939 .iter()
11940 .map(|operation| match operation {
11941 ParallelOperation::Group(children) => parallel_leaf_count(children),
11942 _ => 1,
11943 })
11944 .sum()
11945}
11946
11947fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
11948 match operation {
11949 ParallelOperation::Activity { .. } => Some("activity"),
11950 ParallelOperation::ChildWorkflow { .. } => Some("child"),
11951 ParallelOperation::Timer(_) => Some("timer"),
11952 ParallelOperation::Signal(_) => Some("signal"),
11953 ParallelOperation::Condition { .. } => Some("condition"),
11954 ParallelOperation::Group(children) => parallel_group_kind(children),
11955 }
11956}
11957
11958fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
11959 let mut kind = None;
11960 for operation in operations {
11961 let Some(operation_kind) = parallel_operation_kind(operation) else {
11962 continue;
11963 };
11964 match kind {
11965 None => kind = Some(operation_kind),
11966 Some(current) if current == operation_kind => {}
11967 Some(_) => return Some("mixed"),
11968 }
11969 }
11970 kind
11971}
11972
11973fn validate_parallel_operations(
11974 operations: &[ParallelOperation],
11975 member_path: &mut Vec<usize>,
11976 root: bool,
11977) -> Result<()> {
11978 let leaves = parallel_leaf_count(operations);
11979 if leaves > MAX_PARALLEL_OPERATIONS {
11980 return Err(Error::InvalidParallelGroup(ParallelGroupError {
11981 reason: "fan_out_limit_exceeded",
11982 member_path: member_path.clone(),
11983 message: format!(
11984 "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
11985 ),
11986 }));
11987 }
11988 if !root && operations.is_empty() {
11989 return Err(Error::InvalidParallelGroup(ParallelGroupError {
11990 reason: "nested_group_empty",
11991 member_path: member_path.clone(),
11992 message: "a nested group must contain at least one durable leaf".to_string(),
11993 }));
11994 }
11995
11996 for (index, operation) in operations.iter().enumerate() {
11997 member_path.push(index);
11998 match operation {
11999 ParallelOperation::Activity {
12000 options, arguments, ..
12001 } => {
12002 options
12003 .validate()
12004 .map_err(|error| Error::InvalidActivityOptions(error))?;
12005 if let Err(error) = arguments {
12006 return Err(Error::InvalidParallelGroup(ParallelGroupError {
12007 reason: "arguments_invalid",
12008 member_path: member_path.clone(),
12009 message: error.to_string(),
12010 }));
12011 }
12012 }
12013 ParallelOperation::ChildWorkflow {
12014 options, arguments, ..
12015 } => {
12016 validate_parallel_child_options(options)?;
12017 if let Err(error) = arguments {
12018 return Err(Error::InvalidParallelGroup(ParallelGroupError {
12019 reason: "arguments_invalid",
12020 member_path: member_path.clone(),
12021 message: error.to_string(),
12022 }));
12023 }
12024 }
12025 ParallelOperation::Timer(duration)
12026 if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
12027 {
12028 return Err(Error::TimerDurationOverflow);
12029 }
12030 ParallelOperation::Timer(_) => {}
12031 ParallelOperation::Signal(signal_name) => {
12032 validate_user_signal_name(signal_name)?;
12033 if signal_name.trim().is_empty() {
12034 return Err(Error::InvalidParallelGroup(ParallelGroupError {
12035 reason: "signal_name_empty",
12036 member_path: member_path.clone(),
12037 message: "signal wait name must not be empty".to_string(),
12038 }));
12039 }
12040 }
12041 ParallelOperation::Condition { options, .. } => {
12042 options.validate()?;
12043 }
12044 ParallelOperation::Group(children) => {
12045 validate_parallel_operations(children, member_path, false)?;
12046 }
12047 }
12048 member_path.pop();
12049 }
12050 Ok(())
12051}
12052
12053fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
12054 if options.task_queue.trim().is_empty() {
12055 return Err(Error::InvalidChildWorkflowOptions(
12056 "task_queue must not be empty".to_string(),
12057 ));
12058 }
12059 for (name, value) in [
12060 (
12061 "execution_timeout_seconds",
12062 options.execution_timeout_seconds,
12063 ),
12064 ("run_timeout_seconds", options.run_timeout_seconds),
12065 ] {
12066 if value == Some(0) {
12067 return Err(Error::InvalidChildWorkflowOptions(format!(
12068 "{name} must be at least 1"
12069 )));
12070 }
12071 }
12072 if options
12073 .retry_policy
12074 .as_ref()
12075 .is_some_and(|policy| policy.max_attempts == Some(0))
12076 {
12077 return Err(Error::InvalidChildWorkflowOptions(
12078 "retry_policy.max_attempts must be at least 1".to_string(),
12079 ));
12080 }
12081 Ok(())
12082}
12083
12084fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
12085 ParallelShape::Group(
12086 operations
12087 .iter()
12088 .map(|operation| match operation {
12089 ParallelOperation::Group(children) => parallel_shape(children),
12090 _ => ParallelShape::Leaf,
12091 })
12092 .collect(),
12093 )
12094}
12095
12096fn parallel_descriptors(
12097 operations: Vec<ParallelOperation>,
12098 base_sequence: u64,
12099) -> Result<Vec<ParallelDescriptor>> {
12100 let size = parallel_leaf_count(&operations);
12101 let kind = parallel_group_kind(&operations).unwrap_or("activity");
12102 let mut descriptors = Vec::with_capacity(size);
12103 let mut cursor = 0;
12104
12105 for (index, operation) in operations.into_iter().enumerate() {
12106 match operation {
12107 ParallelOperation::Group(children) => {
12108 let child_base = base_sequence
12109 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
12110 .ok_or(Error::TimerDurationOverflow)?;
12111 for mut descriptor in parallel_descriptors(children, child_base)? {
12112 let outer_index = cursor + descriptor.offset;
12113 descriptor.group_path.insert(
12114 0,
12115 parallel_group_entry(base_sequence, size, outer_index, kind),
12116 );
12117 descriptor.member_path.insert(0, index);
12118 descriptor.offset = outer_index;
12119 descriptors.push(descriptor);
12120 }
12121 cursor = descriptors.len();
12122 }
12123 operation => {
12124 descriptors.push(ParallelDescriptor {
12125 operation,
12126 offset: cursor,
12127 member_path: vec![index],
12128 group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
12129 });
12130 cursor += 1;
12131 }
12132 }
12133 }
12134 Ok(descriptors)
12135}
12136
12137enum ParallelLeafCall {
12138 Activity(ActivityCall),
12139 ChildWorkflow(ChildWorkflowCall),
12140 Timer(TimerCall),
12141 Signal(SignalCall),
12142 Condition(ConditionWaitCall),
12143}
12144
12145fn parallel_leaf_call(
12146 ctx: &WorkflowContext,
12147 operation: ParallelOperation,
12148 parallel_group_path: Vec<ParallelGroupMetadata>,
12149) -> ParallelLeafCall {
12150 match operation {
12151 ParallelOperation::Activity {
12152 activity_type,
12153 options,
12154 arguments,
12155 } => ParallelLeafCall::Activity(ActivityCall {
12156 ctx: ctx.clone(),
12157 activity_type,
12158 options,
12159 worker_session: None,
12160 args: Some(arguments),
12161 scheduled: false,
12162 local: false,
12163 local_result: None,
12164 parallel_group_path,
12165 }),
12166 ParallelOperation::ChildWorkflow {
12167 workflow_type,
12168 options,
12169 arguments,
12170 } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
12171 ctx: ctx.clone(),
12172 workflow_type,
12173 options,
12174 args: Some(arguments),
12175 scheduled: false,
12176 matched_pending: false,
12177 parallel_group_path,
12178 }),
12179 ParallelOperation::Timer(duration) => {
12180 let delay_seconds = duration
12181 .as_secs()
12182 .checked_add(u64::from(duration.subsec_nanos() > 0));
12183 ParallelLeafCall::Timer(TimerCall {
12184 ctx: ctx.clone(),
12185 delay_seconds,
12186 scheduled: false,
12187 matched_pending: false,
12188 parallel_group_path,
12189 })
12190 }
12191 ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
12192 ctx: ctx.clone(),
12193 signal_name,
12194 runtime_reserved_allowed: false,
12195 opened_wait: false,
12196 matched_pending: false,
12197 parallel_group_path,
12198 }),
12199 ParallelOperation::Condition { options, predicate } => {
12200 ParallelLeafCall::Condition(ConditionWaitCall {
12201 ctx: ctx.clone(),
12202 options,
12203 predicate,
12204 occurrence_id: None,
12205 opened_wait: false,
12206 parallel_group_path,
12207 })
12208 }
12209 ParallelOperation::Group(_) => {
12210 unreachable!("parallel descriptors contain only durable leaves")
12211 }
12212 }
12213}
12214
12215impl ParallelLeafCall {
12216 fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
12217 match self {
12218 Self::Activity(call) => Pin::new(call)
12219 .poll_avro_value(cx)
12220 .map_ok(ParallelAvroResult::Activity),
12221 Self::ChildWorkflow(call) => Pin::new(call)
12222 .poll_avro_value(cx)
12223 .map_ok(ParallelAvroResult::ChildWorkflow),
12224 Self::Timer(call) => Pin::new(call)
12225 .poll(cx)
12226 .map_ok(|()| ParallelAvroResult::Timer),
12227 Self::Signal(call) => Pin::new(call)
12228 .poll_avro_value(cx)
12229 .map_ok(ParallelAvroResult::Signal),
12230 Self::Condition(call) => Pin::new(call)
12231 .poll(cx)
12232 .map_ok(ParallelAvroResult::Condition),
12233 }
12234 }
12235}
12236
12237struct ParallelLeaf {
12238 call: ParallelLeafCall,
12239 sequence: u64,
12240 member_path: Vec<usize>,
12241 group_path: Vec<ParallelGroupMetadata>,
12242 result: Option<ParallelAvroResult>,
12243}
12244
12245pub struct ParallelCall {
12247 ctx: WorkflowContext,
12248 operations: Option<Vec<ParallelOperation>>,
12249 shape: Option<ParallelShape>,
12250 leaves: Vec<ParallelLeaf>,
12251 pending_scope_delivery: bool,
12252 worker_session: Option<WorkerSessionOptions>,
12253}
12254
12255impl ParallelCall {
12256 fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
12257 Self {
12258 ctx,
12259 operations: Some(operations),
12260 shape: None,
12261 leaves: Vec::new(),
12262 pending_scope_delivery: false,
12263 worker_session: None,
12264 }
12265 }
12266
12267 fn initialize(&mut self) -> Result<()> {
12268 if let Some(session) = &self.worker_session {
12269 session.to_wire()?;
12270 }
12271 let operations = self.operations.take().unwrap_or_default();
12272 validate_parallel_operations(&operations, &mut Vec::new(), true)?;
12273 self.shape = Some(parallel_shape(&operations));
12274 if operations.is_empty() {
12275 return Ok(());
12276 }
12277
12278 let base_sequence = {
12279 let state = self
12280 .ctx
12281 .state
12282 .lock()
12283 .map_err(|_| Error::WorkflowStatePoisoned)?;
12284 if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
12285 recorded.sequence()
12286 } else {
12287 let last = state
12288 .recorded_commands
12289 .last()
12290 .map(RecordedCommand::sequence)
12291 .unwrap_or(0);
12292 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
12293 .and_then(|sequence| sequence.checked_add(1))
12294 .ok_or_else(|| {
12295 Error::InvalidParallelGroup(ParallelGroupError {
12296 reason: "sequence_overflow",
12297 member_path: Vec::new(),
12298 message: "parallel group sequence identity overflowed u64".to_string(),
12299 })
12300 })?
12301 }
12302 };
12303
12304 let descriptors = parallel_descriptors(operations, base_sequence)?;
12305 {
12306 let mut state = self
12307 .ctx
12308 .state
12309 .lock()
12310 .map_err(|_| Error::WorkflowStatePoisoned)?;
12311 let cursor = state.command_cursor;
12312 self.ctx.validate_scope_membership(&mut state, cursor)?;
12313 if state.prepare_parallel_scope_cancellation(&descriptors)? {
12314 self.pending_scope_delivery = true;
12315 return Ok(());
12316 }
12317 state.expand_cancellation_group(&descriptors)?;
12318 state.prepare_group_cancellation(&descriptors)?;
12319 }
12320 self.leaves = descriptors
12321 .into_iter()
12322 .map(|descriptor| {
12323 let mut call = parallel_leaf_call(
12324 &self.ctx,
12325 descriptor.operation,
12326 descriptor.group_path.clone(),
12327 );
12328 if let (Some(session), ParallelLeafCall::Activity(activity)) =
12329 (&self.worker_session, &mut call)
12330 {
12331 activity.worker_session = Some(session.clone());
12332 }
12333 ParallelLeaf {
12334 call,
12335 sequence: base_sequence + descriptor.offset as u64,
12336 member_path: descriptor.member_path,
12337 group_path: descriptor.group_path,
12338 result: None,
12339 }
12340 })
12341 .collect();
12342 Ok(())
12343 }
12344
12345 fn poll_avro_value(
12346 mut self: Pin<&mut Self>,
12347 cx: &mut TaskContext<'_>,
12348 ) -> Poll<Result<Vec<ParallelAvroResult>>> {
12349 if self.operations.is_some() {
12350 if let Err(error) = self.initialize() {
12351 return Poll::Ready(Err(error));
12352 }
12353 }
12354 if self.pending_scope_delivery {
12355 return Poll::Pending;
12356 }
12357 if self.leaves.is_empty() {
12358 return Poll::Ready(Ok(Vec::new()));
12359 }
12360
12361 let mut failures = Vec::new();
12362 let mut pending = false;
12363 for (index, leaf) in self.leaves.iter_mut().enumerate() {
12364 if leaf.result.is_some() {
12365 continue;
12366 }
12367 match leaf.call.poll_avro_value(cx) {
12368 Poll::Ready(Ok(result)) => leaf.result = Some(result),
12369 Poll::Ready(Err(error)) => failures.push((index, error)),
12370 Poll::Pending => pending = true,
12371 }
12372 }
12373
12374 if !failures.is_empty() {
12375 if let Some(position) = failures
12376 .iter()
12377 .position(|(_, error)| workflow_task_integrity_error(error))
12378 {
12379 return Poll::Ready(Err(failures.remove(position).1));
12380 }
12381 if self
12382 .ctx
12383 .state
12384 .lock()
12385 .map_err(|_| Error::WorkflowStatePoisoned)?
12386 .cancellation_delivery_intent
12387 .is_some()
12388 {
12389 return Poll::Pending;
12390 }
12391 if let Some(position) = failures
12392 .iter()
12393 .position(|(_, error)| matches!(error, Error::CooperativeCancellationRequested(_)))
12394 {
12395 return Poll::Ready(Err(failures.remove(position).1));
12396 }
12397 failures.sort_by_key(|(index, _)| *index);
12398 let (failed_index, cause) = failures.remove(0);
12399 let failed = &self.leaves[failed_index];
12400 let completed = self
12401 .leaves
12402 .iter()
12403 .filter_map(|leaf| {
12404 leaf.result
12405 .clone()
12406 .and_then(|result| result.into_json_result().ok())
12407 .map(|result| ParallelCompletion {
12408 member_path: leaf.member_path.clone(),
12409 result,
12410 })
12411 })
12412 .collect();
12413 let group_id = failed
12414 .group_path
12415 .first()
12416 .map(|entry| entry.parallel_group_id.clone())
12417 .unwrap_or_default();
12418 let sequences = self
12419 .leaves
12420 .iter()
12421 .map(|leaf| leaf.sequence)
12422 .collect::<Vec<_>>();
12423 self.ctx
12424 .state
12425 .lock()
12426 .map_err(|_| Error::WorkflowStatePoisoned)?
12427 .advance_cancellation_sequences(&sequences, Some(failed.sequence));
12428 return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
12429 group_id,
12430 member_path: failed.member_path.clone(),
12431 group_path: failed.group_path.clone(),
12432 completed,
12433 cause: Box::new(cause),
12434 })));
12435 }
12436 if pending
12437 || self
12438 .ctx
12439 .state
12440 .lock()
12441 .map_err(|_| Error::WorkflowStatePoisoned)?
12442 .cancellation_delivery_intent
12443 .is_some()
12444 {
12445 return Poll::Pending;
12446 }
12447
12448 let mut flat_results = self
12449 .leaves
12450 .iter_mut()
12451 .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
12452 .collect::<Vec<_>>()
12453 .into_iter();
12454 let results = parallel_results_for_shape(
12455 self.shape.as_ref().expect("initialized parallel shape"),
12456 &mut flat_results,
12457 );
12458 let sequences = self
12459 .leaves
12460 .iter()
12461 .map(|leaf| leaf.sequence)
12462 .collect::<Vec<_>>();
12463 self.ctx
12464 .state
12465 .lock()
12466 .map_err(|_| Error::WorkflowStatePoisoned)?
12467 .advance_cancellation_sequences(&sequences, None);
12468 Poll::Ready(Ok(match results {
12469 ParallelAvroResult::Group(results) => results,
12470 ParallelAvroResult::Activity(_)
12471 | ParallelAvroResult::ChildWorkflow(_)
12472 | ParallelAvroResult::Timer
12473 | ParallelAvroResult::Signal(_)
12474 | ParallelAvroResult::Condition(_) => {
12475 unreachable!("root parallel shape is a group")
12476 }
12477 }))
12478 }
12479}
12480
12481fn parallel_results_for_shape(
12482 shape: &ParallelShape,
12483 flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
12484) -> ParallelAvroResult {
12485 match shape {
12486 ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
12487 ParallelShape::Group(children) => ParallelAvroResult::Group(
12488 children
12489 .iter()
12490 .map(|child| parallel_results_for_shape(child, flat_results))
12491 .collect(),
12492 ),
12493 }
12494}
12495
12496impl Future for ParallelCall {
12497 type Output = Result<Vec<ParallelResult>>;
12498
12499 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12500 self.poll_avro_value(cx)
12501 .map_ok(|results| {
12502 results
12503 .into_iter()
12504 .map(ParallelAvroResult::into_json_result)
12505 .collect::<Result<Vec<_>>>()
12506 })
12507 .map_ok(|result| result)
12508 .flatten_result()
12509 }
12510}
12511
12512#[derive(Clone, Debug)]
12513struct SelectionMemberPlan {
12514 key: SelectionKey,
12515 index: usize,
12516 base_sequence: u64,
12517 size: usize,
12518 kind: String,
12519 shape: ParallelShape,
12520 leaf_start: usize,
12521}
12522
12523fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
12524 match operation {
12525 ParallelOperation::Activity { .. } => "activity",
12526 ParallelOperation::ChildWorkflow { .. } => "child",
12527 ParallelOperation::Timer(_) => "timer",
12528 ParallelOperation::Signal(_) => "signal",
12529 ParallelOperation::Condition { .. } => "condition",
12530 ParallelOperation::Group(_) => "group",
12531 }
12532}
12533
12534fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
12535 match operation {
12536 ParallelOperation::Group(children) => parallel_shape(children),
12537 _ => ParallelShape::Leaf,
12538 }
12539}
12540
12541fn selection_descriptors(
12542 operations: Vec<(SelectionKey, ParallelOperation)>,
12543 base_sequence: u64,
12544) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
12545 if operations.is_empty() {
12546 return Err(Error::InvalidParallelGroup(ParallelGroupError {
12547 reason: "selection_empty",
12548 member_path: Vec::new(),
12549 message: "durable selection requires at least one operation".to_string(),
12550 }));
12551 }
12552 let operation_refs = operations
12553 .iter()
12554 .map(|(_, operation)| operation)
12555 .collect::<Vec<_>>();
12556 let total_size = operation_refs
12557 .iter()
12558 .map(|operation| match operation {
12559 ParallelOperation::Group(children) => parallel_leaf_count(children),
12560 _ => 1,
12561 })
12562 .sum::<usize>();
12563 if total_size > MAX_PARALLEL_OPERATIONS {
12564 return Err(Error::InvalidParallelGroup(ParallelGroupError {
12565 reason: "fan_out_limit_exceeded",
12566 member_path: Vec::new(),
12567 message: format!(
12568 "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
12569 ),
12570 }));
12571 }
12572 let group_kind = {
12573 let mut kind = None;
12574 for operation in &operation_refs {
12575 let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
12576 match kind {
12577 None => kind = Some(operation_kind),
12578 Some(current) if current == operation_kind => {}
12579 Some(_) => {
12580 kind = Some("mixed");
12581 break;
12582 }
12583 }
12584 }
12585 kind.unwrap_or("mixed")
12586 };
12587
12588 let mut descriptors = Vec::with_capacity(total_size);
12589 let mut members = Vec::with_capacity(operations.len());
12590 let mut cursor = 0usize;
12591 let mut seen_keys: Vec<SelectionKey> = Vec::new();
12592 for (member_index, (key, operation)) in operations.into_iter().enumerate() {
12593 if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
12594 return Err(Error::InvalidParallelGroup(ParallelGroupError {
12595 reason: "selection_key_invalid",
12596 member_path: vec![member_index],
12597 message: "selection member keys must be non-empty strings or non-negative integers"
12598 .to_string(),
12599 }));
12600 }
12601 if seen_keys.contains(&key) {
12602 return Err(Error::InvalidParallelGroup(ParallelGroupError {
12603 reason: "selection_key_duplicate",
12604 member_path: vec![member_index],
12605 message: format!("selection member key {key:?} is duplicated"),
12606 }));
12607 }
12608 seen_keys.push(key.clone());
12609 let member_size = match &operation {
12610 ParallelOperation::Group(children) => parallel_leaf_count(children),
12611 _ => 1,
12612 };
12613 if member_size == 0 {
12614 return Err(Error::InvalidParallelGroup(ParallelGroupError {
12615 reason: "selection_member_empty",
12616 member_path: vec![member_index],
12617 message: "a selection member must contain at least one durable leaf".to_string(),
12618 }));
12619 }
12620 let member_base = base_sequence
12621 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
12622 .ok_or(Error::TimerDurationOverflow)?;
12623 let member_kind = selection_operation_kind(&operation).to_string();
12624 let member_shape = selection_operation_shape(&operation);
12625 let leaf_start = descriptors.len();
12626 match operation {
12627 ParallelOperation::Group(children) => {
12628 validate_parallel_operations(&children, &mut vec![member_index], false)?;
12629 for mut descriptor in parallel_descriptors(children, member_base)? {
12630 let flat_index = cursor + descriptor.offset;
12631 descriptor.group_path.insert(
12632 0,
12633 selection_group_entry(
12634 base_sequence,
12635 total_size,
12636 flat_index,
12637 group_kind,
12638 &SelectionMemberMetadata {
12639 key: key.clone(),
12640 index: member_index,
12641 base_sequence: member_base,
12642 size: member_size,
12643 kind: member_kind.clone(),
12644 },
12645 ),
12646 );
12647 descriptor.member_path.insert(0, member_index);
12648 descriptor.offset = flat_index;
12649 descriptors.push(descriptor);
12650 }
12651 }
12652 operation => {
12653 validate_parallel_operations(
12654 std::slice::from_ref(&operation),
12655 &mut Vec::new(),
12656 true,
12657 )?;
12658 descriptors.push(ParallelDescriptor {
12659 operation,
12660 offset: cursor,
12661 member_path: vec![member_index],
12662 group_path: vec![selection_group_entry(
12663 base_sequence,
12664 total_size,
12665 cursor,
12666 group_kind,
12667 &SelectionMemberMetadata {
12668 key: key.clone(),
12669 index: member_index,
12670 base_sequence: member_base,
12671 size: member_size,
12672 kind: member_kind.clone(),
12673 },
12674 )],
12675 });
12676 }
12677 }
12678 members.push(SelectionMemberPlan {
12679 key,
12680 index: member_index,
12681 base_sequence: member_base,
12682 size: member_size,
12683 kind: member_kind,
12684 shape: member_shape,
12685 leaf_start,
12686 });
12687 cursor += member_size;
12688 }
12689 Ok((descriptors, members))
12690}
12691
12692struct SelectionLeaf {
12693 call: ParallelLeafCall,
12694 outcome: Option<Result<ParallelAvroResult>>,
12695}
12696
12697#[derive(Clone)]
12699pub struct DurableOperationHandle {
12700 ctx: WorkflowContext,
12701 pub key: SelectionKey,
12702 pub index: usize,
12703 pub kind: String,
12704 pub identity: String,
12705 pub base_sequence: u64,
12706 pub size: usize,
12707 pub selection_group_id: String,
12708 shape: ParallelShape,
12709}
12710
12711impl std::fmt::Debug for DurableOperationHandle {
12712 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
12713 formatter
12714 .debug_struct("DurableOperationHandle")
12715 .field("key", &self.key)
12716 .field("index", &self.index)
12717 .field("kind", &self.kind)
12718 .field("identity", &self.identity)
12719 .field("base_sequence", &self.base_sequence)
12720 .field("size", &self.size)
12721 .field("selection_group_id", &self.selection_group_id)
12722 .finish()
12723 }
12724}
12725
12726impl DurableOperationHandle {
12727 pub fn await_result(&self) -> DurableOperationAwaitCall {
12729 DurableOperationAwaitCall {
12730 handle: self.clone(),
12731 }
12732 }
12733
12734 pub fn cancel(&self) -> CancelDurableOperationCall {
12738 CancelDurableOperationCall {
12739 handle: self.clone(),
12740 emitted: false,
12741 }
12742 }
12743}
12744
12745#[derive(Debug)]
12747pub struct SelectionResult {
12748 pub key: SelectionKey,
12749 pub index: usize,
12750 pub kind: String,
12751 pub identity: String,
12752 pub value: Option<ParallelResult>,
12753 pub failure: Option<Error>,
12754 pub winner: DurableOperationHandle,
12755 pub handles: Vec<DurableOperationHandle>,
12756}
12757
12758impl SelectionResult {
12759 pub fn succeeded(&self) -> bool {
12760 self.failure.is_none()
12761 }
12762
12763 pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
12764 self.handles.iter().find(|handle| &handle.key == key)
12765 }
12766
12767 pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
12768 self.handles
12769 .iter()
12770 .filter(|handle| handle.index != self.index)
12771 .collect()
12772 }
12773
12774 pub fn into_result(self) -> Result<ParallelResult> {
12775 match (self.value, self.failure) {
12776 (Some(value), None) => Ok(value),
12777 (_, Some(error)) => Err(error),
12778 _ => Err(Error::WorkerLoop(
12779 "selection result contained neither a value nor a failure".to_string(),
12780 )),
12781 }
12782 }
12783}
12784
12785pub struct SelectCall {
12787 ctx: WorkflowContext,
12788 operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
12789 members: Vec<SelectionMemberPlan>,
12790 leaves: Vec<SelectionLeaf>,
12791 group_id: Option<String>,
12792}
12793
12794impl SelectCall {
12795 fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
12796 Self {
12797 ctx,
12798 operations: Some(operations),
12799 members: Vec::new(),
12800 leaves: Vec::new(),
12801 group_id: None,
12802 }
12803 }
12804
12805 fn initialize(&mut self) -> Result<()> {
12806 let operations = self.operations.take().unwrap_or_default();
12807 let base_sequence = {
12808 let state = self
12809 .ctx
12810 .state
12811 .lock()
12812 .map_err(|_| Error::WorkflowStatePoisoned)?;
12813 if let Some(RecordedCommand::CancellationGroup { sequence, .. }) =
12814 state.recorded_commands.get(state.command_cursor)
12815 {
12816 *sequence
12817 } else if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor)
12818 {
12819 marker.selection_group_base_sequence
12820 } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
12821 recorded.sequence()
12822 } else {
12823 let last = state
12824 .recorded_commands
12825 .last()
12826 .map(RecordedCommand::sequence)
12827 .unwrap_or(0);
12828 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
12829 .and_then(|sequence| sequence.checked_add(1))
12830 .ok_or(Error::TimerDurationOverflow)?
12831 }
12832 };
12833 let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
12834 let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
12835 {
12836 let mut state = self
12837 .ctx
12838 .state
12839 .lock()
12840 .map_err(|_| Error::WorkflowStatePoisoned)?;
12841 state.expand_cancellation_group(&descriptors)?;
12842 state.prepare_group_cancellation(&descriptors)?;
12843 }
12844 self.leaves = descriptors
12845 .into_iter()
12846 .map(|descriptor| SelectionLeaf {
12847 call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
12848 outcome: None,
12849 })
12850 .collect();
12851 self.members = members;
12852 self.group_id = Some(group_id);
12853 Ok(())
12854 }
12855}
12856
12857impl Future for SelectCall {
12858 type Output = Result<SelectionResult>;
12859
12860 fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12861 if self.operations.is_some() {
12862 if let Err(error) = self.initialize() {
12863 return Poll::Ready(Err(error));
12864 }
12865 }
12866
12867 for leaf in &mut self.leaves {
12868 if leaf.outcome.is_some() {
12869 continue;
12870 }
12871 if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
12872 if outcome
12873 .as_ref()
12874 .err()
12875 .is_some_and(workflow_task_integrity_error)
12876 {
12877 return Poll::Ready(outcome.map(|_| unreachable!()));
12878 }
12879 leaf.outcome = Some(outcome);
12880 }
12881 }
12882
12883 if self
12884 .ctx
12885 .state
12886 .lock()
12887 .map_err(|_| Error::WorkflowStatePoisoned)?
12888 .cancellation_delivery_intent
12889 .is_some()
12890 {
12891 return Poll::Pending;
12892 }
12893
12894 if let Some(leaf) = self.leaves.iter_mut().find(|leaf| {
12895 matches!(
12896 leaf.outcome,
12897 Some(Err(Error::CooperativeCancellationRequested(_)))
12898 )
12899 }) {
12900 return Poll::Ready(
12901 leaf.outcome
12902 .take()
12903 .expect("matched cancellation")
12904 .map(|_| unreachable!()),
12905 );
12906 }
12907
12908 let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
12909 let selection_member_range = self
12910 .members
12911 .first()
12912 .map(|member| member.base_sequence)
12913 .zip(self.leaves.len().try_into().ok())
12914 .map(|(base_sequence, size): (u64, u64)| {
12915 base_sequence..base_sequence.saturating_add(size)
12916 });
12917 let marker = {
12918 let mut state = match self.ctx.state.lock() {
12919 Ok(state) => state,
12920 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12921 };
12922 let marker = state
12923 .selection_markers
12924 .get(state.selection_marker_cursor)
12925 .cloned();
12926 if marker.is_none()
12927 && all_members_terminal
12928 && selection_member_range.as_ref().is_some_and(|member_range| {
12929 state
12930 .recorded_commands
12931 .iter()
12932 .any(|command| member_range.contains(&command.sequence()))
12933 })
12934 {
12935 state.matched_recorded_pending = true;
12941 }
12942 marker
12943 };
12944 let Some(marker) = marker else {
12945 return Poll::Pending;
12946 };
12947 if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
12948 || marker.selection_group_size != self.leaves.len()
12949 || self.members.first().map(|member| member.base_sequence)
12950 != Some(marker.selection_group_base_sequence)
12951 {
12952 return Poll::Ready(Err(invalid_recorded_history(
12953 "selection_group_shape_mismatch",
12954 marker.selection_group_base_sequence,
12955 self.group_id
12956 .as_deref()
12957 .unwrap_or("initialized selection group"),
12958 &marker.selection_group_id,
12959 "recorded selection group differs from current workflow code",
12960 )));
12961 }
12962 let Some(member_position) = self.members.iter().position(|member| {
12963 member.key == marker.member_key
12964 && member.index == marker.member_index
12965 && member.base_sequence == marker.member_base_sequence
12966 && member.size == marker.member_size
12967 && member.kind == marker.operation_kind
12968 }) else {
12969 return Poll::Ready(Err(invalid_recorded_history(
12970 "selection_member_shape_mismatch",
12971 marker.member_base_sequence,
12972 "winner member matching current workflow code",
12973 &format!("{:?}", marker.member_key),
12974 "recorded selection winner differs from the authored member identity",
12975 )));
12976 };
12977 let member = self.members[member_position].clone();
12978 let (handles, resolution_sequence) = {
12979 let mut state = match self.ctx.state.lock() {
12980 Ok(state) => state,
12981 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12982 };
12983 let identities = self
12984 .members
12985 .iter()
12986 .map(|candidate| {
12987 selection_operation_identity(
12988 &state,
12989 &candidate.kind,
12990 candidate.base_sequence,
12991 candidate.size,
12992 )
12993 })
12994 .collect::<Vec<_>>();
12995 if let Some((position, missing)) = identities
12996 .iter()
12997 .enumerate()
12998 .find(|(_, identity)| identity.is_empty())
12999 .map(|(position, identity)| (position, identity.clone()))
13000 {
13001 let candidate = &self.members[position];
13002 return Poll::Ready(Err(invalid_recorded_history(
13003 "selection_operation_identity_missing",
13004 candidate.base_sequence,
13005 &format!(
13006 "durable {} resource identity from scheduled/open history",
13007 candidate.kind
13008 ),
13009 &missing,
13010 "selection member history is missing its canonical durable identity",
13011 )));
13012 }
13013 let expected_winner_identity = &identities[member_position];
13014 let resolution_sequence = match validated_selection_resolution_sequence(
13015 &state,
13016 &marker,
13017 &member,
13018 expected_winner_identity,
13019 ) {
13020 Ok(sequence) => sequence,
13021 Err(error) => return Poll::Ready(Err(error)),
13022 };
13023 let handles = self
13024 .members
13025 .iter()
13026 .zip(identities)
13027 .map(|(member, identity)| DurableOperationHandle {
13028 ctx: self.ctx.clone(),
13029 key: member.key.clone(),
13030 index: member.index,
13031 kind: member.kind.clone(),
13032 identity,
13033 base_sequence: member.base_sequence,
13034 size: member.size,
13035 selection_group_id: marker.selection_group_id.clone(),
13036 shape: member.shape.clone(),
13037 })
13038 .collect::<Vec<_>>();
13039 if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
13040 return Poll::Ready(Err(error));
13041 }
13042 state.selection_marker_cursor += 1;
13043 (handles, resolution_sequence)
13044 };
13045
13046 let mut winner_failure = None;
13047 let mut flat_results = Vec::with_capacity(member.size);
13048 if marker.outcome == "failed" {
13049 let resolution_offset = match resolution_sequence
13050 .checked_sub(member.base_sequence)
13051 .and_then(|offset| usize::try_from(offset).ok())
13052 {
13053 Some(offset) if offset < member.size => offset,
13054 _ => {
13055 return Poll::Ready(Err(invalid_recorded_history(
13056 "selection_resolution_event_mismatch",
13057 member.base_sequence,
13058 "failure event within selected member bounds",
13059 &resolution_sequence.to_string(),
13060 "selection failure event is outside the authored member",
13061 )))
13062 }
13063 };
13064 let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
13065 match leaf.outcome.take() {
13066 Some(Err(error)) => winner_failure = Some(error),
13067 _ => {
13068 return Poll::Ready(Err(invalid_recorded_history(
13069 "selection_winner_outcome_mismatch",
13070 member.base_sequence,
13071 "exact failed terminal history referenced by SelectionResolved",
13072 "missing or successful resolution event",
13073 "selection winner marker disagrees with terminal operation history",
13074 )))
13075 }
13076 }
13077 } else {
13078 for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
13079 match leaf.outcome.take() {
13080 Some(Ok(result)) => flat_results.push(result),
13081 Some(Err(_)) => {
13082 return Poll::Ready(Err(invalid_recorded_history(
13083 "selection_winner_outcome_mismatch",
13084 member.base_sequence,
13085 "fully completed nested selection member",
13086 "failed durable leaf",
13087 "completed selection winner contains a failed leaf",
13088 )))
13089 }
13090 None => {
13091 return Poll::Ready(Err(invalid_recorded_history(
13092 "selection_winner_unresolved",
13093 member.base_sequence,
13094 "terminal history for every completed winner leaf",
13095 "pending member history",
13096 "completed SelectionResolved member has an unfinished durable barrier",
13097 )))
13098 }
13099 }
13100 }
13101 }
13102 let value = if winner_failure.is_none() {
13103 let mut flat_results = flat_results.into_iter();
13104 let value = parallel_results_for_shape(&member.shape, &mut flat_results);
13105 match value.into_json_result() {
13106 Ok(value) => Some(value),
13107 Err(error) => return Poll::Ready(Err(error)),
13108 }
13109 } else {
13110 None
13111 };
13112 let winner = handles[member_position].clone();
13113 self.ctx
13114 .state
13115 .lock()
13116 .map_err(|_| Error::WorkflowStatePoisoned)?
13117 .advance_cancellation_selection(&marker.selection_group_id);
13118 Poll::Ready(Ok(SelectionResult {
13119 key: winner.key.clone(),
13120 index: winner.index,
13121 kind: winner.kind.clone(),
13122 identity: winner.identity.clone(),
13123 value,
13124 failure: winner_failure,
13125 winner,
13126 handles,
13127 }))
13128 }
13129}
13130
13131fn selection_operation_identity(
13132 state: &WorkflowState,
13133 kind: &str,
13134 base_sequence: u64,
13135 size: usize,
13136) -> String {
13137 if kind == "group" {
13138 return format!("group:{base_sequence}:{size}");
13139 }
13140 let fields: &[&str] = match kind {
13141 "activity" => &["activity_execution_id"],
13142 "child" => &["child_workflow_run_id"],
13143 "timer" => &["timer_id"],
13144 "signal" => &["signal_wait_id"],
13145 "condition" => &["condition_wait_id"],
13146 _ => &[],
13147 };
13148 for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
13149 for event in state
13150 .history_events
13151 .iter()
13152 .filter(|event| durable_event_sequence(event) == Some(sequence))
13153 {
13154 for field in fields {
13155 if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
13156 if !identity.is_empty() {
13157 return identity.to_string();
13158 }
13159 }
13160 }
13161 }
13162 }
13163 String::new()
13164}
13165
13166fn validated_selection_resolution_sequence(
13167 state: &WorkflowState,
13168 marker: &SelectionMarker,
13169 member: &SelectionMemberPlan,
13170 expected_identity: &str,
13171) -> Result<u64> {
13172 if expected_identity.is_empty() {
13173 return Err(invalid_recorded_history(
13174 "selection_operation_identity_missing",
13175 member.base_sequence,
13176 &format!(
13177 "durable {} resource identity from scheduled/open history",
13178 member.kind
13179 ),
13180 "missing operation identity",
13181 "selection member history is missing its canonical durable identity",
13182 ));
13183 }
13184 if marker.operation_identity != expected_identity {
13185 return Err(invalid_recorded_history(
13186 "selection_operation_identity_mismatch",
13187 member.base_sequence,
13188 expected_identity,
13189 &marker.operation_identity,
13190 "selection winner identity does not match durable scheduled/open history",
13191 ));
13192 }
13193
13194 let failure_types = [
13195 "ActivityFailed",
13196 "ActivityCancelled",
13197 "ActivityTimedOut",
13198 "ChildRunFailed",
13199 "ChildRunCancelled",
13200 "ChildRunTerminated",
13201 ];
13202 let success_types = [
13203 "ActivityCompleted",
13204 "ChildRunCompleted",
13205 "TimerFired",
13206 "SignalApplied",
13207 "ConditionWaitSatisfied",
13208 "ConditionWaitTimedOut",
13209 ];
13210 let terminal_types: &[&str] = if marker.outcome == "failed" {
13211 &failure_types
13212 } else {
13213 &success_types
13214 };
13215 let mut candidates = Vec::new();
13216 let member_commands = (member.base_sequence
13217 ..member.base_sequence.saturating_add(member.size as u64))
13218 .filter_map(|sequence| selection_member_recorded_command(state, sequence))
13219 .map(|command| (command.sequence(), command))
13220 .collect::<BTreeMap<_, _>>();
13221 let mut resolved_conditions = BTreeSet::new();
13222 for event in state.history_events.iter() {
13223 let Some(sequence) = durable_event_sequence(event) else {
13224 continue;
13225 };
13226 let Some(command) = member_commands.get(&sequence) else {
13227 continue;
13228 };
13229 if !terminal_types.contains(&event.event_type.as_str()) {
13230 continue;
13231 }
13232 if matches!(command, RecordedCommand::ConditionWait { .. }) {
13233 if !matches!(
13234 event.event_type.as_str(),
13235 "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13236 ) && !(event.event_type == "TimerFired"
13237 && event.payload.get("timer_kind").and_then(Value::as_str)
13238 == Some("condition_timeout"))
13239 {
13240 continue;
13241 }
13242 if !resolved_conditions.insert(sequence) {
13243 continue;
13244 }
13245 }
13246 let event_id = event
13247 .raw
13248 .get("id")
13249 .or_else(|| event.raw.get("event_id"))
13250 .and_then(Value::as_str)
13251 .filter(|value| !value.is_empty())
13252 .ok_or_else(|| {
13253 invalid_recorded_history(
13254 "selection_resolution_event_id_missing",
13255 member.base_sequence,
13256 "terminal selection history with a durable event id",
13257 &event.payload.to_string(),
13258 "selection terminal history cannot be bound to its winner marker",
13259 )
13260 })?;
13261 candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
13262 }
13263 let resolution = if marker.outcome == "failed" {
13264 candidates.first()
13265 } else {
13266 candidates.last()
13267 };
13268 let Some((event_id, event_type, sequence)) = resolution else {
13269 return Err(invalid_recorded_history(
13270 "selection_resolution_event_missing",
13271 member.base_sequence,
13272 "terminal history for the selected member",
13273 &format!("{:?}", marker.member_key),
13274 "selection winner marker has no matching durable terminal event",
13275 ));
13276 };
13277 if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
13278 return Err(invalid_recorded_history(
13279 "selection_resolution_event_mismatch",
13280 member.base_sequence,
13281 &format!("{event_type}:{event_id}"),
13282 &format!(
13283 "{}:{}",
13284 marker.resolution_event_type, marker.resolution_event_id
13285 ),
13286 "selection winner marker does not reference the event that made its member terminal",
13287 ));
13288 }
13289 Ok(*sequence)
13290}
13291
13292fn selection_member_recorded_command(
13293 state: &WorkflowState,
13294 authored_sequence: u64,
13295) -> Option<&RecordedCommand> {
13296 let original = state
13297 .recorded_commands
13298 .iter()
13299 .find(|command| command.sequence() == authored_sequence)?;
13300 let RecordedCommand::ConditionWait {
13301 occurrence_id,
13302 parallel_group_path: Some(_),
13303 ..
13304 } = original
13305 else {
13306 return Some(original);
13307 };
13308 state
13312 .recorded_commands
13313 .iter()
13314 .filter(|command| {
13315 matches!(command, RecordedCommand::ConditionWait {
13316 occurrence_id: candidate, ..
13317 } if candidate == occurrence_id)
13318 })
13319 .max_by_key(|command| command.sequence())
13320}
13321
13322fn recorded_selection_member_outcome(
13323 state: &WorkflowState,
13324 handle: &DurableOperationHandle,
13325) -> Result<Option<ParallelResult>> {
13326 for event in state.history_events.iter() {
13327 let Some(sequence) = durable_event_sequence(event) else {
13328 continue;
13329 };
13330 if sequence < handle.base_sequence
13331 || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
13332 || !matches!(
13333 event.event_type.as_str(),
13334 "ActivityFailed"
13335 | "ActivityCancelled"
13336 | "ActivityTimedOut"
13337 | "ChildRunFailed"
13338 | "ChildRunCancelled"
13339 | "ChildRunTerminated"
13340 )
13341 {
13342 continue;
13343 }
13344 let Some(command) = state
13345 .recorded_commands
13346 .iter()
13347 .find(|command| command.sequence() == sequence)
13348 else {
13349 continue;
13350 };
13351 match command {
13352 RecordedCommand::Activity {
13353 outcome: Some(Err(failure)),
13354 ..
13355 } => return Err(Error::ActivityFailed(failure.clone())),
13356 RecordedCommand::ChildWorkflow {
13357 outcome: Some(Err(failure)),
13358 ..
13359 } => return Err(Error::ChildWorkflowFailed(failure.clone())),
13360 _ => {}
13361 }
13362 }
13363
13364 let mut results = Vec::with_capacity(handle.size);
13365 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
13366 let Some(command) = selection_member_recorded_command(state, sequence) else {
13367 return Ok(None);
13368 };
13369 let result = match command {
13370 RecordedCommand::Activity { outcome, .. } => match outcome {
13371 Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
13372 Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
13373 None => return Ok(None),
13374 },
13375 RecordedCommand::Timer { fired, .. } => {
13376 if !fired {
13377 return Ok(None);
13378 }
13379 ParallelAvroResult::Timer
13380 }
13381 RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
13382 Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
13383 Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
13384 None => return Ok(None),
13385 },
13386 RecordedCommand::SignalWait { value, .. } => match value {
13387 Some(value) => ParallelAvroResult::Signal(value.clone()),
13388 None => return Ok(None),
13389 },
13390 RecordedCommand::ConditionWait { result, .. } => match result {
13391 Some(result) => ParallelAvroResult::Condition(*result),
13392 None => return Ok(None),
13393 },
13394 other => {
13395 return Err(command_mismatch(
13396 other,
13397 format!("selected {} member", handle.kind),
13398 ))
13399 }
13400 };
13401 results.push(result);
13402 }
13403 let mut results = results.into_iter();
13404 parallel_results_for_shape(&handle.shape, &mut results)
13405 .into_json_result()
13406 .map(Some)
13407}
13408
13409fn recorded_selection_member_is_terminal(
13410 state: &WorkflowState,
13411 handle: &DurableOperationHandle,
13412) -> bool {
13413 let mut completed = 0usize;
13414 let mut all_completed = true;
13415 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
13416 let Some(command) = selection_member_recorded_command(state, sequence) else {
13417 all_completed = false;
13418 continue;
13419 };
13420 let terminal = match command {
13421 RecordedCommand::Activity {
13422 outcome: Some(Err(_)),
13423 ..
13424 }
13425 | RecordedCommand::ChildWorkflow {
13426 outcome: Some(Err(_)),
13427 ..
13428 } => return true,
13429 RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
13430 RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
13431 RecordedCommand::Timer { fired, .. } => *fired,
13432 RecordedCommand::SignalWait { value, .. } => value.is_some(),
13433 RecordedCommand::ConditionWait { result, .. } => result.is_some(),
13434 RecordedCommand::SearchAttributes { .. }
13435 | RecordedCommand::CancellationScope { .. }
13436 | RecordedCommand::SideEffect { .. }
13437 | RecordedCommand::VersionMarker { .. }
13438 | RecordedCommand::Memo { .. }
13439 | RecordedCommand::CancellationBoundary { .. }
13440 | RecordedCommand::CancellationGroup { .. } => false,
13441 };
13442 if !terminal {
13443 all_completed = false;
13444 continue;
13445 }
13446 completed += 1;
13447 }
13448 all_completed && completed == handle.size
13449}
13450
13451fn selection_cancellation_for_handle(
13452 state: &WorkflowState,
13453 handle: &DurableOperationHandle,
13454) -> Result<bool> {
13455 let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
13456 recorded.selection_group_id == handle.selection_group_id
13457 && recorded.member_base_sequence == handle.base_sequence
13458 }) else {
13459 return Ok(false);
13460 };
13461 validate_selection_cancellation_marker(marker, handle)?;
13462 Ok(true)
13463}
13464
13465fn validate_selection_cancellations_for_handles(
13466 state: &WorkflowState,
13467 handles: &[DurableOperationHandle],
13468) -> Result<()> {
13469 let Some(group_id) = handles
13470 .first()
13471 .map(|handle| handle.selection_group_id.as_str())
13472 else {
13473 return Ok(());
13474 };
13475 for marker in state
13476 .cancelled_selection_members
13477 .iter()
13478 .filter(|marker| marker.selection_group_id == group_id)
13479 {
13480 let Some(handle) = handles
13481 .iter()
13482 .find(|handle| handle.base_sequence == marker.member_base_sequence)
13483 else {
13484 return Err(invalid_recorded_history(
13485 "selection_cancellation_member_mismatch",
13486 marker.member_base_sequence,
13487 "SelectionOperationCancelled matching an authored selection handle",
13488 &format!("{marker:?}"),
13489 "selection cancellation member base does not name an authored member",
13490 ));
13491 };
13492 validate_selection_cancellation_marker(marker, handle)?;
13493 }
13494 Ok(())
13495}
13496
13497fn validate_selection_cancellation_marker(
13498 marker: &SelectionCancellation,
13499 handle: &DurableOperationHandle,
13500) -> Result<()> {
13501 if marker.selection_group_id != handle.selection_group_id
13502 || marker.member_key != handle.key
13503 || marker.member_index != handle.index
13504 || marker.member_base_sequence != handle.base_sequence
13505 || marker.member_size != handle.size
13506 || marker.operation_kind != handle.kind
13507 || marker.operation_identity != handle.identity
13508 {
13509 return Err(invalid_recorded_history(
13510 "selection_cancellation_member_mismatch",
13511 handle.base_sequence,
13512 "SelectionOperationCancelled matching the authored selection handle",
13513 &format!("{marker:?}"),
13514 "selection cancellation history targets different authored member metadata",
13515 ));
13516 }
13517 Ok(())
13518}
13519
13520pub struct DurableOperationAwaitCall {
13522 handle: DurableOperationHandle,
13523}
13524
13525impl Future for DurableOperationAwaitCall {
13526 type Output = Result<ParallelResult>;
13527
13528 fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
13529 let mut state = match self.handle.ctx.state.lock() {
13530 Ok(state) => state,
13531 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
13532 };
13533 if let Err(error) = state.replay_selection_handle_cancellation(&self.handle) {
13534 return Poll::Ready(Err(error));
13535 }
13536 match state.prepare_selection_handle_cancellation(&self.handle) {
13537 Ok(true) => return Poll::Pending,
13538 Ok(false) => {}
13539 Err(error) => return Poll::Ready(Err(error)),
13540 }
13541 match selection_cancellation_for_handle(&state, &self.handle) {
13542 Err(error) => return Poll::Ready(Err(error)),
13543 Ok(false) => {}
13544 Ok(true) => {
13545 state.advance_cancellation_receipt(
13546 &self.handle.selection_group_id,
13547 self.handle.base_sequence,
13548 );
13549 return Poll::Ready(Err(Error::DurableOperationCancelled(
13550 DurableOperationCancelled {
13551 selection_group_id: self.handle.selection_group_id.clone(),
13552 member_key: self.handle.key.clone(),
13553 member_index: self.handle.index,
13554 operation_kind: self.handle.kind.clone(),
13555 operation_identity: self.handle.identity.clone(),
13556 },
13557 )));
13558 }
13559 }
13560 match recorded_selection_member_outcome(&state, &self.handle) {
13561 Ok(Some(result)) => {
13562 state.advance_cancellation_handle(&self.handle, false);
13563 Poll::Ready(Ok(result))
13564 }
13565 Ok(None) => Poll::Pending,
13566 Err(error) => {
13567 if !workflow_task_integrity_error(&error) {
13568 state.advance_cancellation_handle(&self.handle, true);
13569 }
13570 Poll::Ready(Err(error))
13571 }
13572 }
13573 }
13574}
13575
13576fn validate_selection_delivery_handle(
13577 state: &WorkflowState,
13578 handle: &DurableOperationHandle,
13579) -> Result<()> {
13580 let path = state
13581 .recorded_commands
13582 .iter()
13583 .find(|command| command.sequence() == handle.base_sequence)
13584 .and_then(|command| match command {
13585 RecordedCommand::Activity {
13586 parallel_group_path,
13587 ..
13588 }
13589 | RecordedCommand::ChildWorkflow {
13590 parallel_group_path,
13591 ..
13592 }
13593 | RecordedCommand::Timer {
13594 parallel_group_path,
13595 ..
13596 }
13597 | RecordedCommand::SignalWait {
13598 parallel_group_path,
13599 ..
13600 }
13601 | RecordedCommand::ConditionWait {
13602 parallel_group_path,
13603 ..
13604 } => parallel_group_path.as_ref(),
13605 _ => None,
13606 })
13607 .and_then(|path| path.first());
13608 let matches = path.is_some_and(|entry| {
13609 entry.parallel_group_mode.as_deref() == Some("select")
13610 && entry.parallel_group_id == handle.selection_group_id
13611 && entry.selection_member_key.as_ref() == Some(&handle.key)
13612 && entry.selection_member_index == Some(handle.index)
13613 && entry.selection_member_base_sequence == Some(handle.base_sequence)
13614 && entry.selection_member_size == Some(handle.size)
13615 && entry.selection_member_kind.as_deref() == Some(handle.kind.as_str())
13616 });
13617 if !matches
13618 || selection_operation_identity(state, &handle.kind, handle.base_sequence, handle.size)
13619 != handle.identity
13620 {
13621 return Err(invalid_recorded_history(
13622 "cooperative_cancellation_call_mismatch",
13623 handle.base_sequence,
13624 "selection handle matching its authored durable identity",
13625 &format!("{handle:?}"),
13626 "selection handle identity or member metadata changed before cancellation replay",
13627 ));
13628 }
13629 Ok(())
13630}
13631
13632pub struct CancelDurableOperationCall {
13634 handle: DurableOperationHandle,
13635 emitted: bool,
13636}
13637
13638impl Future for CancelDurableOperationCall {
13639 type Output = Result<()>;
13640
13641 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
13642 let ctx = self.handle.ctx.clone();
13643 let mut state = match ctx.state.lock() {
13644 Ok(state) => state,
13645 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
13646 };
13647 match selection_cancellation_for_handle(&state, &self.handle) {
13648 Err(error) => return Poll::Ready(Err(error)),
13649 Ok(true) => {
13650 state.advance_cancellation_receipt(
13651 &self.handle.selection_group_id,
13652 self.handle.base_sequence,
13653 );
13654 return Poll::Ready(Ok(()));
13655 }
13656 Ok(false) => {}
13657 }
13658 if recorded_selection_member_is_terminal(&state, &self.handle) {
13659 return Poll::Ready(Ok(()));
13660 }
13661 if !self.emitted {
13662 state.commands.push(json!({
13663 "type": "cancel_selection_operation",
13664 "selection_group_id": self.handle.selection_group_id,
13665 "member_key": self.handle.key,
13666 "member_index": self.handle.index,
13667 "member_base_sequence": self.handle.base_sequence,
13668 "member_size": self.handle.size,
13669 "operation_kind": self.handle.kind,
13670 "operation_identity": self.handle.identity,
13671 }));
13672 self.emitted = true;
13673 }
13674 Poll::Pending
13678 }
13679}
13680
13681trait PollNestedResultExt<T> {
13682 fn flatten_result(self) -> Poll<Result<T>>;
13683}
13684
13685impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
13686 fn flatten_result(self) -> Poll<Result<T>> {
13687 match self {
13688 Poll::Ready(Ok(result)) => Poll::Ready(result),
13689 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
13690 Poll::Pending => Poll::Pending,
13691 }
13692 }
13693}
13694
13695struct SagaCompensation {
13696 activity_type: String,
13697 options: ActivityOptions,
13698 arguments: AvroValue,
13699 registration_order: usize,
13700}
13701
13702pub struct Saga {
13710 ctx: WorkflowContext,
13711 compensations: Vec<SagaCompensation>,
13712}
13713
13714impl Saga {
13715 fn new(ctx: WorkflowContext) -> Self {
13716 Self {
13717 ctx,
13718 compensations: Vec::new(),
13719 }
13720 }
13721
13722 pub fn add_compensation<T: Serialize>(
13723 &mut self,
13724 activity_type: impl Into<String>,
13725 args: T,
13726 ) -> Result<&mut Self> {
13727 self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
13728 }
13729
13730 pub fn add_compensation_with_options<T: Serialize>(
13731 &mut self,
13732 activity_type: impl Into<String>,
13733 options: ActivityOptions,
13734 args: T,
13735 ) -> Result<&mut Self> {
13736 let activity_type = activity_type.into();
13737 if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
13738 return Err(Error::Codec(
13739 "saga compensation activity type must be non-empty without surrounding whitespace"
13740 .to_string(),
13741 ));
13742 }
13743 options.validate().map_err(Error::InvalidActivityOptions)?;
13744 let arguments = AvroValue::from_serialize(&args)?;
13745 let registration_order = self.compensations.len() + 1;
13746 self.compensations.push(SagaCompensation {
13747 activity_type,
13748 options,
13749 arguments,
13750 registration_order,
13751 });
13752 Ok(self)
13753 }
13754
13755 pub async fn compensate(mut self, initiating_failure: Error) -> Error {
13757 let _shield = match self.ctx.cancellation_shield() {
13758 Ok(shield) => shield,
13759 Err(error) => return error,
13760 };
13761 while let Some(compensation) = self.compensations.pop() {
13762 if let Err(compensation_failure) = self
13763 .ctx
13764 .activity_with_options(
13765 compensation.activity_type.clone(),
13766 compensation.options,
13767 compensation.arguments,
13768 )
13769 .await
13770 {
13771 if workflow_task_integrity_error(&compensation_failure) {
13772 return compensation_failure;
13773 }
13774 return Error::SagaCompensationFailed(SagaCompensationFailure {
13775 initiating_failure: Box::new(initiating_failure),
13776 compensation_failure: Box::new(compensation_failure),
13777 compensation_activity_type: compensation.activity_type,
13778 compensation_registration_order: compensation.registration_order,
13779 });
13780 }
13781 }
13782 initiating_failure
13783 }
13784
13785 pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
13787 match outcome {
13788 Ok(value) => Ok(value),
13789 Err(error) => Err(self.compensate(error).await),
13790 }
13791 }
13792}
13793
13794pub struct ActivityCall {
13795 ctx: WorkflowContext,
13796 activity_type: String,
13797 options: ActivityOptions,
13798 worker_session: Option<WorkerSessionOptions>,
13799 args: Option<Result<AvroValue>>,
13800 scheduled: bool,
13801 local: bool,
13802 local_result: Option<Arc<Mutex<Option<ActivityOutcome>>>>,
13803 parallel_group_path: Vec<ParallelGroupMetadata>,
13804}
13805
13806impl ActivityCall {
13807 fn poll_avro_value(
13808 mut self: Pin<&mut Self>,
13809 _cx: &mut TaskContext<'_>,
13810 ) -> Poll<Result<AvroValue>> {
13811 let ctx = self.ctx.clone();
13812 let mut state = match ctx.state.lock() {
13813 Ok(state) => state,
13814 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
13815 };
13816
13817 if let Some(result) = &self.local_result {
13818 return match result.lock() {
13819 Ok(mut result) => result.take().map_or(Poll::Pending, |result| {
13820 Poll::Ready(result.map_err(Error::ActivityFailed))
13821 }),
13822 Err(_) => Poll::Ready(Err(Error::WorkflowStatePoisoned)),
13823 };
13824 }
13825 if self.scheduled {
13826 return Poll::Pending;
13827 }
13828
13829 let options = match self.options.validate() {
13830 Ok(options) => options,
13831 Err(error) => {
13832 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
13833 }
13834 };
13835 if self.local {
13836 if self.worker_session.is_some() {
13837 return Poll::Ready(Err(Error::WorkerLoop(
13838 "local activities cannot use worker-session routing".into(),
13839 )));
13840 }
13841 if let Err(error) = local_activity::validate(&options) {
13842 return Poll::Ready(Err(error));
13843 }
13844 }
13845 let session = match self
13846 .worker_session
13847 .as_ref()
13848 .map(WorkerSessionOptions::to_wire)
13849 .transpose()
13850 {
13851 Ok(value) => value,
13852 Err(error) => return Poll::Ready(Err(error)),
13853 };
13854 let session_queue = session
13855 .as_ref()
13856 .and_then(|session| session["queue"].as_str());
13857 let task_queue = options
13858 .task_queue
13859 .clone()
13860 .or_else(|| session_queue.map(str::to_owned))
13861 .unwrap_or_else(|| state.task_queue.clone());
13862 if session_queue.is_some_and(|queue| queue != task_queue) {
13863 return Poll::Ready(Err(Error::WorkerLoop(
13864 "worker-session queue must match the activity task queue".into(),
13865 )));
13866 }
13867 let current_recorded_options = RecordedActivityOptions {
13868 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
13869 execution_mode: RecordedSnapshotValue::Known(self.local.then(|| "local".to_string())),
13870 retry_policy: current_activity_retry_snapshot(&options),
13871 worker_session: session,
13872 };
13873
13874 let cursor = state.command_cursor;
13875 let call_kind = if self.local {
13876 CancellationCallKind::LocalActivity
13877 } else {
13878 CancellationCallKind::Activity
13879 };
13880 if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) {
13881 return Poll::Ready(Err(error));
13882 }
13883 let recorded = match state.cancellation_replay_command(cursor, call_kind) {
13884 Ok(recorded) => recorded,
13885 Err(error) => return Poll::Ready(Err(error)),
13886 };
13887 if let Some(recorded) = recorded {
13888 let sequence = recorded.sequence();
13889 match recorded {
13890 RecordedCommand::Activity {
13891 activity_type,
13892 cancellation_policy,
13893 options: recorded_options,
13894 outcome,
13895 parallel_group_path,
13896 ..
13897 } => {
13898 let current_policy = options
13899 .cancellation_policy
13900 .unwrap_or(CancellationPolicy::TryCancel)
13901 .as_str();
13902 if cancellation_policy != current_policy {
13903 return Poll::Ready(Err(Error::NonDeterministicReplay(
13904 ReplayFailure::new(
13905 "activity_cancellation_policy_changed",
13906 Some(sequence),
13907 Some(cancellation_policy),
13908 Some(current_policy.to_string()),
13909 "Activity cancellation policy changed during replay",
13910 ),
13911 )));
13912 }
13913 if let Err(error) = ensure_parallel_path_matches(
13914 sequence,
13915 parallel_group_path.as_deref(),
13916 &self.parallel_group_path,
13917 ) {
13918 return Poll::Ready(Err(error));
13919 }
13920 if let Some(recorded_type) = activity_type {
13921 if recorded_type != self.activity_type {
13922 return Poll::Ready(Err(Error::NonDeterministicReplay(
13923 ReplayFailure::new(
13924 "recorded_command_detail_mismatch",
13925 Some(sequence),
13926 Some(format!("activity:{recorded_type}")),
13927 Some(format!("activity:{}", self.activity_type)),
13928 "recorded activity type differs from the current workflow command",
13929 ),
13930 )));
13931 }
13932 }
13933 if let Some(recorded_options) = recorded_options {
13934 if recorded_options.worker_session
13935 != current_recorded_options.worker_session
13936 {
13937 return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
13938 "activity_worker_session_mismatch", Some(sequence),
13939 Some(activity_options_description(&recorded_options)),
13940 Some(activity_options_description(¤t_recorded_options)),
13941 "recorded worker-session routing differs from the current activity",
13942 ))));
13943 }
13944 if !recorded_options
13945 .task_queue
13946 .matches_current(¤t_recorded_options.task_queue)
13947 {
13948 return Poll::Ready(Err(Error::NonDeterministicReplay(
13949 ReplayFailure::new(
13950 "activity_task_queue_mismatch",
13951 Some(sequence),
13952 Some(activity_options_description(&recorded_options)),
13953 Some(activity_options_description(¤t_recorded_options)),
13954 "recorded activity task queue differs from the current workflow command",
13955 ),
13956 )));
13957 }
13958 if (self.local
13959 && matches!(
13960 recorded_options.execution_mode,
13961 RecordedSnapshotValue::Unknown
13962 ))
13963 || !recorded_options
13964 .execution_mode
13965 .matches_current(¤t_recorded_options.execution_mode)
13966 {
13967 return Poll::Ready(Err(Error::NonDeterministicReplay(
13968 ReplayFailure::new(
13969 "activity_execution_mode_mismatch",
13970 Some(sequence),
13971 Some(activity_options_description(&recorded_options)),
13972 Some(activity_options_description(¤t_recorded_options)),
13973 "recorded activity execution mode differs from the current workflow command",
13974 ),
13975 )));
13976 }
13977 if !recorded_options
13978 .retry_policy
13979 .matches_current(¤t_recorded_options.retry_policy)
13980 {
13981 return Poll::Ready(Err(Error::NonDeterministicReplay(
13982 ReplayFailure::new(
13983 "activity_retry_policy_mismatch",
13984 Some(sequence),
13985 Some(activity_options_description(&recorded_options)),
13986 Some(activity_options_description(¤t_recorded_options)),
13987 "recorded activity retry policy differs from the current workflow command",
13988 ),
13989 )));
13990 }
13991 }
13992 if let Err(error) = state.replay_cancellation_at(cursor, call_kind) {
13993 return Poll::Ready(Err(error));
13994 }
13995 match state.prepare_scalar_cancellation(
13996 cursor,
13997 call_kind,
13998 &self.parallel_group_path,
13999 ) {
14000 Ok(true) => {
14001 self.scheduled = true;
14002 return Poll::Pending;
14003 }
14004 Ok(false) => {}
14005 Err(error) => return Poll::Ready(Err(error)),
14006 }
14007 state.command_cursor += 1;
14008 if let Some(outcome) = outcome {
14009 state.advance_cancellation_sequence(sequence, &self.parallel_group_path);
14010 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
14011 }
14012 state.matched_recorded_pending = true;
14013 self.scheduled = true;
14014 return Poll::Pending;
14015 }
14016 other => {
14017 return Poll::Ready(Err(command_mismatch(
14018 &other,
14019 format!("activity:{}", self.activity_type),
14020 )));
14021 }
14022 }
14023 }
14024
14025 if !self.scheduled {
14026 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
14027 Ok(args) => args,
14028 Err(error) => return Poll::Ready(Err(error)),
14029 };
14030 let arguments = normalize_avro_arguments(args);
14031 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
14032 Ok(envelope) => envelope,
14033 Err(error) => return Poll::Ready(Err(error)),
14034 };
14035
14036 let mut command = serde_json::Map::from_iter([
14037 (
14038 "type".to_string(),
14039 json!(if self.local {
14040 "record_local_activity"
14041 } else {
14042 "schedule_activity"
14043 }),
14044 ),
14045 (
14046 "activity_type".to_string(),
14047 json!(self.activity_type.clone()),
14048 ),
14049 ("queue".to_string(), json!(task_queue)),
14050 ("arguments".to_string(), envelope),
14051 ]);
14052 for (field, value) in [
14053 ("start_to_close_timeout", options.start_to_close_timeout),
14054 (
14055 "schedule_to_start_timeout",
14056 options.schedule_to_start_timeout,
14057 ),
14058 (
14059 "schedule_to_close_timeout",
14060 options.schedule_to_close_timeout,
14061 ),
14062 ("heartbeat_timeout", options.heartbeat_timeout),
14063 ] {
14064 if let Some(value) = value {
14065 command.insert(field.to_string(), json!(value));
14066 }
14067 }
14068 if let Some(retry_policy) = &options.retry_policy {
14069 command.insert("retry_policy".to_string(), retry_policy.clone());
14070 }
14071 if let Some(policy) = options.cancellation_policy {
14072 command.insert("cancellation_policy".to_string(), json!(policy.as_str()));
14073 }
14074 if let Some(session) = current_recorded_options.worker_session {
14075 command.insert("worker_session".into(), session);
14076 }
14077 apply_parallel_group_path(&mut command, &self.parallel_group_path);
14078 ctx.apply_scope_membership(&mut command);
14079 match state.prepare_scalar_cancellation(cursor, call_kind, &self.parallel_group_path) {
14080 Ok(true) => {
14081 self.scheduled = true;
14082 return Poll::Pending;
14083 }
14084 Ok(false) => {}
14085 Err(error) => return Poll::Ready(Err(error)),
14086 }
14087 if self.local {
14088 command.remove("queue");
14089 command.insert("execution_mode".into(), json!("local"));
14090 let result = Arc::new(Mutex::new(None));
14091 let command_index = state.commands.len();
14092 state.local_activity_requests.push(local_activity::Request {
14093 command_index,
14094 options,
14095 arguments,
14096 result: result.clone(),
14097 });
14098 self.local_result = Some(result);
14099 }
14100 state.commands.push(Value::Object(command));
14101 self.scheduled = true;
14102 }
14103
14104 Poll::Pending
14105 }
14106}
14107
14108impl Future for ActivityCall {
14109 type Output = Result<Value>;
14110
14111 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
14112 match self.poll_avro_value(cx) {
14113 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
14114 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
14115 Poll::Pending => Poll::Pending,
14116 }
14117 }
14118}
14119
14120pub struct TimerCall {
14122 ctx: WorkflowContext,
14123 delay_seconds: Option<u64>,
14124 scheduled: bool,
14125 matched_pending: bool,
14126 parallel_group_path: Vec<ParallelGroupMetadata>,
14127}
14128
14129impl Future for TimerCall {
14130 type Output = Result<()>;
14131
14132 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
14133 if self.matched_pending {
14134 return Poll::Pending;
14135 }
14136
14137 let ctx = self.ctx.clone();
14138 let Some(requested_delay) = self.delay_seconds else {
14139 return Poll::Ready(Err(Error::TimerDurationOverflow));
14140 };
14141 let mut state = match ctx.state.lock() {
14142 Ok(state) => state,
14143 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14144 };
14145
14146 let cursor = state.command_cursor;
14147 if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) {
14148 return Poll::Ready(Err(error));
14149 }
14150 let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Timer)
14151 {
14152 Ok(recorded) => recorded,
14153 Err(error) => return Poll::Ready(Err(error)),
14154 };
14155 if let Some(recorded) = recorded {
14156 match recorded {
14157 RecordedCommand::Timer {
14158 sequence,
14159 delay_seconds,
14160 fired,
14161 parallel_group_path,
14162 ..
14163 } => {
14164 if let Err(error) =
14165 state.validate_scope_cleanup_timer(&ctx.cancellation_scope_id, sequence)
14166 {
14167 return Poll::Ready(Err(error));
14168 }
14169 if let Err(error) = ensure_parallel_path_matches(
14170 sequence,
14171 parallel_group_path.as_deref(),
14172 &self.parallel_group_path,
14173 ) {
14174 return Poll::Ready(Err(error));
14175 }
14176 if delay_seconds != requested_delay {
14177 return Poll::Ready(Err(Error::NonDeterministicReplay(
14178 ReplayFailure::new(
14179 "timer_delay_mismatch",
14180 Some(sequence),
14181 Some(format!("timer:{delay_seconds}s")),
14182 Some(format!("timer:{requested_delay}s")),
14183 "recorded timer delay differs from the current workflow command",
14184 ),
14185 )));
14186 }
14187 if let Err(error) =
14188 state.replay_cancellation_at(cursor, CancellationCallKind::Timer)
14189 {
14190 return Poll::Ready(Err(error));
14191 }
14192 match state.prepare_scalar_cancellation(
14193 cursor,
14194 CancellationCallKind::Timer,
14195 &self.parallel_group_path,
14196 ) {
14197 Ok(true) => {
14198 self.scheduled = true;
14199 self.matched_pending = true;
14200 return Poll::Pending;
14201 }
14202 Ok(false) => {}
14203 Err(error) => return Poll::Ready(Err(error)),
14204 }
14205 state.command_cursor += 1;
14206 if fired {
14207 state.advance_cancellation_sequence(sequence, &self.parallel_group_path);
14208 return Poll::Ready(Ok(()));
14209 }
14210 state.matched_recorded_pending = true;
14211 self.scheduled = true;
14212 self.matched_pending = true;
14213 return Poll::Pending;
14214 }
14215 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
14216 }
14217 }
14218
14219 if !self.scheduled {
14220 let mut command = serde_json::Map::from_iter([
14221 ("type".to_string(), json!("start_timer")),
14222 ("delay_seconds".to_string(), json!(requested_delay)),
14223 ]);
14224 match state.scope_cleanup_timer_proof(&ctx.cancellation_scope_id) {
14225 Ok(Some(proof)) => {
14226 command.insert("cancellation_cleanup".into(), proof);
14227 }
14228 Ok(None) => {}
14229 Err(error) => return Poll::Ready(Err(error)),
14230 }
14231 apply_parallel_group_path(&mut command, &self.parallel_group_path);
14232 ctx.apply_scope_membership(&mut command);
14233 match state.prepare_scalar_cancellation(
14234 cursor,
14235 CancellationCallKind::Timer,
14236 &self.parallel_group_path,
14237 ) {
14238 Ok(true) => {
14239 self.scheduled = true;
14240 self.matched_pending = true;
14241 return Poll::Pending;
14242 }
14243 Ok(false) => {}
14244 Err(error) => return Poll::Ready(Err(error)),
14245 }
14246 state.commands.push(Value::Object(command));
14247 self.scheduled = true;
14248 }
14249
14250 Poll::Pending
14251 }
14252}
14253
14254pub struct ConditionWaitCall {
14256 ctx: WorkflowContext,
14257 options: ConditionWaitOptions,
14258 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
14259 occurrence_id: Option<String>,
14260 opened_wait: bool,
14261 parallel_group_path: Vec<ParallelGroupMetadata>,
14262}
14263
14264impl Future for ConditionWaitCall {
14265 type Output = Result<ConditionWaitResult>;
14266
14267 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
14268 if self.opened_wait {
14269 return Poll::Pending;
14270 }
14271
14272 let options = match self.options.validate() {
14273 Ok(options) => options,
14274 Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
14275 };
14276 let ctx = self.ctx.clone();
14277 let occurrence_id = match self.occurrence_id.as_ref() {
14278 Some(occurrence_id) => occurrence_id.clone(),
14279 None => {
14280 let mut state = match ctx.state.lock() {
14281 Ok(state) => state,
14282 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14283 };
14284 let ordinal = state.condition_wait_occurrence_counter;
14285 state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
14286 Some(next) => next,
14287 None => {
14288 return Poll::Ready(Err(Error::WorkerLoop(
14289 "condition wait occurrence counter overflowed".to_string(),
14290 )))
14291 }
14292 };
14293 let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
14294 drop(state);
14295 self.occurrence_id = Some(occurrence_id.clone());
14296 occurrence_id
14297 }
14298 };
14299
14300 let recorded_result = {
14301 let mut state = match ctx.state.lock() {
14302 Ok(state) => state,
14303 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14304 };
14305 let initial_cursor = state.command_cursor;
14306 if let Err(error) = ctx.validate_scope_membership(&mut state, initial_cursor) {
14307 return Poll::Ready(Err(error));
14308 }
14309 let recorded = match state
14310 .cancellation_replay_command(initial_cursor, CancellationCallKind::Condition)
14311 {
14312 Ok(recorded) => recorded,
14313 Err(error) => return Poll::Ready(Err(error)),
14314 };
14315 let Some(recorded) = recorded else {
14316 drop(state);
14317 return self.poll_new_condition(options);
14318 };
14319 if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
14320 return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
14321 }
14322
14323 let mut cursor = state.command_cursor;
14324 let mut result = None;
14325 let mut pending_delivery = false;
14326 loop {
14327 if cursor > initial_cursor
14328 && matches!(
14329 state.recorded_commands.get(cursor),
14330 Some(RecordedCommand::CancellationBoundary { original: None, .. })
14331 )
14332 {
14333 break;
14334 }
14335 let recorded = match state
14336 .cancellation_replay_command(cursor, CancellationCallKind::Condition)
14337 {
14338 Ok(recorded) => recorded,
14339 Err(error) => return Poll::Ready(Err(error)),
14340 };
14341 let Some(RecordedCommand::ConditionWait {
14342 sequence,
14343 occurrence_id: ref recorded_occurrence_id,
14344 ref condition_key,
14345 ref predicate_identity,
14346 timeout_seconds,
14347 result: recorded_result,
14348 ref parallel_group_path,
14349 ..
14350 }) = recorded
14351 else {
14352 break;
14353 };
14354
14355 if cursor > initial_cursor && recorded_occurrence_id != &occurrence_id {
14356 break;
14357 }
14358 if let Err(error) = ensure_parallel_path_matches(
14359 sequence,
14360 parallel_group_path.as_deref(),
14361 &self.parallel_group_path,
14362 ) {
14363 return Poll::Ready(Err(error));
14364 }
14365 if let Err(error) = validate_recorded_condition_wait(
14366 sequence,
14367 recorded_occurrence_id,
14368 condition_key.as_deref(),
14369 predicate_identity,
14370 timeout_seconds,
14371 &occurrence_id,
14372 &options,
14373 ) {
14374 return Poll::Ready(Err(error));
14375 }
14376 if result == Some(ConditionWaitResult::TimedOut) {
14377 return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
14378 "condition_wait_reopened_after_timeout",
14379 Some(sequence),
14380 Some("timed-out condition is terminal".to_string()),
14381 Some("another physical wait-open".to_string()),
14382 "condition history reopened one logical wait after its durable timeout",
14383 ))));
14384 }
14385 if let Err(error) =
14386 state.replay_cancellation_at(cursor, CancellationCallKind::Condition)
14387 {
14388 return Poll::Ready(Err(error));
14389 }
14390 match state.prepare_scalar_cancellation(
14391 cursor,
14392 CancellationCallKind::Condition,
14393 &self.parallel_group_path,
14394 ) {
14395 Ok(pending) => pending_delivery |= pending,
14396 Err(error) => return Poll::Ready(Err(error)),
14397 }
14398 result = recorded_result;
14399 if result.is_some() {
14400 state.advance_cancellation_sequence(sequence, &self.parallel_group_path);
14401 }
14402 cursor += 1;
14403 }
14404 state.command_cursor = cursor;
14405 if pending_delivery {
14406 self.opened_wait = true;
14407 return Poll::Pending;
14408 }
14409 result
14410 };
14411
14412 if let Some(result) = recorded_result {
14413 return Poll::Ready(Ok(result));
14414 }
14415
14416 self.poll_open_condition(options, true)
14417 }
14418}
14419
14420impl ConditionWaitCall {
14421 fn poll_new_condition(
14422 self: Pin<&mut Self>,
14423 options: ValidatedConditionWaitOptions,
14424 ) -> Poll<Result<ConditionWaitResult>> {
14425 self.poll_open_condition(options, false)
14426 }
14427
14428 fn poll_open_condition(
14429 mut self: Pin<&mut Self>,
14430 options: ValidatedConditionWaitOptions,
14431 recorded_wait: bool,
14432 ) -> Poll<Result<ConditionWaitResult>> {
14433 {
14434 let ctx = self.ctx.clone();
14435 let mut state = match ctx.state.lock() {
14436 Ok(state) => state,
14437 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14438 };
14439 let cursor = state.command_cursor;
14440 match state.prepare_scalar_cancellation(
14441 cursor,
14442 CancellationCallKind::Condition,
14443 &self.parallel_group_path,
14444 ) {
14445 Ok(true) => {
14446 self.opened_wait = true;
14447 return Poll::Pending;
14448 }
14449 Ok(false) => {}
14450 Err(error) => return Poll::Ready(Err(error)),
14451 }
14452 }
14453 let selection_member = self
14454 .parallel_group_path
14455 .first()
14456 .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
14457 match (self.predicate)() {
14458 Ok(true) if !selection_member || recorded_wait => {
14459 return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
14460 }
14461 Ok(_) => {}
14462 Err(error) => return Poll::Ready(Err(error)),
14463 }
14464 if options.timeout_seconds == Some(0) && !selection_member {
14465 return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
14466 }
14467
14468 let ctx = self.ctx.clone();
14469 let mut state = match ctx.state.lock() {
14470 Ok(state) => state,
14471 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14472 };
14473 let mut command = serde_json::Map::from_iter([
14474 ("type".to_string(), json!("open_condition_wait")),
14475 (
14476 "condition_wait_occurrence_id".to_string(),
14477 json!(self.occurrence_id.as_deref().unwrap_or_default()),
14478 ),
14479 ("condition_key".to_string(), json!(options.condition_key)),
14480 (
14481 "condition_definition_fingerprint".to_string(),
14482 json!(options.predicate_identity),
14483 ),
14484 ]);
14485 if let Some(timeout_seconds) = options.timeout_seconds {
14486 command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
14487 }
14488 apply_parallel_group_path(&mut command, &self.parallel_group_path);
14489 ctx.apply_scope_membership(&mut command);
14490 state.commands.push(Value::Object(command));
14491 drop(state);
14492 self.opened_wait = true;
14493 Poll::Pending
14494 }
14495}
14496
14497fn validate_recorded_condition_wait(
14498 sequence: u64,
14499 recorded_occurrence_id: &str,
14500 recorded_key: Option<&str>,
14501 recorded_predicate_identity: &str,
14502 recorded_timeout_seconds: Option<u64>,
14503 current_occurrence_id: &str,
14504 current: &ValidatedConditionWaitOptions,
14505) -> Result<()> {
14506 if recorded_occurrence_id != current_occurrence_id {
14507 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
14508 "condition_wait_occurrence_mismatch",
14509 Some(sequence),
14510 Some(recorded_occurrence_id.to_string()),
14511 Some(current_occurrence_id.to_string()),
14512 "recorded condition occurrence differs from the current authored wait position",
14513 )));
14514 }
14515 if recorded_key != Some(current.condition_key.as_str()) {
14516 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
14517 "condition_wait_key_mismatch",
14518 Some(sequence),
14519 recorded_key.map(str::to_string),
14520 Some(current.condition_key.clone()),
14521 "recorded condition identity differs from the current workflow wait",
14522 )));
14523 }
14524 if recorded_predicate_identity != current.predicate_identity {
14525 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
14526 "condition_wait_predicate_mismatch",
14527 Some(sequence),
14528 Some(recorded_predicate_identity.to_string()),
14529 Some(current.predicate_identity.clone()),
14530 "recorded condition predicate behavior differs from current workflow code",
14531 )));
14532 }
14533 if recorded_timeout_seconds != current.timeout_seconds {
14534 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
14535 "condition_wait_timeout_mismatch",
14536 Some(sequence),
14537 recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
14538 current.timeout_seconds.map(|seconds| format!("{seconds}s")),
14539 "recorded condition timeout differs from the current workflow wait",
14540 )));
14541 }
14542 Ok(())
14543}
14544
14545pub struct ChildWorkflowCall {
14547 ctx: WorkflowContext,
14548 workflow_type: String,
14549 options: ChildWorkflowOptions,
14550 args: Option<Result<AvroValue>>,
14551 scheduled: bool,
14552 matched_pending: bool,
14553 parallel_group_path: Vec<ParallelGroupMetadata>,
14554}
14555
14556impl ChildWorkflowCall {
14557 fn poll_avro_value(
14558 mut self: Pin<&mut Self>,
14559 _cx: &mut TaskContext<'_>,
14560 ) -> Poll<Result<ChildWorkflowAvroResult>> {
14561 if self.matched_pending {
14562 return Poll::Pending;
14563 }
14564
14565 let ctx = self.ctx.clone();
14566 let mut state = match ctx.state.lock() {
14567 Ok(state) => state,
14568 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14569 };
14570
14571 let cursor = state.command_cursor;
14572 if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) {
14573 return Poll::Ready(Err(error));
14574 }
14575 let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Child)
14576 {
14577 Ok(recorded) => recorded,
14578 Err(error) => return Poll::Ready(Err(error)),
14579 };
14580 if let Some(recorded) = recorded {
14581 let sequence = recorded.sequence();
14582 match recorded {
14583 RecordedCommand::ChildWorkflow {
14584 workflow_type,
14585 policies,
14586 outcome,
14587 parallel_group_path,
14588 ..
14589 } => {
14590 if let Err(error) = ensure_parallel_path_matches(
14591 sequence,
14592 parallel_group_path.as_deref(),
14593 &self.parallel_group_path,
14594 ) {
14595 return Poll::Ready(Err(error));
14596 }
14597 if let Some(recorded_type) = workflow_type {
14598 if recorded_type != self.workflow_type {
14599 return Poll::Ready(Err(Error::NonDeterministicReplay(
14600 ReplayFailure::new(
14601 "recorded_command_detail_mismatch",
14602 Some(sequence),
14603 Some(format!("child workflow:{recorded_type}")),
14604 Some(format!("child workflow:{}", self.workflow_type)),
14605 "recorded child workflow type differs from the current workflow command",
14606 ),
14607 )));
14608 }
14609 }
14610 if let Err(error) =
14611 ensure_child_policies_match(sequence, &policies, &self.options)
14612 {
14613 return Poll::Ready(Err(error));
14614 }
14615 if let Err(error) =
14616 state.replay_cancellation_at(cursor, CancellationCallKind::Child)
14617 {
14618 return Poll::Ready(Err(error));
14619 }
14620 match state.prepare_scalar_cancellation(
14621 cursor,
14622 CancellationCallKind::Child,
14623 &self.parallel_group_path,
14624 ) {
14625 Ok(true) => {
14626 self.scheduled = true;
14627 self.matched_pending = true;
14628 return Poll::Pending;
14629 }
14630 Ok(false) => {}
14631 Err(error) => return Poll::Ready(Err(error)),
14632 }
14633 state.command_cursor += 1;
14634 if let Some(outcome) = outcome {
14635 state.advance_cancellation_sequence(sequence, &self.parallel_group_path);
14636 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
14637 }
14638 state.matched_recorded_pending = true;
14639 self.scheduled = true;
14640 self.matched_pending = true;
14641 return Poll::Pending;
14642 }
14643 other => {
14644 return Poll::Ready(Err(command_mismatch(
14645 &other,
14646 format!("child workflow:{}", self.workflow_type),
14647 )));
14648 }
14649 }
14650 }
14651
14652 if !self.scheduled {
14653 if self.options.task_queue.trim().is_empty() {
14654 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
14655 "task_queue must not be empty".to_string(),
14656 )));
14657 }
14658 for (name, value) in [
14659 (
14660 "execution_timeout_seconds",
14661 self.options.execution_timeout_seconds,
14662 ),
14663 ("run_timeout_seconds", self.options.run_timeout_seconds),
14664 ] {
14665 if value == Some(0) {
14666 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
14667 "{name} must be at least 1"
14668 ))));
14669 }
14670 }
14671
14672 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
14673 Ok(args) => args,
14674 Err(error) => return Poll::Ready(Err(error)),
14675 };
14676 let arguments = match encode_typed_envelope(
14677 &normalize_avro_arguments(args),
14678 &state.payload_codec,
14679 ) {
14680 Ok(arguments) => arguments,
14681 Err(error) => return Poll::Ready(Err(error)),
14682 };
14683 let mut command = json!({
14684 "type": "start_child_workflow",
14685 "workflow_type": self.workflow_type,
14686 "queue": self.options.task_queue,
14687 "parent_close_policy": self.options.parent_close_policy.as_str(),
14688 "arguments": arguments,
14689 });
14690 let object = command
14691 .as_object_mut()
14692 .expect("child workflow command is always an object");
14693 if self.options.cancellation_policy != CancellationPolicy::Abandon {
14694 object.insert(
14695 "cancellation_policy".to_string(),
14696 json!(self.options.cancellation_policy.as_str()),
14697 );
14698 }
14699 if let Some(policy) = &self.options.retry_policy {
14700 let mut retry_policy = serde_json::Map::new();
14701 if let Some(max_attempts) = policy.max_attempts {
14702 if max_attempts == 0 {
14703 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
14704 "retry_policy.max_attempts must be at least 1".to_string(),
14705 )));
14706 }
14707 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
14708 }
14709 if !policy.backoff_seconds.is_empty() {
14710 retry_policy
14711 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
14712 }
14713 if !policy.non_retryable_error_types.is_empty() {
14714 retry_policy.insert(
14715 "non_retryable_error_types".to_string(),
14716 json!(policy.non_retryable_error_types),
14717 );
14718 }
14719 if retry_policy.is_empty() {
14720 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
14721 "retry_policy must configure at least one field".to_string(),
14722 )));
14723 }
14724 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
14725 }
14726 if let Some(seconds) = self.options.execution_timeout_seconds {
14727 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
14728 }
14729 if let Some(seconds) = self.options.run_timeout_seconds {
14730 object.insert("run_timeout_seconds".to_string(), json!(seconds));
14731 }
14732 apply_parallel_group_path(object, &self.parallel_group_path);
14733 ctx.apply_scope_membership(object);
14734 match state.prepare_scalar_cancellation(
14735 cursor,
14736 CancellationCallKind::Child,
14737 &self.parallel_group_path,
14738 ) {
14739 Ok(true) => {
14740 self.scheduled = true;
14741 self.matched_pending = true;
14742 return Poll::Pending;
14743 }
14744 Ok(false) => {}
14745 Err(error) => return Poll::Ready(Err(error)),
14746 }
14747 state.commands.push(command);
14748 self.scheduled = true;
14749 }
14750
14751 Poll::Pending
14752 }
14753}
14754
14755impl Future for ChildWorkflowCall {
14756 type Output = Result<ChildWorkflowResult>;
14757
14758 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
14759 match self.poll_avro_value(cx) {
14760 Poll::Ready(Ok(result)) => match result.result.into_json() {
14761 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
14762 parent: result.parent,
14763 child: result.child,
14764 child_workflow_type: result.child_workflow_type,
14765 result: projected,
14766 })),
14767 Err(error) => Poll::Ready(Err(error)),
14768 },
14769 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
14770 Poll::Pending => Poll::Pending,
14771 }
14772 }
14773}
14774
14775fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
14776 Error::NonDeterministicReplay(ReplayFailure::new(
14777 "recorded_command_mismatch",
14778 Some(recorded.sequence()),
14779 Some(recorded.shape().to_string()),
14780 Some(actual.into()),
14781 "current workflow command does not match the recorded durable command sequence",
14782 ))
14783}
14784
14785pub struct SignalCall {
14786 ctx: WorkflowContext,
14787 signal_name: String,
14788 runtime_reserved_allowed: bool,
14789 opened_wait: bool,
14790 matched_pending: bool,
14791 parallel_group_path: Vec<ParallelGroupMetadata>,
14792}
14793
14794impl SignalCall {
14795 fn poll_avro_value(
14796 mut self: Pin<&mut Self>,
14797 _cx: &mut TaskContext<'_>,
14798 ) -> Poll<Result<Vec<AvroValue>>> {
14799 if self.matched_pending {
14800 return Poll::Pending;
14801 }
14802 if !self.runtime_reserved_allowed {
14803 if let Err(error) = validate_user_signal_name(&self.signal_name) {
14804 return Poll::Ready(Err(error));
14805 }
14806 }
14807
14808 let ctx = self.ctx.clone();
14809 let mut state = match ctx.state.lock() {
14810 Ok(state) => state,
14811 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
14812 };
14813
14814 let cursor = state.command_cursor;
14815 if let Err(error) = ctx.validate_scope_membership(&mut state, cursor) {
14816 return Poll::Ready(Err(error));
14817 }
14818 let recorded = match state.cancellation_replay_command(cursor, CancellationCallKind::Signal)
14819 {
14820 Ok(recorded) => recorded,
14821 Err(error) => return Poll::Ready(Err(error)),
14822 };
14823 if let Some(recorded) = recorded {
14824 match recorded {
14825 RecordedCommand::SignalWait {
14826 sequence,
14827 signal_name,
14828 value,
14829 parallel_group_path,
14830 } => {
14831 if let Err(error) = ensure_parallel_path_matches(
14832 sequence,
14833 parallel_group_path.as_deref(),
14834 &self.parallel_group_path,
14835 ) {
14836 return Poll::Ready(Err(error));
14837 }
14838 if signal_name != self.signal_name {
14839 return Poll::Ready(Err(Error::NonDeterministicReplay(
14840 ReplayFailure::new(
14841 "recorded_command_detail_mismatch",
14842 Some(sequence),
14843 Some(format!("signal wait:{signal_name}")),
14844 Some(format!("signal wait:{}", self.signal_name)),
14845 "recorded signal name differs from the current workflow command",
14846 ),
14847 )));
14848 }
14849
14850 if let Err(error) =
14851 state.replay_cancellation_at(cursor, CancellationCallKind::Signal)
14852 {
14853 return Poll::Ready(Err(error));
14854 }
14855 match state.prepare_scalar_cancellation(
14856 cursor,
14857 CancellationCallKind::Signal,
14858 &self.parallel_group_path,
14859 ) {
14860 Ok(true) => {
14861 self.opened_wait = true;
14862 self.matched_pending = true;
14863 return Poll::Pending;
14864 }
14865 Ok(false) => {}
14866 Err(error) => return Poll::Ready(Err(error)),
14867 }
14868 state.command_cursor += 1;
14869 if let Some(value) = value {
14870 state.advance_cancellation_sequence(sequence, &self.parallel_group_path);
14871 return Poll::Ready(Ok(value));
14872 }
14873 if state
14874 .resume_signal
14875 .as_ref()
14876 .is_some_and(|signal| signal.signal_name == self.signal_name)
14877 {
14878 let signal = state
14879 .resume_signal
14880 .take()
14881 .expect("matching resume signal is present");
14882 state.unavailable_cancellation_boundary(&self.parallel_group_path);
14883 return Poll::Ready(Ok(signal.arguments));
14884 }
14885
14886 state.matched_recorded_pending = true;
14887 self.opened_wait = true;
14888 self.matched_pending = true;
14889 return Poll::Pending;
14890 }
14891 other => {
14892 return Poll::Ready(Err(command_mismatch(
14893 &other,
14894 format!("signal wait:{}", self.signal_name),
14895 )));
14896 }
14897 }
14898 }
14899
14900 match state.prepare_scalar_cancellation(
14901 cursor,
14902 CancellationCallKind::Signal,
14903 &self.parallel_group_path,
14904 ) {
14905 Ok(true) => {
14906 self.opened_wait = true;
14907 self.matched_pending = true;
14908 return Poll::Pending;
14909 }
14910 Ok(false) => {}
14911 Err(error) => return Poll::Ready(Err(error)),
14912 }
14913
14914 if state
14915 .resume_signal
14916 .as_ref()
14917 .is_some_and(|signal| signal.signal_name == self.signal_name)
14918 {
14919 let signal = state
14920 .resume_signal
14921 .take()
14922 .expect("matching resume signal is present");
14923 state.unavailable_cancellation_boundary(&self.parallel_group_path);
14924 return Poll::Ready(Ok(signal.arguments));
14925 }
14926
14927 if !self.opened_wait {
14928 let mut command = serde_json::Map::from_iter([
14929 ("type".to_string(), json!("open_signal_wait")),
14930 ("signal_name".to_string(), json!(self.signal_name)),
14931 ]);
14932 apply_parallel_group_path(&mut command, &self.parallel_group_path);
14933 ctx.apply_scope_membership(&mut command);
14934 state.commands.push(Value::Object(command));
14935 self.opened_wait = true;
14936 }
14937
14938 Poll::Pending
14939 }
14940}
14941
14942impl Future for SignalCall {
14943 type Output = Result<Vec<Value>>;
14944
14945 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
14946 match self.poll_avro_value(cx) {
14947 Poll::Ready(Ok(values)) => Poll::Ready(
14948 values
14949 .into_iter()
14950 .map(AvroValue::into_json)
14951 .collect::<Result<Vec<_>>>(),
14952 ),
14953 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
14954 Poll::Pending => Poll::Pending,
14955 }
14956 }
14957}
14958
14959#[derive(Clone, Debug)]
14960pub struct ActivityContext {
14961 client: Client,
14962 pub task_id: String,
14963 pub activity_attempt_id: String,
14964 pub lease_owner: String,
14965 pub activity_type: String,
14966 pub attempt_number: u64,
14967 pub task_queue: String,
14968 pub worker_id: String,
14969 claim_guard: Option<cooperative_cancellation::ActivityClaimGuard>,
14970 local_heartbeats: Option<Arc<Mutex<local_activity::Heartbeats>>>,
14971 worker_session: Option<WorkerSession>,
14972}
14973
14974impl ActivityContext {
14975 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
14976 if let Some(heartbeats) = &self.local_heartbeats {
14977 return heartbeats
14978 .lock()
14979 .map_err(|_| Error::WorkflowStatePoisoned)?
14980 .record(details);
14981 }
14982 if let Some(guard) = &self.claim_guard {
14983 return guard.heartbeat(self, details).await;
14984 }
14985 if let Some(session) = &self.worker_session {
14986 if !session.active() {
14987 return Err(Error::ActivityExecutionAbandoned(
14988 "worker-session lease or TTL is no longer active".into(),
14989 ));
14990 }
14991 let result = self
14992 .client
14993 .heartbeat_activity_task_value(
14994 &self.task_id,
14995 &self.activity_attempt_id,
14996 &self.lease_owner,
14997 details,
14998 )
14999 .await;
15000 return worker_session::settle_activity_heartbeat(
15001 session,
15002 result,
15003 &self.task_id,
15004 &self.activity_attempt_id,
15005 &self.lease_owner,
15006 );
15007 }
15008 self.client
15009 .heartbeat_activity_task(
15010 &self.task_id,
15011 &self.activity_attempt_id,
15012 &self.lease_owner,
15013 details,
15014 )
15015 .await
15016 }
15017}
15018
15019fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
15020 validate_payload_codec(codec)?;
15021 match value {
15022 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
15023 value, codec,
15024 )?)),
15025 None => Ok(AvroValue::Array(Vec::new())),
15026 }
15027}
15028
15029fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
15030 let Some(signal_name) = task
15031 .signal_name
15032 .as_deref()
15033 .filter(|value| !value.is_empty())
15034 else {
15035 return Ok(None);
15036 };
15037 let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
15038 let AvroValue::Array(arguments) = decoded else {
15039 unreachable!("normalize_avro_arguments always returns an array");
15040 };
15041
15042 Ok(Some(ResumeSignal {
15043 signal_name: signal_name.to_string(),
15044 arguments,
15045 }))
15046}
15047
15048fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
15049 validate_payload_codec(&task.payload_codec)?;
15050 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
15051 validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
15052 for event in &task.history_events {
15053 validate_history_event_payloads(event, &task.payload_codec)?;
15054 }
15055 Ok(())
15056}
15057
15058fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
15059 validate_payload_codec(&task.payload_codec)?;
15060 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
15061}
15062
15063fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
15064 validate_payload_codec(&task.payload_codec)?;
15065 validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
15066 validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
15067 for event in &task.history_events {
15068 validate_history_event_payloads(event, &task.payload_codec)?;
15069 }
15070
15071 let Some(export) = task.history_export.as_ref() else {
15072 return Ok(());
15073 };
15074 let export_codec = match export.get("payloads") {
15075 Some(payloads) => declared_payload_codec(payloads, "codec")?,
15076 None => None,
15077 }
15078 .unwrap_or(&task.payload_codec);
15079 validate_payload_codec(export_codec)?;
15080
15081 if let Some(events) = export.get("history_events").and_then(Value::as_array) {
15082 for event in events {
15083 let event_type = event
15084 .get("event_type")
15085 .or_else(|| event.get("type"))
15086 .and_then(Value::as_str)
15087 .unwrap_or_default();
15088 if let Some(payload) = event.get("payload") {
15089 validate_history_payloads(event_type, payload, export_codec)?;
15090 }
15091 }
15092 }
15093 for signal in export
15094 .get("signals")
15095 .and_then(Value::as_array)
15096 .into_iter()
15097 .flatten()
15098 {
15099 let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
15100 validate_payload_codec(codec)?;
15101 validate_optional_inbound_payload(signal.get("arguments"), codec)?;
15102 }
15103 for activity in export
15104 .get("activities")
15105 .and_then(Value::as_array)
15106 .into_iter()
15107 .flatten()
15108 {
15109 let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
15110 validate_payload_codec(codec)?;
15111 validate_optional_inbound_payload(activity.get("arguments"), codec)?;
15112 validate_optional_inbound_payload(activity.get("result"), codec)?;
15113 }
15114 Ok(())
15115}
15116
15117fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
15118 validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
15119}
15120
15121fn validate_history_payloads(
15122 event_type: &str,
15123 payload: &Value,
15124 fallback_codec: &str,
15125) -> Result<()> {
15126 let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
15127 validate_payload_codec(codec)?;
15128 for field in history_payload_fields(event_type) {
15129 validate_optional_inbound_payload(payload.get(*field), codec)?;
15130 }
15131 Ok(())
15132}
15133
15134const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
15135
15136fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
15137 match event_type {
15138 "ActivityCompleted" => &["result"],
15139 "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
15140 "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
15141 "UpdateCompleted" | "SideEffectRecorded" => &["result"],
15142 "ChildRunCompleted" => &["result", "output"],
15143 "WorkflowCompleted" => &["output"],
15144 "ServiceCallStarted"
15145 | "ServiceCallCompleted"
15146 | "ServiceCallFailed"
15147 | "ServiceCallCancelled" => &["request_payload", "response_payload"],
15148 _ => &[],
15149 }
15150}
15151
15152fn signal_history_payload(payload: &Value) -> Option<&Value> {
15153 SIGNAL_HISTORY_PAYLOAD_FIELDS
15154 .iter()
15155 .find_map(|field| payload.get(*field))
15156}
15157
15158fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
15159 match value.get(field) {
15160 None => Ok(None),
15161 Some(Value::String(codec)) => Ok(Some(codec)),
15162 Some(_) => Err(invalid_payload_envelope()),
15163 }
15164}
15165
15166fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
15167 validate_payload_codec(codec)?;
15168 if let Some(value) = value.filter(|value| !value.is_null()) {
15169 decode_wire_avro_value(value, codec)?;
15170 }
15171 Ok(())
15172}
15173
15174fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
15175 let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
15176 invalid_recorded_history(
15177 "parallel_group_metadata_invalid",
15178 sequence,
15179 "non-empty parallel_group_id",
15180 &payload.to_string(),
15181 "parallel-group history is missing its stable identity",
15182 )
15183 })?;
15184 let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
15185 invalid_recorded_history(
15186 "parallel_group_metadata_invalid",
15187 sequence,
15188 "activity, child, timer, signal, condition, or mixed group kind",
15189 &payload.to_string(),
15190 "parallel-group history is missing its group kind",
15191 )
15192 })?;
15193 if !matches!(
15194 kind.as_str(),
15195 "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
15196 ) {
15197 return Err(invalid_recorded_history(
15198 "parallel_group_metadata_invalid",
15199 sequence,
15200 "activity, child, timer, signal, condition, or mixed group kind",
15201 &kind,
15202 "parallel-group history contains an unsupported group kind",
15203 ));
15204 }
15205 let base_sequence = payload
15206 .get("parallel_group_base_sequence")
15207 .and_then(value_as_u64)
15208 .filter(|value| *value > 0)
15209 .ok_or_else(|| {
15210 invalid_recorded_history(
15211 "parallel_group_metadata_invalid",
15212 sequence,
15213 "positive parallel_group_base_sequence",
15214 &payload.to_string(),
15215 "parallel-group history contains an invalid base sequence",
15216 )
15217 })?;
15218 let size = payload
15219 .get("parallel_group_size")
15220 .and_then(value_as_u64)
15221 .and_then(|value| usize::try_from(value).ok())
15222 .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
15223 .ok_or_else(|| {
15224 invalid_recorded_history(
15225 "parallel_group_metadata_invalid",
15226 sequence,
15227 "bounded positive parallel_group_size",
15228 &payload.to_string(),
15229 "parallel-group history contains an invalid group size",
15230 )
15231 })?;
15232 let index = payload
15233 .get("parallel_group_index")
15234 .and_then(value_as_u64)
15235 .and_then(|value| usize::try_from(value).ok())
15236 .filter(|value| *value < size)
15237 .ok_or_else(|| {
15238 invalid_recorded_history(
15239 "parallel_group_metadata_invalid",
15240 sequence,
15241 "parallel_group_index within group bounds",
15242 &payload.to_string(),
15243 "parallel-group history contains an invalid member index",
15244 )
15245 })?;
15246 if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
15247 return Err(invalid_recorded_history(
15248 "parallel_group_metadata_invalid",
15249 sequence,
15250 "base sequence plus member index equals workflow sequence",
15251 &payload.to_string(),
15252 "parallel-group path does not preserve durable workflow position",
15253 ));
15254 }
15255 let mode = payload
15256 .get("parallel_group_mode")
15257 .and_then(Value::as_str)
15258 .unwrap_or("all");
15259 if !matches!(mode, "all" | "select") {
15260 return Err(invalid_recorded_history(
15261 "parallel_group_metadata_invalid",
15262 sequence,
15263 "parallel group mode all or select",
15264 mode,
15265 "parallel-group history contains an unsupported group mode",
15266 ));
15267 }
15268 let expected_id = if mode == "select" {
15269 format!("select-calls:{base_sequence}:{size}")
15270 } else {
15271 format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
15272 };
15273 if group_id != expected_id {
15274 return Err(invalid_recorded_history(
15275 "parallel_group_metadata_invalid",
15276 sequence,
15277 &expected_id,
15278 &group_id,
15279 "parallel-group history contains an incompatible stable group ID",
15280 ));
15281 }
15282 let selection_member_key = if mode == "select" {
15283 Some(selection_key_from_value(
15284 payload.get("selection_member_key"),
15285 sequence,
15286 )?)
15287 } else {
15288 None
15289 };
15290 let selection_member_index = if mode == "select" {
15291 Some(required_parallel_usize(
15292 payload,
15293 "selection_member_index",
15294 sequence,
15295 )?)
15296 } else {
15297 None
15298 };
15299 let selection_member_base_sequence = if mode == "select" {
15300 Some(
15301 payload
15302 .get("selection_member_base_sequence")
15303 .and_then(value_as_u64)
15304 .filter(|value| *value >= base_sequence)
15305 .ok_or_else(|| {
15306 invalid_recorded_history(
15307 "parallel_group_metadata_invalid",
15308 sequence,
15309 "selection member base within its group",
15310 &payload.to_string(),
15311 "selection history contains an invalid member base sequence",
15312 )
15313 })?,
15314 )
15315 } else {
15316 None
15317 };
15318 let selection_member_size = if mode == "select" {
15319 let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
15320 if member_size == 0 {
15321 return Err(invalid_recorded_history(
15322 "parallel_group_metadata_invalid",
15323 sequence,
15324 "positive selection member size",
15325 &payload.to_string(),
15326 "selection history contains an invalid member size",
15327 ));
15328 }
15329 Some(member_size)
15330 } else {
15331 None
15332 };
15333 let selection_member_kind = if mode == "select" {
15334 let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
15335 invalid_recorded_history(
15336 "parallel_group_metadata_invalid",
15337 sequence,
15338 "selection member operation kind",
15339 &payload.to_string(),
15340 "selection history is missing its authored member kind",
15341 )
15342 })?;
15343 if !matches!(
15344 kind.as_str(),
15345 "activity" | "child" | "timer" | "signal" | "condition" | "group"
15346 ) {
15347 return Err(invalid_recorded_history(
15348 "parallel_group_metadata_invalid",
15349 sequence,
15350 "activity, child, timer, signal, condition, or group selection member kind",
15351 &kind,
15352 "selection history contains an unsupported member kind",
15353 ));
15354 }
15355 Some(kind)
15356 } else {
15357 None
15358 };
15359 if let (Some(member_base), Some(member_size)) =
15360 (selection_member_base_sequence, selection_member_size)
15361 {
15362 let member_end = member_base
15363 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
15364 .ok_or_else(|| {
15365 invalid_recorded_history(
15366 "parallel_group_metadata_invalid",
15367 sequence,
15368 "bounded selection member range",
15369 &payload.to_string(),
15370 "selection member range overflowed",
15371 )
15372 })?;
15373 let group_end = base_sequence
15374 .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
15375 .unwrap_or(u64::MAX);
15376 if sequence < member_base || sequence >= member_end || member_end > group_end {
15377 return Err(invalid_recorded_history(
15378 "parallel_group_metadata_invalid",
15379 sequence,
15380 "workflow sequence within one bounded selection member",
15381 &payload.to_string(),
15382 "selection member range does not contain its durable leaf",
15383 ));
15384 }
15385 }
15386 Ok(ParallelGroupMetadata {
15387 parallel_group_id: group_id,
15388 parallel_group_kind: kind,
15389 parallel_group_base_sequence: base_sequence,
15390 parallel_group_size: size,
15391 parallel_group_index: index,
15392 parallel_group_mode: (mode == "select").then(|| "select".to_string()),
15393 selection_member_key,
15394 selection_member_index,
15395 selection_member_base_sequence,
15396 selection_member_size,
15397 selection_member_kind,
15398 })
15399}
15400
15401fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
15402 payload
15403 .get(field)
15404 .and_then(value_as_u64)
15405 .and_then(|value| usize::try_from(value).ok())
15406 .ok_or_else(|| {
15407 invalid_recorded_history(
15408 "parallel_group_metadata_invalid",
15409 sequence,
15410 &format!("non-negative integer {field}"),
15411 &payload.to_string(),
15412 "selection history contains invalid member metadata",
15413 )
15414 })
15415}
15416
15417fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
15418 match value {
15419 Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
15420 Some(value) => value_as_u64(value)
15421 .and_then(|value| usize::try_from(value).ok())
15422 .map(SelectionKey::Index)
15423 .ok_or_else(|| {
15424 invalid_recorded_history(
15425 "selection_member_key_invalid",
15426 sequence,
15427 "non-empty string or non-negative integer member key",
15428 &value.to_string(),
15429 "selection history contains an invalid member key",
15430 )
15431 }),
15432 None => Err(invalid_recorded_history(
15433 "selection_member_key_missing",
15434 sequence,
15435 "selection_member_key",
15436 "<missing>",
15437 "selection history is missing its stable member key",
15438 )),
15439 }
15440}
15441
15442fn recorded_parallel_group_path(
15443 events: &[&HistoryEvent],
15444 sequence: u64,
15445) -> Result<Option<Vec<ParallelGroupMetadata>>> {
15446 let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
15447 for event in events {
15448 let payload = &event.payload;
15449 let has_metadata = payload.get("parallel_group_path").is_some()
15450 || payload.get("parallel_group_id").is_some()
15451 || payload.get("parallel_group_kind").is_some()
15452 || payload.get("parallel_group_base_sequence").is_some()
15453 || payload.get("parallel_group_size").is_some()
15454 || payload.get("parallel_group_index").is_some()
15455 || payload.get("parallel_group_mode").is_some()
15456 || payload.get("selection_member_key").is_some();
15457 if !has_metadata {
15458 continue;
15459 }
15460
15461 let top_level = recorded_parallel_group_entry(payload, sequence)?;
15462 let path = match payload.get("parallel_group_path") {
15463 None => vec![top_level.clone()],
15464 Some(Value::Array(entries)) if !entries.is_empty() => entries
15465 .iter()
15466 .map(|entry| recorded_parallel_group_entry(entry, sequence))
15467 .collect::<Result<Vec<_>>>()?,
15468 Some(value) => {
15469 return Err(invalid_recorded_history(
15470 "parallel_group_metadata_invalid",
15471 sequence,
15472 "non-empty parallel_group_path list",
15473 &value.to_string(),
15474 "parallel-group history contains an invalid group path",
15475 ));
15476 }
15477 };
15478 if path.last() != Some(&top_level) {
15479 return Err(invalid_recorded_history(
15480 "parallel_group_metadata_invalid",
15481 sequence,
15482 &serde_json::to_string(&path.last()).unwrap_or_default(),
15483 &serde_json::to_string(&top_level).unwrap_or_default(),
15484 "parallel-group top-level fields do not match the innermost path entry",
15485 ));
15486 }
15487 if recorded.as_ref().is_some_and(|existing| existing != &path) {
15488 return Err(invalid_recorded_history(
15489 "parallel_group_history_conflict",
15490 sequence,
15491 &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
15492 &serde_json::to_string(&path).unwrap_or_default(),
15493 "parallel-group metadata changed between scheduling and resolution history",
15494 ));
15495 }
15496 recorded = Some(path);
15497 }
15498 Ok(recorded)
15499}
15500
15501fn recorded_commands(
15502 events: &[HistoryEvent],
15503 fallback_codec: &str,
15504 parent: WorkflowIdentity,
15505) -> Result<Vec<RecordedCommand>> {
15506 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
15507 let mut last_new_sequence = None;
15508
15509 for event in events {
15510 let is_activity = matches!(
15511 event.event_type.as_str(),
15512 "ActivityScheduled"
15513 | "ActivityStarted"
15514 | "ActivityHeartbeatRecorded"
15515 | "ActivityRetryScheduled"
15516 | "ActivityCompleted"
15517 | "ActivityFailed"
15518 | "ActivityCancelled"
15519 | "ActivityTimedOut"
15520 );
15521 let is_workflow_timer = matches!(
15522 event.event_type.as_str(),
15523 "TimerScheduled" | "TimerCancelled" | "TimerFired"
15524 ) && !is_internal_timer_event(event);
15525 let is_child_workflow = matches!(
15526 event.event_type.as_str(),
15527 "ChildWorkflowScheduled"
15528 | "ChildRunStarted"
15529 | "ChildRunCompleted"
15530 | "ChildRunFailed"
15531 | "ChildRunCancelled"
15532 | "ChildRunTerminated"
15533 );
15534 let is_signal_wait = is_recorded_signal_wait_event(event);
15535 let is_condition_wait = is_recorded_condition_wait_event(event);
15536 let is_search_attributes = event.event_type == "SearchAttributesUpserted";
15537 let is_side_effect = event.event_type == "SideEffectRecorded";
15538 let is_version_marker = event.event_type == "VersionMarkerRecorded";
15539 let is_memo = event.event_type == "MemoUpserted";
15540 if !is_activity
15541 && !is_workflow_timer
15542 && !is_child_workflow
15543 && !is_signal_wait
15544 && !is_condition_wait
15545 && !is_search_attributes
15546 && !is_side_effect
15547 && !is_version_marker
15548 && !is_memo
15549 {
15550 continue;
15551 }
15552
15553 let sequence = durable_event_sequence(event).ok_or_else(|| {
15554 Error::NonDeterministicReplay(ReplayFailure::new(
15555 "durable_command_sequence_missing",
15556 None,
15557 Some("positive workflow sequence".to_string()),
15558 Some(event.event_type.clone()),
15559 "durable command history event has no workflow sequence",
15560 ))
15561 })?;
15562 if sequence == 0 {
15563 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
15564 "durable_command_sequence_invalid",
15565 Some(sequence),
15566 Some("positive workflow sequence".to_string()),
15567 Some(sequence.to_string()),
15568 "durable command history uses an invalid workflow sequence",
15569 )));
15570 }
15571 if !events_by_sequence.contains_key(&sequence) {
15572 if let Some(previous) = last_new_sequence {
15573 if sequence < previous {
15574 return Err(invalid_recorded_history(
15575 "durable_command_sequence_mismatch",
15576 sequence,
15577 &format!("workflow sequence greater than {previous}"),
15578 &sequence.to_string(),
15579 "durable commands are not strictly ordered by their recorded workflow sequence",
15580 ));
15581 }
15582 }
15583 last_new_sequence = Some(sequence);
15584 }
15585 events_by_sequence.entry(sequence).or_default().push(event);
15586 }
15587
15588 let commands: Vec<RecordedCommand> = events_by_sequence
15589 .into_iter()
15590 .map(|(sequence, sequence_events)| {
15591 let activity_events: Vec<_> = sequence_events
15592 .iter()
15593 .copied()
15594 .filter(|event| event.event_type.starts_with("Activity"))
15595 .collect();
15596 let timer_events: Vec<_> = sequence_events
15597 .iter()
15598 .copied()
15599 .filter(|event| event.event_type.starts_with("Timer"))
15600 .collect();
15601 let child_events: Vec<_> = sequence_events
15602 .iter()
15603 .copied()
15604 .filter(|event| {
15605 event.event_type == "ChildWorkflowScheduled"
15606 || event.event_type.starts_with("ChildRun")
15607 })
15608 .collect();
15609 let signal_wait_events: Vec<_> = sequence_events
15610 .iter()
15611 .copied()
15612 .filter(|event| is_recorded_signal_wait_event(event))
15613 .collect();
15614 let condition_wait_events: Vec<_> = sequence_events
15615 .iter()
15616 .copied()
15617 .filter(|event| is_recorded_condition_wait_event(event))
15618 .collect();
15619 let search_attribute_events: Vec<_> = sequence_events
15620 .iter()
15621 .copied()
15622 .filter(|event| event.event_type == "SearchAttributesUpserted")
15623 .collect();
15624 let side_effect_events: Vec<_> = sequence_events
15625 .iter()
15626 .copied()
15627 .filter(|event| event.event_type == "SideEffectRecorded")
15628 .collect();
15629 let version_marker_events: Vec<_> = sequence_events
15630 .iter()
15631 .copied()
15632 .filter(|event| event.event_type == "VersionMarkerRecorded")
15633 .collect();
15634 let memo_events: Vec<_> = sequence_events
15635 .iter()
15636 .copied()
15637 .filter(|event| event.event_type == "MemoUpserted")
15638 .collect();
15639
15640 let command_kind_count = usize::from(!activity_events.is_empty())
15641 + usize::from(!timer_events.is_empty())
15642 + usize::from(!child_events.is_empty())
15643 + usize::from(!signal_wait_events.is_empty())
15644 + usize::from(!condition_wait_events.is_empty())
15645 + usize::from(!search_attribute_events.is_empty())
15646 + usize::from(!side_effect_events.is_empty())
15647 + usize::from(!version_marker_events.is_empty())
15648 + usize::from(!memo_events.is_empty());
15649 if command_kind_count > 1 {
15650 let actual = [
15651 (!activity_events.is_empty()).then_some("activity"),
15652 (!timer_events.is_empty()).then_some("timer"),
15653 (!child_events.is_empty()).then_some("child workflow"),
15654 (!signal_wait_events.is_empty()).then_some("signal wait"),
15655 (!condition_wait_events.is_empty()).then_some("condition wait"),
15656 (!search_attribute_events.is_empty()).then_some("search-attribute update"),
15657 (!side_effect_events.is_empty()).then_some("side effect"),
15658 (!version_marker_events.is_empty()).then_some("version marker"),
15659 (!memo_events.is_empty()).then_some("memo upsert"),
15660 ]
15661 .into_iter()
15662 .flatten()
15663 .collect::<Vec<_>>()
15664 .join(" and ");
15665 return Err(invalid_recorded_history(
15666 "durable_command_sequence_collision",
15667 sequence,
15668 "one durable command kind",
15669 &actual,
15670 "one workflow sequence records more than one durable command kind",
15671 ));
15672 }
15673
15674 if !activity_events.is_empty() {
15675 let parallel_group_path =
15676 recorded_parallel_group_path(&activity_events, sequence)?;
15677 let scheduled_count = activity_events
15678 .iter()
15679 .filter(|event| event.event_type == "ActivityScheduled")
15680 .count();
15681 if scheduled_count > 1 {
15682 return Err(invalid_recorded_history(
15683 "duplicate_activity_schedule",
15684 sequence,
15685 "at most one ActivityScheduled event",
15686 "multiple ActivityScheduled events",
15687 "activity history schedules more than one command at one workflow sequence",
15688 ));
15689 }
15690 let activity_type = activity_events.iter().find_map(|event| {
15691 event
15692 .payload
15693 .get("activity_type")
15694 .or_else(|| event.payload.get("activity_name"))
15695 .and_then(Value::as_str)
15696 .map(str::to_string)
15697 });
15698 if activity_events.iter().filter_map(|event| {
15699 event
15700 .payload
15701 .get("activity_type")
15702 .or_else(|| event.payload.get("activity_name"))
15703 .and_then(Value::as_str)
15704 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
15705 return Err(invalid_recorded_history(
15706 "activity_identity_mismatch",
15707 sequence,
15708 activity_type.as_deref().unwrap_or("one activity identity"),
15709 "conflicting activity identities",
15710 "activity lifecycle events at one workflow sequence disagree on identity",
15711 ));
15712 }
15713 let terminal: Vec<_> = activity_events
15714 .iter()
15715 .copied()
15716 .filter(|event| {
15717 matches!(
15718 event.event_type.as_str(),
15719 "ActivityCompleted"
15720 | "ActivityFailed"
15721 | "ActivityCancelled"
15722 | "ActivityTimedOut"
15723 )
15724 })
15725 .collect();
15726 let duplicate_delivery = terminal.first().is_some_and(|first| {
15727 terminal.iter().all(|event| {
15728 event.event_type == first.event_type && event.payload == first.payload
15729 })
15730 });
15731 if terminal.len() > 1 && !duplicate_delivery {
15732 return Err(invalid_recorded_history(
15733 "duplicate_activity_terminal_event",
15734 sequence,
15735 "at most one terminal activity event",
15736 "multiple terminal activity events",
15737 "activity history settles one command more than once",
15738 ));
15739 }
15740 let outcome = terminal
15741 .first()
15742 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
15743 .transpose()?;
15744 let execution_mode = recorded_activity_execution_mode(&activity_events, sequence)?;
15745 let activity_snapshot = activity_events
15746 .iter()
15747 .find(|event| event.event_type == "ActivityScheduled")
15748 .and_then(|event| event.payload.get("activity"))
15749 .and_then(Value::as_object);
15750 let options = Some(match activity_snapshot {
15751 Some(activity) => RecordedActivityOptions {
15752 task_queue: recorded_optional_string(activity, "queue"),
15753 execution_mode,
15754 retry_policy: recorded_activity_retry_snapshot(
15755 activity.get("retry_policy"),
15756 ),
15757 worker_session: worker_session::recorded_session(&activity_events, sequence)?,
15758 },
15759 None => RecordedActivityOptions {
15760 task_queue: RecordedSnapshotValue::Unknown,
15761 execution_mode,
15762 retry_policy: recorded_activity_retry_snapshot(None),
15763 worker_session: worker_session::recorded_session(&activity_events, sequence)?,
15764 },
15765 });
15766 return Ok(RecordedCommand::Activity {
15767 sequence,
15768 activity_type,
15769 cancellation_policy: recorded_activity_cancellation_policy(&activity_events, sequence)?,
15770 options,
15771 outcome,
15772 parallel_group_path,
15773 });
15774 }
15775
15776 if !child_events.is_empty() {
15777 let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
15778 let scheduled: Vec<_> = child_events
15779 .iter()
15780 .copied()
15781 .filter(|event| event.event_type == "ChildWorkflowScheduled")
15782 .collect();
15783 if scheduled.len() != 1 {
15784 return Err(invalid_recorded_history(
15785 "child_workflow_schedule_missing_or_duplicate",
15786 sequence,
15787 "one ChildWorkflowScheduled event",
15788 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
15789 "child workflow replay requires exactly one recorded schedule event",
15790 ));
15791 }
15792 let policies = recorded_child_policies(&child_events, scheduled[0], sequence)?;
15793 let workflow_type = child_events.iter().find_map(|event| {
15794 event
15795 .payload
15796 .get("child_workflow_type")
15797 .or_else(|| event.payload.get("workflow_type"))
15798 .and_then(Value::as_str)
15799 .filter(|value| !value.is_empty())
15800 .map(str::to_string)
15801 });
15802 if child_events
15803 .iter()
15804 .filter_map(|event| {
15805 event
15806 .payload
15807 .get("child_workflow_type")
15808 .or_else(|| event.payload.get("workflow_type"))
15809 .and_then(Value::as_str)
15810 })
15811 .any(|candidate| Some(candidate) != workflow_type.as_deref())
15812 {
15813 return Err(invalid_recorded_history(
15814 "child_workflow_identity_mismatch",
15815 sequence,
15816 workflow_type
15817 .as_deref()
15818 .unwrap_or("one child workflow type"),
15819 "conflicting child workflow types",
15820 "child workflow lifecycle events at one sequence disagree on type",
15821 ));
15822 }
15823 let mut outcomes = child_workflow_outcomes(
15824 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
15825 fallback_codec,
15826 parent.clone(),
15827 )?;
15828 let terminal_events = child_events
15829 .iter()
15830 .copied()
15831 .filter(|event| event.event_type.starts_with("ChildRun"))
15832 .collect::<Vec<_>>();
15833 let duplicate_delivery = terminal_events.first().is_some_and(|first| {
15834 terminal_events.iter().all(|event| {
15835 event.event_type == first.event_type && event.payload == first.payload
15836 })
15837 });
15838 if outcomes.len() > 1 && !duplicate_delivery {
15839 return Err(invalid_recorded_history(
15840 "duplicate_child_workflow_terminal_event",
15841 sequence,
15842 "at most one terminal child event",
15843 "multiple terminal child events",
15844 "child workflow history settles one command more than once",
15845 ));
15846 }
15847 return Ok(RecordedCommand::ChildWorkflow {
15848 sequence,
15849 workflow_type,
15850 policies,
15851 outcome: outcomes.pop(),
15852 parallel_group_path,
15853 });
15854 }
15855
15856 if !signal_wait_events.is_empty() {
15857 let opened: Vec<_> = signal_wait_events
15858 .iter()
15859 .copied()
15860 .filter(|event| event.event_type == "SignalWaitOpened")
15861 .collect();
15862 if opened.len() != 1 {
15863 return Err(invalid_recorded_history(
15864 "signal_wait_open_missing_or_duplicate",
15865 sequence,
15866 "one SignalWaitOpened event",
15867 &format!("{} SignalWaitOpened events", opened.len()),
15868 "signal replay requires exactly one canonical wait-open event",
15869 ));
15870 }
15871
15872 let applied: Vec<_> = signal_wait_events
15873 .iter()
15874 .copied()
15875 .filter(|event| event.event_type == "SignalApplied")
15876 .collect();
15877 if applied.len() > 1 {
15878 return Err(invalid_recorded_history(
15879 "duplicate_signal_wait_apply",
15880 sequence,
15881 "at most one SignalApplied event",
15882 "multiple SignalApplied events",
15883 "signal history applies one durable wait more than once",
15884 ));
15885 }
15886
15887 let signal_names = signal_wait_events
15888 .iter()
15889 .map(|event| required_signal_wait_name(event, sequence))
15890 .collect::<Result<Vec<_>>>()?;
15891 let signal_name = signal_names
15892 .first()
15893 .expect("signal wait events are not empty")
15894 .clone();
15895 if signal_names.iter().any(|candidate| candidate != &signal_name) {
15896 return Err(invalid_recorded_history(
15897 "signal_wait_identity_mismatch",
15898 sequence,
15899 &signal_name,
15900 "conflicting signal names",
15901 "signal wait lifecycle events at one workflow sequence disagree on identity",
15902 ));
15903 }
15904 let value = applied
15905 .first()
15906 .map(|event| decode_signal_event_arguments(event, fallback_codec))
15907 .transpose()?;
15908 return Ok(RecordedCommand::SignalWait {
15909 sequence,
15910 signal_name,
15911 value,
15912 parallel_group_path: recorded_parallel_group_path(
15913 &signal_wait_events,
15914 sequence,
15915 )?,
15916 });
15917 }
15918
15919 if !condition_wait_events.is_empty() {
15920 return recorded_condition_wait(
15921 sequence,
15922 &condition_wait_events,
15923 events,
15924 );
15925 }
15926
15927 if !search_attribute_events.is_empty() {
15928 if search_attribute_events.len() != 1 {
15929 return Err(invalid_recorded_history(
15930 "duplicate_search_attribute_update",
15931 sequence,
15932 "one SearchAttributesUpserted event",
15933 &format!(
15934 "{} SearchAttributesUpserted events",
15935 search_attribute_events.len()
15936 ),
15937 "search-attribute history records one workflow command more than once",
15938 ));
15939 }
15940 let payload = &search_attribute_events[0].payload;
15941 let attributes = payload
15942 .get("attributes")
15943 .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
15944 .cloned()
15945 .ok_or_else(|| {
15946 invalid_recorded_history(
15947 "search_attribute_update_missing",
15948 sequence,
15949 "non-empty attributes object",
15950 "missing or invalid attributes",
15951 "search-attribute history is missing its recorded mutation",
15952 )
15953 })?;
15954 let attribute_types =
15955 recorded_search_attribute_types(payload, &attributes, sequence)?;
15956 return Ok(RecordedCommand::SearchAttributes {
15957 sequence,
15958 attributes,
15959 attribute_types,
15960 });
15961 }
15962
15963 if !side_effect_events.is_empty() {
15964 if side_effect_events.len() != 1 {
15965 return Err(invalid_recorded_history(
15966 "duplicate_side_effect_record",
15967 sequence,
15968 "one SideEffectRecorded event",
15969 &format!("{} SideEffectRecorded events", side_effect_events.len()),
15970 "side-effect history records one workflow command more than once",
15971 ));
15972 }
15973 let event = side_effect_events[0];
15974 let result = event.payload.get("result").ok_or_else(|| {
15975 invalid_recorded_history(
15976 "side_effect_result_missing",
15977 sequence,
15978 "recorded result payload",
15979 "missing result",
15980 "side-effect history is missing its recorded value",
15981 )
15982 })?;
15983 let has_published_envelope = result.as_str().is_some()
15984 || result.as_object().is_some_and(|envelope| {
15985 envelope.get("codec").and_then(Value::as_str).is_some()
15986 && envelope.get("blob").and_then(Value::as_str).is_some()
15987 });
15988 if !has_published_envelope {
15989 return Err(invalid_recorded_history(
15990 "side_effect_payload_malformed",
15991 sequence,
15992 "payload blob or {codec, blob} envelope",
15993 &result.to_string(),
15994 "side-effect history result does not use a published payload envelope",
15995 ));
15996 }
15997 let codec = event
15998 .payload
15999 .get("payload_codec")
16000 .and_then(Value::as_str)
16001 .unwrap_or(fallback_codec);
16002 let value = decode_wire_avro_value(result, codec).map_err(|error| {
16003 if error.to_string().contains("unsupported_payload_codec") {
16004 return error;
16005 }
16006
16007 invalid_recorded_history(
16008 "side_effect_payload_incompatible",
16009 sequence,
16010 &format!("valid {codec} payload envelope"),
16011 &error.to_string(),
16012 "side-effect history payload cannot be decoded with its recorded codec",
16013 )
16014 })?;
16015 return Ok(RecordedCommand::SideEffect { sequence, value });
16016 }
16017
16018 if !version_marker_events.is_empty() {
16019 if version_marker_events.len() != 1 {
16020 return Err(invalid_recorded_history(
16021 "duplicate_version_marker_record",
16022 sequence,
16023 "one VersionMarkerRecorded event",
16024 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
16025 "version-marker history records one workflow command more than once",
16026 ));
16027 }
16028 let payload = &version_marker_events[0].payload;
16029 let change_id = payload
16030 .get("change_id")
16031 .and_then(Value::as_str)
16032 .filter(|value| !value.is_empty())
16033 .map(str::to_string)
16034 .ok_or_else(|| {
16035 invalid_recorded_history(
16036 "version_marker_field_missing",
16037 sequence,
16038 "non-empty change_id",
16039 "missing or invalid change_id",
16040 "version-marker history is missing its stable change ID",
16041 )
16042 })?;
16043 let version = required_version_i32(payload, "version", sequence)?;
16044 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
16045 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
16046 if min_supported > max_supported || version < min_supported || version > max_supported {
16047 return Err(invalid_recorded_history(
16048 "version_marker_history_range_invalid",
16049 sequence,
16050 "min_supported <= version <= max_supported",
16051 &format!("{min_supported} <= {version} <= {max_supported}"),
16052 "recorded version marker contains an internally incompatible range",
16053 ));
16054 }
16055 return Ok(RecordedCommand::VersionMarker {
16056 sequence,
16057 change_id,
16058 version,
16059 });
16060 }
16061
16062 if !memo_events.is_empty() {
16063 if memo_events.len() != 1 {
16064 return Err(invalid_recorded_history(
16065 "duplicate_memo_upsert_record",
16066 sequence,
16067 "one MemoUpserted event",
16068 &format!("{} MemoUpserted events", memo_events.len()),
16069 "memo history records one workflow update more than once",
16070 ));
16071 }
16072 let payload = &memo_events[0].payload;
16073 let entries = payload.get("entries").cloned().ok_or_else(|| {
16074 invalid_recorded_history(
16075 "memo_entries_missing",
16076 sequence,
16077 "memo entries object",
16078 "missing entries",
16079 "MemoUpserted history is missing replay identity entries",
16080 )
16081 })?;
16082 let entries = decode_memo_history_map(&entries, true).map_err(|error| {
16083 invalid_recorded_history(
16084 "memo_entries_invalid",
16085 sequence,
16086 "valid canonical memo entries",
16087 &error.to_string(),
16088 "MemoUpserted history contains invalid replay identity entries",
16089 )
16090 })?;
16091 let merged = payload.get("merged").cloned().ok_or_else(|| {
16092 invalid_recorded_history(
16093 "memo_merged_projection_missing",
16094 sequence,
16095 "merged memo projection",
16096 "missing merged",
16097 "MemoUpserted history is missing its merged projection",
16098 )
16099 })?;
16100 decode_memo_history_map(&merged, false).map_err(|error| {
16101 invalid_recorded_history(
16102 "memo_merged_projection_invalid",
16103 sequence,
16104 "valid merged memo projection",
16105 &error.to_string(),
16106 "MemoUpserted history contains an invalid merged projection",
16107 )
16108 })?;
16109
16110 return Ok(RecordedCommand::Memo { sequence, entries });
16111 }
16112 let scheduled: Vec<_> = timer_events
16113 .iter()
16114 .copied()
16115 .filter(|event| event.event_type == "TimerScheduled")
16116 .collect();
16117 let fired: Vec<_> = timer_events
16118 .iter()
16119 .copied()
16120 .filter(|event| event.event_type == "TimerFired")
16121 .collect();
16122 if scheduled.len() != 1 {
16123 return Err(invalid_recorded_history(
16124 "timer_schedule_missing_or_duplicate",
16125 sequence,
16126 "one TimerScheduled event",
16127 &format!("{} TimerScheduled events", scheduled.len()),
16128 "timer replay requires exactly one recorded schedule event",
16129 ));
16130 }
16131 if fired.len() > 1 {
16132 return Err(invalid_recorded_history(
16133 "duplicate_timer_fire",
16134 sequence,
16135 "at most one TimerFired event",
16136 "multiple TimerFired events",
16137 "timer history contains more than one fire event for a workflow sequence",
16138 ));
16139 }
16140
16141 let scheduled = scheduled[0];
16142 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
16143 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
16144 if let Some(fired) = fired.first() {
16145 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
16146 if fired_timer_id != timer_id {
16147 return Err(invalid_recorded_history(
16148 "timer_identity_mismatch",
16149 sequence,
16150 &timer_id,
16151 &fired_timer_id,
16152 "TimerFired does not correspond to the recorded TimerScheduled event",
16153 ));
16154 }
16155 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
16156 if fired_delay != delay_seconds {
16157 return Err(invalid_recorded_history(
16158 "timer_history_delay_mismatch",
16159 sequence,
16160 &delay_seconds.to_string(),
16161 &fired_delay.to_string(),
16162 "TimerScheduled and TimerFired record different delays",
16163 ));
16164 }
16165 }
16166
16167 Ok(RecordedCommand::Timer {
16168 sequence,
16169 delay_seconds,
16170 fired: !fired.is_empty(),
16171 parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
16172 })
16173 })
16174 .collect::<Result<_>>()?;
16175
16176 let mut marker_decisions = HashMap::new();
16177 for command in &commands {
16178 if let RecordedCommand::VersionMarker {
16179 sequence,
16180 change_id,
16181 version,
16182 } = command
16183 {
16184 if let Some((first_version, first_sequence)) = marker_decisions.get(change_id) {
16185 if version != first_version {
16186 return Err(invalid_recorded_history(
16187 "version_marker_conflict",
16188 *sequence,
16189 &format!("version {first_version} for change ID {change_id:?} at sequence {first_sequence}"),
16190 &format!("version {version} at sequence {sequence}"),
16191 "workflow history contains conflicting decisions for one stable change ID",
16192 ));
16193 }
16194 } else {
16195 marker_decisions.insert(change_id.clone(), (*version, *sequence));
16196 }
16197 }
16198 }
16199
16200 Ok(commands)
16201}
16202
16203fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
16204 payload
16205 .get(field)
16206 .and_then(Value::as_i64)
16207 .and_then(|value| i32::try_from(value).ok())
16208 .ok_or_else(|| {
16209 invalid_recorded_history(
16210 "version_marker_field_missing",
16211 sequence,
16212 &format!("integer {field}"),
16213 "missing or out-of-range integer",
16214 "version-marker history is missing a required integer field",
16215 )
16216 })
16217}
16218
16219fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
16220 event
16221 .payload
16222 .get("sequence")
16223 .or_else(|| event.payload.get("workflow_sequence"))
16224 .or_else(|| event.raw.get("sequence"))
16225 .or_else(|| event.raw.get("workflow_sequence"))
16226 .and_then(value_as_u64)
16227}
16228
16229fn is_internal_timer_event(event: &HistoryEvent) -> bool {
16230 matches!(
16231 event
16232 .payload
16233 .get("timer_kind")
16234 .or_else(|| event.raw.get("timer_kind"))
16235 .and_then(Value::as_str),
16236 Some("condition_timeout" | "signal_timeout")
16237 )
16238}
16239
16240fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
16241 matches!(
16242 event.event_type.as_str(),
16243 "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
16244 )
16245}
16246
16247fn recorded_condition_wait(
16248 sequence: u64,
16249 condition_events: &[&HistoryEvent],
16250 all_events: &[HistoryEvent],
16251) -> Result<RecordedCommand> {
16252 let opened = condition_events
16253 .iter()
16254 .copied()
16255 .filter(|event| event.event_type == "ConditionWaitOpened")
16256 .collect::<Vec<_>>();
16257 if opened.len() != 1 {
16258 return Err(invalid_recorded_history(
16259 "condition_wait_open_missing_or_duplicate",
16260 sequence,
16261 "one ConditionWaitOpened event",
16262 &format!("{} ConditionWaitOpened events", opened.len()),
16263 "condition replay requires exactly one canonical wait-open event",
16264 ));
16265 }
16266 let terminal = condition_events
16267 .iter()
16268 .copied()
16269 .filter(|event| {
16270 matches!(
16271 event.event_type.as_str(),
16272 "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
16273 )
16274 })
16275 .collect::<Vec<_>>();
16276 if terminal.len() > 1 {
16277 return Err(invalid_recorded_history(
16278 "duplicate_condition_wait_terminal_event",
16279 sequence,
16280 "at most one condition terminal event",
16281 "multiple condition terminal events",
16282 "condition history settles one durable wait more than once",
16283 ));
16284 }
16285
16286 let opened = opened[0];
16287 let condition_wait_id = required_condition_wait_id(opened, sequence)?;
16288 let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
16289 for event in condition_events
16290 .iter()
16291 .copied()
16292 .filter(|event| !std::ptr::eq(*event, opened))
16293 {
16294 let event_wait_id = required_condition_wait_id(event, sequence)?;
16295 if event_wait_id != condition_wait_id {
16296 return Err(invalid_recorded_history(
16297 "condition_wait_id_mismatch",
16298 sequence,
16299 &condition_wait_id,
16300 &event_wait_id,
16301 "condition lifecycle events at one sequence disagree on wait identity",
16302 ));
16303 }
16304 let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
16305 if event_occurrence_id != occurrence_id {
16306 return Err(invalid_recorded_history(
16307 "condition_wait_occurrence_history_mismatch",
16308 sequence,
16309 &occurrence_id,
16310 &event_occurrence_id,
16311 "condition lifecycle events at one sequence disagree on authored occurrence identity",
16312 ));
16313 }
16314 }
16315
16316 let condition_key = optional_non_empty_history_string(opened, "condition_key");
16317 let predicate_identity = opened
16318 .payload
16319 .get("condition_definition_fingerprint")
16320 .and_then(Value::as_str)
16321 .filter(|value| !value.is_empty())
16322 .map(str::to_string)
16323 .ok_or_else(|| {
16324 invalid_recorded_history(
16325 "condition_wait_predicate_fingerprint_missing",
16326 sequence,
16327 "non-empty condition_definition_fingerprint",
16328 &opened.event_type,
16329 "canonical condition history is missing its predicate identity",
16330 )
16331 })?;
16332 let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
16333 for event in condition_events
16334 .iter()
16335 .copied()
16336 .filter(|event| !std::ptr::eq(*event, opened))
16337 {
16338 for (field, opened_value) in [
16339 ("condition_key", condition_key.as_deref()),
16340 (
16341 "condition_definition_fingerprint",
16342 Some(predicate_identity.as_str()),
16343 ),
16344 ] {
16345 if let Some(value) = optional_non_empty_history_string(event, field) {
16346 if opened_value.is_some_and(|opened_value| opened_value != value) {
16347 return Err(invalid_recorded_history(
16348 "condition_wait_definition_history_mismatch",
16349 sequence,
16350 opened_value.unwrap_or_default(),
16351 &value,
16352 "condition lifecycle events disagree on the recorded definition",
16353 ));
16354 }
16355 }
16356 }
16357 if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
16358 if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
16359 return Err(invalid_recorded_history(
16360 "condition_wait_definition_history_mismatch",
16361 sequence,
16362 &format!("{}s", timeout_seconds.unwrap_or_default()),
16363 &format!("{event_timeout}s"),
16364 "condition lifecycle events disagree on the recorded timeout",
16365 ));
16366 }
16367 }
16368 }
16369
16370 let timeout_timer_events = all_events
16371 .iter()
16372 .filter(|event| {
16373 matches!(
16374 event.event_type.as_str(),
16375 "TimerScheduled" | "TimerCancelled" | "TimerFired"
16376 ) && event.payload.get("timer_kind").and_then(Value::as_str)
16377 == Some("condition_timeout")
16378 && event
16379 .payload
16380 .get("condition_wait_id")
16381 .and_then(Value::as_str)
16382 == Some(condition_wait_id.as_str())
16383 })
16384 .collect::<Vec<_>>();
16385 let scheduled = timeout_timer_events
16386 .iter()
16387 .copied()
16388 .filter(|event| event.event_type == "TimerScheduled")
16389 .collect::<Vec<_>>();
16390 let fired = timeout_timer_events
16391 .iter()
16392 .copied()
16393 .filter(|event| event.event_type == "TimerFired")
16394 .collect::<Vec<_>>();
16395 if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
16396 return Err(invalid_recorded_history(
16397 "condition_wait_timeout_history_invalid",
16398 sequence,
16399 "one timeout schedule and at most one fire",
16400 &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
16401 "condition timeout history has a missing or duplicate lifecycle event",
16402 ));
16403 }
16404 if let Some(scheduled) = scheduled.first() {
16405 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
16406 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
16407 if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
16408 return Err(invalid_recorded_history(
16409 "condition_wait_timeout_delay_mismatch",
16410 sequence,
16411 &format!("{}s", timeout_seconds.unwrap_or_default()),
16412 &format!("{delay_seconds}s"),
16413 "condition timeout timer differs from the wait definition",
16414 ));
16415 }
16416 if let Some(fired) = fired.first() {
16417 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
16418 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
16419 if fired_timer_id != timer_id || fired_delay != delay_seconds {
16420 return Err(invalid_recorded_history(
16421 "condition_wait_timeout_identity_mismatch",
16422 sequence,
16423 &format!("{timer_id}:{delay_seconds}s"),
16424 &format!("{fired_timer_id}:{fired_delay}s"),
16425 "condition timeout fire does not match its durable schedule",
16426 ));
16427 }
16428 }
16429 }
16430
16431 let result = terminal.first().map(|event| {
16432 if event.event_type == "ConditionWaitTimedOut" {
16433 ConditionWaitResult::TimedOut
16434 } else {
16435 ConditionWaitResult::Satisfied
16436 }
16437 });
16438 let result = if !fired.is_empty() {
16439 if result == Some(ConditionWaitResult::Satisfied) {
16440 return Err(invalid_recorded_history(
16441 "condition_wait_terminal_conflict",
16442 sequence,
16443 "one satisfied or timed-out outcome",
16444 "satisfied event and fired timeout",
16445 "condition history records conflicting terminal outcomes",
16446 ));
16447 }
16448 Some(ConditionWaitResult::TimedOut)
16449 } else {
16450 result
16451 };
16452
16453 Ok(RecordedCommand::ConditionWait {
16454 sequence,
16455 occurrence_id,
16456 condition_key,
16457 predicate_identity,
16458 timeout_seconds,
16459 result,
16460 parallel_group_path: recorded_condition_group_path(condition_events, all_events, sequence)?,
16461 })
16462}
16463
16464fn recorded_condition_group_path(
16465 condition_events: &[&HistoryEvent],
16466 all_events: &[HistoryEvent],
16467 sequence: u64,
16468) -> Result<Option<Vec<ParallelGroupMetadata>>> {
16469 let opened = condition_events
16470 .iter()
16471 .copied()
16472 .find(|event| event.event_type == "ConditionWaitOpened")
16473 .expect("canonical condition opener is validated first");
16474 let occurrence = required_condition_wait_occurrence_id(opened, sequence)?;
16475 let previous_events = all_events
16476 .iter()
16477 .take_while(|event| !std::ptr::eq(*event, opened))
16478 .collect::<Vec<_>>();
16479 let originals = previous_events
16480 .iter()
16481 .copied()
16482 .filter(|event| event.event_type == "ConditionWaitOpened")
16483 .filter(|event| {
16484 event
16485 .payload
16486 .get("condition_wait_occurrence_id")
16487 .and_then(Value::as_str)
16488 == Some(occurrence.as_str())
16489 })
16490 .collect::<Vec<_>>();
16491 let Some(original) = originals.first().copied() else {
16492 return recorded_parallel_group_path(condition_events, sequence);
16493 };
16494 let authored_sequence = required_history_u64(original, "sequence", sequence)?;
16495 if authored_sequence == sequence {
16496 return recorded_parallel_group_path(condition_events, sequence);
16497 }
16498 let original_path = recorded_parallel_group_path(&[original], authored_sequence)?;
16499 if original_path.is_none() {
16500 return recorded_parallel_group_path(condition_events, sequence);
16501 }
16502 let reopened_path = recorded_parallel_group_path(condition_events, authored_sequence)?;
16503 let identity_matches = [
16504 "condition_key",
16505 "condition_definition_fingerprint",
16506 "timeout_seconds",
16507 ]
16508 .iter()
16509 .all(|field| opened.payload.get(*field) == original.payload.get(*field));
16510 let previous = originals
16511 .iter()
16512 .copied()
16513 .filter(|event| durable_event_sequence(event).is_some_and(|physical| physical < sequence))
16514 .max_by_key(|event| durable_event_sequence(event));
16515 let previously_satisfied = previous.is_some_and(|previous| {
16516 previous_events.iter().any(|event| {
16517 event.event_type == "ConditionWaitSatisfied"
16518 && event.payload.get("condition_wait_id")
16519 == previous.payload.get("condition_wait_id")
16520 && event.payload.get("condition_wait_occurrence_id")
16521 == previous.payload.get("condition_wait_occurrence_id")
16522 && durable_event_sequence(event) == durable_event_sequence(previous)
16523 })
16524 });
16525 if !identity_matches || reopened_path != original_path || !previously_satisfied {
16526 return Err(invalid_recorded_history(
16527 "condition_wait_group_reopen_mismatch",
16528 sequence,
16529 "same authored condition and group path after a satisfied physical wait",
16530 &opened.payload.to_string(),
16531 "grouped condition reopen has no matching recorded predecessor",
16532 ));
16533 }
16534 Ok(reopened_path)
16535}
16536
16537fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
16538 event
16539 .payload
16540 .get("condition_wait_occurrence_id")
16541 .and_then(Value::as_str)
16542 .filter(|value| !value.is_empty())
16543 .map(str::to_string)
16544 .ok_or_else(|| {
16545 invalid_recorded_history(
16546 "condition_wait_occurrence_id_missing",
16547 sequence,
16548 "non-empty condition_wait_occurrence_id",
16549 &event.event_type,
16550 "condition history is missing authored occurrence identity",
16551 )
16552 })
16553}
16554
16555fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
16556 event
16557 .payload
16558 .get("condition_wait_id")
16559 .and_then(Value::as_str)
16560 .filter(|value| !value.is_empty())
16561 .map(str::to_string)
16562 .ok_or_else(|| {
16563 invalid_recorded_history(
16564 "condition_wait_id_missing",
16565 sequence,
16566 "non-empty condition_wait_id",
16567 &event.event_type,
16568 "canonical condition history is missing its durable wait identity",
16569 )
16570 })
16571}
16572
16573fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
16574 event
16575 .payload
16576 .get(field)
16577 .and_then(Value::as_str)
16578 .filter(|value| !value.is_empty())
16579 .map(str::to_string)
16580}
16581
16582fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
16583 match event.payload.get(field) {
16584 None | Some(Value::Null) => Ok(None),
16585 Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
16586 invalid_recorded_history(
16587 "condition_wait_definition_invalid",
16588 sequence,
16589 &format!("non-negative integer {field}"),
16590 &value.to_string(),
16591 "condition history contains an invalid numeric definition field",
16592 )
16593 }),
16594 }
16595}
16596
16597fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
16598 event
16599 .payload
16600 .get("signal_name")
16601 .or_else(|| event.raw.get("signal_name"))
16602 .and_then(Value::as_str)
16603 .filter(|value| !value.is_empty())
16604 .map(str::to_string)
16605 .ok_or_else(|| {
16606 invalid_recorded_history(
16607 "signal_wait_name_missing",
16608 sequence,
16609 "non-empty signal_name",
16610 &event.event_type,
16611 "canonical signal-wait history is missing its signal identity",
16612 )
16613 })
16614}
16615
16616fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
16617 if event.event_type == "SignalApplied"
16620 && event
16621 .payload
16622 .get("signal_wait_id")
16623 .and_then(Value::as_str)
16624 .is_some_and(|id| id.starts_with("signal-command:"))
16625 && event.payload.get("sequence").is_none()
16626 && event.payload.get("workflow_sequence").is_none()
16627 {
16628 return false;
16629 }
16630 matches!(
16631 event.event_type.as_str(),
16632 "SignalWaitOpened" | "SignalApplied"
16633 )
16634}
16635
16636fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
16637 event
16638 .payload
16639 .get(field)
16640 .and_then(Value::as_str)
16641 .filter(|value| !value.is_empty())
16642 .map(str::to_string)
16643 .ok_or_else(|| {
16644 invalid_recorded_history(
16645 "timer_history_field_missing",
16646 sequence,
16647 field,
16648 &event.event_type,
16649 "timer history is missing a required identity field",
16650 )
16651 })
16652}
16653
16654fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
16655 event
16656 .payload
16657 .get(field)
16658 .and_then(value_as_u64)
16659 .ok_or_else(|| {
16660 invalid_recorded_history(
16661 "timer_history_field_missing",
16662 sequence,
16663 field,
16664 &event.event_type,
16665 "timer history is missing a required numeric field",
16666 )
16667 })
16668}
16669
16670fn recorded_search_attribute_types(
16671 payload: &Value,
16672 attributes: &Value,
16673 sequence: u64,
16674) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
16675 let Some(raw_types) = payload.get("attribute_types") else {
16676 return Ok(RecordedSnapshotValue::Unknown);
16680 };
16681 let Some(raw_types) = raw_types.as_object() else {
16682 return Err(invalid_recorded_history(
16683 "search_attribute_types_malformed",
16684 sequence,
16685 "canonical attribute type map",
16686 &raw_types.to_string(),
16687 "search-attribute history contains malformed type identity",
16688 ));
16689 };
16690 let attribute_keys = attributes
16691 .as_object()
16692 .expect("recorded search attributes were validated as an object");
16693 let mut types = BTreeMap::new();
16694 for (key, value) in raw_types {
16695 let Some(attribute_type) = value.as_str() else {
16696 return Err(invalid_recorded_history(
16697 "search_attribute_types_malformed",
16698 sequence,
16699 "canonical string type name",
16700 &value.to_string(),
16701 "search-attribute history contains a non-string type identity",
16702 ));
16703 };
16704 if !attribute_keys.contains_key(key)
16705 || !matches!(
16706 attribute_type,
16707 "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
16708 )
16709 {
16710 return Err(invalid_recorded_history(
16711 "search_attribute_types_malformed",
16712 sequence,
16713 "canonical types for keys present in attributes",
16714 &format!("{key}:{attribute_type}"),
16715 "search-attribute history contains unsupported or orphaned type identity",
16716 ));
16717 }
16718 types.insert(key.clone(), attribute_type.to_string());
16719 }
16720 Ok(RecordedSnapshotValue::Known(types))
16721}
16722
16723fn invalid_recorded_history(
16724 reason: &str,
16725 sequence: u64,
16726 expected: &str,
16727 actual: &str,
16728 message: &str,
16729) -> Error {
16730 Error::NonDeterministicReplay(ReplayFailure::new(
16731 reason,
16732 Some(sequence),
16733 Some(expected.to_string()),
16734 Some(actual.to_string()),
16735 message,
16736 ))
16737}
16738
16739type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
16740
16741fn activity_outcome(
16742 event: &HistoryEvent,
16743 fallback_codec: &str,
16744 recorded_activity_type: Option<String>,
16745) -> Result<ActivityOutcome> {
16746 if event.event_type == "ActivityCompleted" {
16747 let codec = event
16748 .payload
16749 .get("payload_codec")
16750 .and_then(Value::as_str)
16751 .unwrap_or(fallback_codec);
16752 return Ok(Ok(decode_wire_avro_value(
16753 event.payload.get("result").unwrap_or(&Value::Null),
16754 codec,
16755 )?));
16756 }
16757
16758 let payload = &event.payload;
16759 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
16760 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
16761 "ActivityCancelled" => (
16762 ActivityFailureKind::Cancelled,
16763 "cancelled",
16764 "activity was cancelled",
16765 ),
16766 "ActivityTimedOut" => (
16767 ActivityFailureKind::TimedOut,
16768 "timeout",
16769 "activity timed out",
16770 ),
16771 _ => unreachable!("activity_outcome is called only for terminal activity events"),
16772 };
16773 let exception = payload
16774 .get("exception")
16775 .filter(|value| !value.is_null())
16776 .cloned();
16777 let failure_category = payload_string(payload, "failure_category");
16778 let timeout_kind = payload_string(payload, "timeout_kind");
16779 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
16780 ActivityFailureKind::Failed => failure_category
16781 .clone()
16782 .unwrap_or_else(|| fallback_reason.to_string()),
16783 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
16784 ActivityFailureKind::TimedOut => timeout_kind
16785 .clone()
16786 .unwrap_or_else(|| fallback_reason.to_string()),
16787 });
16788 let message = payload_string(payload, "message")
16789 .or_else(|| {
16790 exception
16791 .as_ref()
16792 .and_then(|value| payload_string(value, "message"))
16793 })
16794 .unwrap_or_else(|| fallback_message.to_string());
16795
16796 Ok(Err(ActivityFailure {
16797 kind,
16798 reason,
16799 message,
16800 activity_execution_id: payload_string(payload, "activity_execution_id"),
16801 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
16802 activity_type: payload_string(payload, "activity_type")
16803 .or_else(|| payload_string(payload, "activity_name"))
16804 .or(recorded_activity_type),
16805 activity_class: payload_string(payload, "activity_class"),
16806 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
16807 failure_id: payload_string(payload, "failure_id"),
16808 failure_category,
16809 timeout_kind,
16810 non_retryable: payload
16811 .get("non_retryable")
16812 .and_then(Value::as_bool)
16813 .unwrap_or(false),
16814 exception_type: payload_string(payload, "exception_type").or_else(|| {
16815 exception
16816 .as_ref()
16817 .and_then(|value| payload_string(value, "type"))
16818 }),
16819 exception_class: payload_string(payload, "exception_class").or_else(|| {
16820 exception
16821 .as_ref()
16822 .and_then(|value| payload_string(value, "class"))
16823 }),
16824 code: payload
16825 .get("code")
16826 .filter(|value| !value.is_null())
16827 .cloned(),
16828 exception,
16829 }))
16830}
16831
16832type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
16833
16834fn child_workflow_outcomes(
16835 events: &[HistoryEvent],
16836 fallback_codec: &str,
16837 parent: WorkflowIdentity,
16838) -> Result<Vec<ChildWorkflowOutcome>> {
16839 let mut outcomes = Vec::new();
16840
16841 for event in events {
16842 let kind = match event.event_type.as_str() {
16843 "ChildRunCompleted" => None,
16844 "ChildRunFailed" => Some((
16845 ChildWorkflowFailureKind::Failed,
16846 "child_workflow",
16847 "child workflow failed",
16848 )),
16849 "ChildRunCancelled" => Some((
16850 ChildWorkflowFailureKind::Cancelled,
16851 "cancelled",
16852 "child workflow was cancelled",
16853 )),
16854 "ChildRunTerminated" => Some((
16855 ChildWorkflowFailureKind::Terminated,
16856 "terminated",
16857 "child workflow was terminated",
16858 )),
16859 _ => continue,
16860 };
16861 let payload = &event.payload;
16862 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
16863 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
16864 let child_workflow_type = payload_string(payload, "child_workflow_type");
16865
16866 if let Some((kind, reason, fallback_message)) = kind {
16867 let exception = payload
16868 .get("exception")
16869 .filter(|value| !value.is_null())
16870 .cloned();
16871 let message = payload_string(payload, "message")
16872 .or_else(|| {
16873 exception
16874 .as_ref()
16875 .and_then(|value| payload_string(value, "message"))
16876 })
16877 .unwrap_or_else(|| fallback_message.to_string());
16878 let exception_type = payload_string(payload, "exception_type").or_else(|| {
16879 exception
16880 .as_ref()
16881 .and_then(|value| payload_string(value, "type"))
16882 });
16883 let exception_class = payload_string(payload, "exception_class").or_else(|| {
16884 exception
16885 .as_ref()
16886 .and_then(|value| payload_string(value, "class"))
16887 });
16888 outcomes.push(Err(ChildWorkflowFailure {
16889 kind,
16890 reason: reason.to_string(),
16891 message,
16892 parent_workflow_id: parent.workflow_id.clone(),
16893 parent_workflow_run_id: parent.run_id.clone(),
16894 child_workflow_id,
16895 child_workflow_run_id,
16896 child_workflow_type,
16897 failure_id: payload_string(payload, "failure_id"),
16898 failure_category: payload_string(payload, "failure_category"),
16899 exception_type,
16900 exception_class,
16901 non_retryable: payload
16902 .get("non_retryable")
16903 .and_then(Value::as_bool)
16904 .unwrap_or(false),
16905 code: payload
16906 .get("code")
16907 .filter(|value| !value.is_null())
16908 .cloned(),
16909 exception,
16910 }));
16911 continue;
16912 }
16913
16914 let codec = payload
16915 .get("payload_codec")
16916 .and_then(Value::as_str)
16917 .unwrap_or(fallback_codec);
16918 let result = payload
16919 .get("result")
16920 .or_else(|| payload.get("output"))
16921 .unwrap_or(&Value::Null);
16922 outcomes.push(Ok(ChildWorkflowAvroResult {
16923 parent: parent.clone(),
16924 child: WorkflowIdentity {
16925 workflow_id: child_workflow_id,
16926 run_id: child_workflow_run_id,
16927 },
16928 child_workflow_type,
16929 result: decode_wire_avro_value(result, codec)?,
16930 }));
16931 }
16932
16933 Ok(outcomes)
16934}
16935
16936fn payload_string(payload: &Value, key: &str) -> Option<String> {
16937 payload
16938 .get(key)
16939 .and_then(Value::as_str)
16940 .filter(|value| !value.is_empty())
16941 .map(str::to_string)
16942}
16943
16944fn recorded_activity_failure_boundary(
16945 ctx: &WorkflowContext,
16946 error: &Error,
16947) -> Option<(u64, String)> {
16948 let Error::ActivityFailed(failure) = error else {
16949 return None;
16950 };
16951 if failure.kind != ActivityFailureKind::Failed {
16952 return None;
16953 }
16954 let activity_execution_id = failure
16955 .activity_execution_id
16956 .as_deref()
16957 .filter(|value| !value.is_empty())?;
16958 let state = ctx.state.lock().ok()?;
16959 let mut matches = state.recorded_commands.iter().filter_map(|command| {
16960 let RecordedCommand::Activity {
16961 sequence,
16962 outcome: Some(Err(recorded_failure)),
16963 ..
16964 } = command
16965 else {
16966 return None;
16967 };
16968 (*sequence > 0 && recorded_failure == failure)
16969 .then_some((*sequence, activity_execution_id.to_string()))
16970 });
16971 let boundary = matches.next()?;
16972 matches.next().is_none().then_some(boundary)
16973}
16974
16975fn workflow_failure_command(
16976 error: &Error,
16977 failed_activity_boundary: Option<(u64, String)>,
16978) -> Value {
16979 let (exception_type, exception_class, properties) = match error {
16980 Error::ActivityFailed(failure) => (
16981 match failure.kind {
16982 ActivityFailureKind::Failed => "ActivityFailed",
16983 ActivityFailureKind::Cancelled => "ActivityCancelled",
16984 ActivityFailureKind::TimedOut => "ActivityTimedOut",
16985 },
16986 "durable_workflow::ActivityFailure",
16987 json!({
16988 "reason": failure.reason,
16989 "activity_execution_id": failure.activity_execution_id,
16990 "activity_attempt_id": failure.activity_attempt_id,
16991 "activity_type": failure.activity_type,
16992 "activity_class": failure.activity_class,
16993 "attempt_number": failure.attempt_number,
16994 "failure_id": failure.failure_id,
16995 "failure_category": failure.failure_category,
16996 "timeout_kind": failure.timeout_kind,
16997 "activity_non_retryable": failure.non_retryable,
16998 "activity_exception_type": failure.exception_type,
16999 "activity_exception_class": failure.exception_class,
17000 "activity_code": failure.code,
17001 "activity_exception": failure.exception,
17002 }),
17003 ),
17004 Error::ChildWorkflowFailed(failure) => (
17005 match failure.kind {
17006 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
17007 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
17008 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
17009 },
17010 "durable_workflow::ChildWorkflowFailure",
17011 json!({
17012 "reason": failure.reason,
17013 "parent_workflow_id": failure.parent_workflow_id,
17014 "parent_workflow_run_id": failure.parent_workflow_run_id,
17015 "child_workflow_id": failure.child_workflow_id,
17016 "child_workflow_run_id": failure.child_workflow_run_id,
17017 "child_workflow_type": failure.child_workflow_type,
17018 "failure_id": failure.failure_id,
17019 "failure_category": failure.failure_category,
17020 "child_exception_type": failure.exception_type,
17021 "child_exception_class": failure.exception_class,
17022 "child_non_retryable": failure.non_retryable,
17023 "child_code": failure.code,
17024 "child_exception": failure.exception,
17025 }),
17026 ),
17027 Error::ParallelFailed(failure) => (
17028 "ParallelFailed",
17029 "durable_workflow::ParallelFailure",
17030 json!({
17031 "parallel_group_id": failure.group_id,
17032 "parallel_member_path": failure.member_path,
17033 "parallel_group_path": failure.group_path,
17034 "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
17035 "cause_type": workflow_error_type(&failure.cause),
17036 "cause_message": failure.cause.to_string(),
17037 }),
17038 ),
17039 Error::SagaCompensationFailed(failure) => (
17040 "SagaCompensationFailed",
17041 "durable_workflow::SagaCompensationFailure",
17042 json!({
17043 "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
17044 "initiating_failure_message": failure.initiating_failure.to_string(),
17045 "compensation_activity_type": failure.compensation_activity_type,
17046 "compensation_registration_order": failure.compensation_registration_order,
17047 "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
17048 "compensation_failure_message": failure.compensation_failure.to_string(),
17049 }),
17050 ),
17051 Error::WorkflowCancellationRequested(_) => (
17052 "WorkflowCancellationRequested",
17053 "durable_workflow::WorkflowCancellationRequested",
17054 json!({"reason": "cancelled"}),
17055 ),
17056 Error::CooperativeCancellationRequested(cancellation) => (
17057 "WorkflowCancellationRequested",
17058 "durable_workflow::CooperativeCancellationRequested",
17059 json!({"reason": "cancelled", "request_id": cancellation.request.request_id,
17060 "cleanup_deadline_at": cancellation.request.cleanup_deadline_at}),
17061 ),
17062 Error::CancellationScopeRequested(cancellation) => (
17063 "WorkflowCancellationRequested",
17064 "durable_workflow::CancellationScopeRequested",
17065 json!({"reason":"scope_cancelled", "request_id":cancellation.context.request_id(),
17066 "cleanup_deadline_at":cancellation.context.deadline().to_rfc3339(),
17067 "cancellation":cancellation.context.to_value()}),
17068 ),
17069 Error::NonDeterministicReplay(_) => (
17070 "NonDeterministicReplay",
17071 "durable_workflow::Error",
17072 Value::Null,
17073 ),
17074 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
17075 };
17076 let non_retryable = match error {
17077 Error::ActivityFailed(failure) => failure.non_retryable,
17078 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
17079 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
17080 Error::SagaCompensationFailed(failure) => {
17081 workflow_error_non_retryable(&failure.compensation_failure)
17082 }
17083 Error::WorkflowCancellationRequested(_)
17084 | Error::CooperativeCancellationRequested(_)
17085 | Error::CancellationScopeRequested(_) => true,
17086 Error::NonDeterministicReplay(_) => true,
17087 _ => false,
17088 };
17089
17090 let mut command = json!({
17091 "type": "fail_workflow",
17092 "message": error.to_string(),
17093 "exception_type": exception_type,
17094 "exception_class": exception_class,
17095 "non_retryable": non_retryable,
17096 "exception": {
17097 "type": exception_type,
17098 "class": exception_class,
17099 "message": error.to_string(),
17100 "properties": properties,
17101 }
17102 });
17103
17104 if let Some((sequence, activity_execution_id)) = failed_activity_boundary {
17105 command["failed_step_sequence"] = json!(sequence);
17106 command["failed_activity_execution_id"] = json!(activity_execution_id);
17107 }
17108
17109 command
17110}
17111
17112fn workflow_error_type(error: &Error) -> &'static str {
17113 match error {
17114 Error::ActivityFailed(failure) => match failure.kind {
17115 ActivityFailureKind::Failed => "ActivityFailed",
17116 ActivityFailureKind::Cancelled => "ActivityCancelled",
17117 ActivityFailureKind::TimedOut => "ActivityTimedOut",
17118 },
17119 Error::ChildWorkflowFailed(failure) => match failure.kind {
17120 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
17121 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
17122 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
17123 },
17124 Error::ParallelFailed(_) => "ParallelFailed",
17125 Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
17126 Error::WorkflowCancellationRequested(_)
17127 | Error::CooperativeCancellationRequested(_)
17128 | Error::CancellationScopeRequested(_) => "WorkflowCancellationRequested",
17129 Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
17130 _ => "RustWorkflowError",
17131 }
17132}
17133
17134fn workflow_error_non_retryable(error: &Error) -> bool {
17135 match error {
17136 Error::ActivityFailed(failure) => failure.non_retryable,
17137 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
17138 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
17139 Error::SagaCompensationFailed(failure) => {
17140 workflow_error_non_retryable(&failure.compensation_failure)
17141 }
17142 Error::WorkflowCancellationRequested(_)
17143 | Error::CooperativeCancellationRequested(_)
17144 | Error::CancellationScopeRequested(_)
17145 | Error::NonDeterministicReplay(_) => true,
17146 _ => false,
17147 }
17148}
17149
17150fn workflow_task_integrity_error(error: &Error) -> bool {
17151 matches!(
17152 error,
17153 Error::NonDeterministicReplay(_)
17154 | Error::Protocol(_)
17155 | Error::CancellationScopeExecutionUnavailable
17156 | Error::MissingWorkflowCommandIdentity
17157 | Error::WorkflowStatePoisoned
17158 )
17159}
17160
17161fn decode_signal_event_arguments(
17162 event: &HistoryEvent,
17163 fallback_codec: &str,
17164) -> Result<Vec<AvroValue>> {
17165 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
17166 validate_payload_codec(codec)?;
17167 let raw = signal_history_payload(&event.payload);
17168 let decoded = match raw.filter(|value| !value.is_null()) {
17169 Some(value) => decode_wire_avro_value(value, codec)?,
17170 None => AvroValue::Array(Vec::new()),
17171 };
17172 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
17173 unreachable!("normalize_avro_arguments always returns an array");
17174 };
17175 Ok(arguments)
17176}
17177
17178fn decode_update_event_arguments(
17179 event: &HistoryEvent,
17180 fallback_codec: &str,
17181) -> Result<Vec<AvroValue>> {
17182 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
17183 validate_payload_codec(codec)?;
17184 let decoded = match event
17185 .payload
17186 .get("arguments")
17187 .filter(|value| !value.is_null())
17188 {
17189 Some(value) => decode_wire_avro_value(value, codec)?,
17190 None => AvroValue::Array(Vec::new()),
17191 };
17192 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
17193 unreachable!("normalize_avro_arguments always returns an array");
17194 };
17195 Ok(arguments)
17196}
17197
17198fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
17199 let Some(export_events) = task
17200 .history_export
17201 .as_ref()
17202 .and_then(|export| export.get("history_events"))
17203 .and_then(Value::as_array)
17204 else {
17205 return Ok(());
17206 };
17207
17208 if export_events.len() > task.history_events.len() {
17209 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
17210 }
17211
17212 Ok(())
17213}
17214
17215fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
17216 let Some(export) = task.history_export.as_ref() else {
17217 return Ok(());
17218 };
17219 let signals = export
17220 .get("signals")
17221 .and_then(Value::as_array)
17222 .cloned()
17223 .unwrap_or_default();
17224 let activities = export
17225 .get("activities")
17226 .and_then(Value::as_array)
17227 .cloned()
17228 .unwrap_or_default();
17229 let export_codec = export
17230 .get("payloads")
17231 .and_then(|payloads| payloads.get("codec"))
17232 .and_then(Value::as_str)
17233 .unwrap_or(&task.payload_codec)
17234 .to_string();
17235 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
17236
17237 for event in &mut task.history_events {
17238 if event.event_type == "ActivityCompleted" {
17239 let sequence = event
17240 .payload
17241 .get("sequence")
17242 .or_else(|| event.payload.get("workflow_sequence"))
17243 .and_then(value_as_u64);
17244 let Some(activity) = sequence.and_then(|sequence| {
17245 activities.iter().find(|activity| {
17246 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
17247 })
17248 }) else {
17249 continue;
17250 };
17251 let Some(payload) = event.payload.as_object_mut() else {
17252 continue;
17253 };
17254 if missing_payload(payload.get("result")) {
17255 if let Some(result) = activity
17256 .get("result")
17257 .filter(|value| !missing_payload(Some(value)))
17258 {
17259 payload.insert("result".to_string(), result.clone());
17260 }
17261 }
17262 for field in ["payload_codec", "activity_type"] {
17263 if payload
17264 .get(field)
17265 .and_then(Value::as_str)
17266 .unwrap_or_default()
17267 .is_empty()
17268 {
17269 if let Some(value) = activity.get(field) {
17270 payload.insert(field.to_string(), value.clone());
17271 }
17272 }
17273 }
17274 continue;
17275 }
17276
17277 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
17278 continue;
17279 }
17280 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
17281 let command_id = event
17282 .payload
17283 .get("workflow_command_id")
17284 .or_else(|| event.raw.get("workflow_command_id"))
17285 .and_then(Value::as_str);
17286 let signal_name = event
17287 .payload
17288 .get("signal_name")
17289 .and_then(Value::as_str)
17290 .unwrap_or_default()
17291 .to_string();
17292 let matched = signals
17293 .iter()
17294 .find(|signal| {
17295 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
17296 })
17297 .or_else(|| {
17298 signals.iter().find(|signal| {
17299 command_id.is_some()
17300 && signal.get("command_id").and_then(Value::as_str) == command_id
17301 })
17302 })
17303 .or_else(|| {
17304 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
17305 let signal = signals
17306 .iter()
17307 .filter(|signal| {
17308 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
17309 })
17310 .nth(*offset);
17311 if signal.is_some() {
17312 *offset += 1;
17313 }
17314 signal
17315 });
17316 let Some(signal) = matched else {
17317 continue;
17318 };
17319 let signal_codec = signal
17320 .get("payload_codec")
17321 .and_then(Value::as_str)
17322 .unwrap_or(&export_codec);
17323 let Some(payload) = event.payload.as_object_mut() else {
17324 continue;
17325 };
17326 if missing_payload(payload.get("arguments")) {
17327 if let Some(arguments) = signal
17328 .get("arguments")
17329 .filter(|value| !missing_payload(Some(value)))
17330 {
17331 let envelope = match arguments {
17332 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
17333 other => other.clone(),
17334 };
17335 payload.insert("arguments".to_string(), envelope);
17336 }
17337 }
17338 if payload
17339 .get("payload_codec")
17340 .and_then(Value::as_str)
17341 .unwrap_or_default()
17342 .is_empty()
17343 {
17344 payload.insert("payload_codec".to_string(), json!(signal_codec));
17345 }
17346 }
17347
17348 Ok(())
17349}
17350
17351fn missing_payload(value: Option<&Value>) -> bool {
17352 match value {
17353 None | Some(Value::Null) => true,
17354 Some(Value::String(value)) => value.is_empty(),
17355 Some(_) => false,
17356 }
17357}
17358
17359fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
17360 history_signal_events(
17361 &task.history_events,
17362 task.history_export.as_ref(),
17363 &task.payload_codec,
17364 )
17365}
17366
17367fn history_signal_events(
17368 history_events: &[HistoryEvent],
17369 history_export: Option<&Value>,
17370 payload_codec: &str,
17371) -> Result<Vec<QuerySignal>> {
17372 let export_signals = history_export
17373 .and_then(|export| export.get("signals"))
17374 .and_then(Value::as_array)
17375 .cloned()
17376 .unwrap_or_default();
17377 let export_codec = history_export
17378 .and_then(|export| export.get("payloads"))
17379 .and_then(|payloads| payloads.get("codec"))
17380 .and_then(Value::as_str)
17381 .unwrap_or(payload_codec);
17382 let mut name_offsets: HashMap<String, usize> = HashMap::new();
17383 let mut signals: Vec<QuerySignal> = Vec::new();
17384 let mut signal_indices = HashMap::new();
17385 let mut command_indices = HashMap::new();
17386
17387 for event in history_events {
17388 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
17389 continue;
17390 }
17391
17392 let name = event
17393 .payload
17394 .get("signal_name")
17395 .and_then(Value::as_str)
17396 .unwrap_or_default();
17397 if name.is_empty() {
17398 continue;
17399 }
17400 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
17401 let command_id = event
17402 .payload
17403 .get("workflow_command_id")
17404 .or_else(|| event.raw.get("workflow_command_id"))
17405 .and_then(Value::as_str);
17406 let matched_export = export_signals
17407 .iter()
17408 .find(|candidate| {
17409 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
17410 })
17411 .or_else(|| {
17412 export_signals.iter().find(|candidate| {
17413 command_id.is_some()
17414 && candidate.get("command_id").and_then(Value::as_str) == command_id
17415 })
17416 })
17417 .or_else(|| {
17418 let offset = name_offsets.entry(name.to_string()).or_default();
17419 let candidate = export_signals
17420 .iter()
17421 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
17422 .nth(*offset);
17423 if candidate.is_some() {
17424 *offset += 1;
17425 }
17426 candidate
17427 });
17428 let codec = event
17429 .payload
17430 .get("payload_codec")
17431 .and_then(Value::as_str)
17432 .or_else(|| {
17433 matched_export
17434 .and_then(|signal| signal.get("payload_codec"))
17435 .and_then(Value::as_str)
17436 })
17437 .unwrap_or(export_codec);
17438 let raw_arguments = signal_history_payload(&event.payload)
17439 .filter(|value| !value.is_null())
17440 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
17441 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
17442 let workflow_sequence = event
17443 .payload
17444 .get("workflow_sequence")
17445 .and_then(value_as_u64)
17446 .or_else(|| {
17447 matched_export
17448 .and_then(|signal| signal.get("workflow_sequence"))
17449 .and_then(value_as_u64)
17450 });
17451
17452 let id = signal_id.or_else(|| {
17453 matched_export
17454 .and_then(|signal| signal.get("id"))
17455 .and_then(Value::as_str)
17456 });
17457 let command_id = command_id.or_else(|| {
17458 matched_export
17459 .and_then(|signal| signal.get("command_id"))
17460 .and_then(Value::as_str)
17461 });
17462 let duplicate = id
17463 .and_then(|id| signal_indices.get(id))
17464 .or_else(|| command_id.and_then(|id| command_indices.get(id)))
17465 .copied();
17466 let index = duplicate.unwrap_or(signals.len());
17467 if let Some(index) = duplicate {
17468 let original = &mut signals[index];
17469 let same_arguments = if event.event_type == "SignalApplied" {
17470 if let Some(value) = event.payload.get("value") {
17471 let expected = match original.avro_arguments.as_slice() {
17472 [] => AvroValue::Boolean(true),
17473 [value] => value.clone(),
17474 values => AvroValue::Array(values.to_vec()),
17475 };
17476 let applied = decode_wire_avro_value(value, codec)?;
17477 applied == expected
17478 || applied == AvroValue::Array(original.avro_arguments.clone())
17479 } else {
17480 original.avro_arguments == avro_arguments
17481 }
17482 } else {
17483 original.avro_arguments == avro_arguments
17484 };
17485 if original.name != name
17486 || !same_arguments
17487 || matches!((original.id.as_deref(), id), (Some(first), Some(next)) if first != next)
17488 {
17489 return Err(Error::Codec("conflicting committed signal identity".into()));
17490 }
17491 if original.id.is_none() {
17492 original.id = id.map(str::to_string);
17493 }
17494 original.workflow_sequence = original.workflow_sequence.or(workflow_sequence);
17495 } else {
17496 signals.push(QuerySignal {
17497 id: id.map(str::to_string),
17498 name: name.to_string(),
17499 arguments,
17500 avro_arguments,
17501 workflow_sequence,
17502 });
17503 }
17504 if let Some(id) = id {
17505 signal_indices.insert(id.to_string(), index);
17506 }
17507 if let Some(id) = command_id {
17508 command_indices.insert(id.to_string(), index);
17509 }
17510 }
17511
17512 if signals.is_empty() {
17513 for signal in export_signals {
17514 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
17515 continue;
17516 }
17517 let Some(name) = signal.get("name").and_then(Value::as_str) else {
17518 continue;
17519 };
17520 let codec = signal
17521 .get("payload_codec")
17522 .and_then(Value::as_str)
17523 .unwrap_or(export_codec);
17524 let (arguments, avro_arguments) =
17525 decode_query_signal_arguments(signal.get("arguments"), codec)?;
17526 signals.push(QuerySignal {
17527 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
17528 name: name.to_string(),
17529 arguments,
17530 avro_arguments,
17531 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
17532 });
17533 }
17534 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
17535 }
17536
17537 Ok(signals)
17538}
17539
17540fn decode_query_signal_arguments(
17541 raw: Option<&Value>,
17542 codec: &str,
17543) -> Result<(Vec<Value>, Vec<AvroValue>)> {
17544 validate_payload_codec(codec)?;
17545 let decoded = match raw.filter(|value| !value.is_null()) {
17546 Some(value) => decode_wire_avro_value(value, codec)?,
17547 None => AvroValue::Array(Vec::new()),
17548 };
17549 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
17550 unreachable!("normalize_avro_arguments always returns an array");
17551 };
17552 let arguments = avro_arguments
17553 .iter()
17554 .cloned()
17555 .map(AvroValue::into_json)
17556 .collect::<Result<Vec<_>>>()?;
17557 Ok((arguments, avro_arguments))
17558}
17559
17560fn value_as_u64(value: &Value) -> Option<u64> {
17561 value
17562 .as_u64()
17563 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
17564}
17565
17566#[cfg(test)]
17567mod tests {
17568 use super::*;
17569 mod activity_cancellation_policies;
17570 mod cancellation_scope_admission;
17571 mod cancellation_scope_authoring;
17572 mod cancellation_scope_history;
17573 mod cancellation_scope_opening;
17574 mod cancellation_scope_replay;
17575 mod child_workflow_policies;
17576 mod cooperative_cancellation;
17577 mod local_activity;
17578 mod runtime_payloads;
17579 mod runtime_uploads;
17580 mod worker_session;
17581 use std::{
17582 fs,
17583 io::{Read, Write},
17584 net::{SocketAddr, TcpListener, TcpStream},
17585 process::Command as ProcessCommand,
17586 sync::atomic::AtomicUsize,
17587 thread,
17588 };
17589
17590 #[derive(Clone, Copy, Debug)]
17591 enum InvalidTaskPayloadCodec {
17592 Missing,
17593 Null,
17594 NonString,
17595 }
17596
17597 impl InvalidTaskPayloadCodec {
17598 fn label(self) -> &'static str {
17599 match self {
17600 Self::Missing => "missing",
17601 Self::Null => "null",
17602 Self::NonString => "non-string",
17603 }
17604 }
17605
17606 fn apply(self, task: &mut Value) {
17607 let task = task.as_object_mut().expect("task fixture object");
17608 match self {
17609 Self::Missing => {
17610 task.remove("payload_codec");
17611 }
17612 Self::Null => {
17613 task.insert("payload_codec".to_string(), Value::Null);
17614 }
17615 Self::NonString => {
17616 task.insert("payload_codec".to_string(), json!(42));
17617 }
17618 }
17619 }
17620 }
17621
17622 fn fixture_envelope(value: Value) -> Value {
17623 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
17624 }
17625
17626 fn fixture_blob(value: Value) -> String {
17627 encode_payload(&value, DEFAULT_CODEC)
17628 .expect("encode Avro test fixture")
17629 .blob
17630 }
17631
17632 #[test]
17633 fn client_builder_rejects_the_sdk_owned_api_suffix() {
17634 for base_url in [
17635 "http://127.0.0.1:8080/api",
17636 "http://localhost:8080/api/",
17637 "https://runtime.example.test/namespaces/orders/api",
17638 ] {
17639 let error = Client::builder(base_url)
17640 .build()
17641 .expect_err("SDK-owned /api suffix must be rejected during build");
17642
17643 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
17644 assert!(
17645 error.to_string().contains("SDK appends /api automatically"),
17646 "the validation error must explain how to fix the endpoint"
17647 );
17648 }
17649 }
17650
17651 #[test]
17652 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
17653 for (base_url, expected) in [
17654 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
17655 (
17656 "http://localhost:8080/durable-workflow/",
17657 "http://localhost:8080/durable-workflow",
17658 ),
17659 (
17660 "https://runtime.example.test/namespaces/orders",
17661 "https://runtime.example.test/namespaces/orders",
17662 ),
17663 (
17664 "https://runtime.example.test/gateway/api/namespaces/orders",
17665 "https://runtime.example.test/gateway/api/namespaces/orders",
17666 ),
17667 (
17668 "https://api.example.test/runtime/orders/",
17669 "https://api.example.test/runtime/orders",
17670 ),
17671 ] {
17672 let client = Client::builder(base_url)
17673 .build()
17674 .expect("Server and Cloud runtime base URL must remain valid");
17675
17676 assert_eq!(client.base_url, expected);
17677 }
17678 }
17679
17680 #[test]
17681 fn workflow_completion_uses_the_additive_command_protocol_floor() {
17682 assert_eq!(
17683 workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
17684 WORKER_PROTOCOL_VERSION
17685 );
17686 assert_eq!(
17687 workflow_completion_protocol_version(&[json!({
17688 "type": "upsert_search_attributes",
17689 "attributes": {"OrderStatus": "waiting"},
17690 })]),
17691 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
17692 );
17693 assert_eq!(
17694 workflow_completion_protocol_version(&[json!({
17695 "type": "upsert_search_attributes",
17696 "attributes": {"OrderStatus": "waiting"},
17697 "attribute_types": {"OrderStatus": "keyword"},
17698 })]),
17699 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
17700 );
17701 assert_eq!(
17702 workflow_completion_protocol_version(&[
17703 json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
17704 json!({"type": "open_condition_wait", "condition_key": "ready"}),
17705 ]),
17706 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
17707 );
17708 assert_eq!(
17709 workflow_completion_protocol_version(&[
17710 json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
17711 json!({"type": "open_condition_wait", "condition_key": "ready"}),
17712 ]),
17713 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
17714 );
17715 assert_eq!(
17716 workflow_completion_protocol_version(&[json!({
17717 "type": "open_condition_wait",
17718 "condition_wait_occurrence_id": "rust:condition-wait:0",
17719 "condition_key": "ready",
17720 })]),
17721 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
17722 );
17723 assert_eq!(
17724 workflow_completion_protocol_version_with_message_streams(
17725 &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
17726 true,
17727 ),
17728 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
17729 );
17730 assert_eq!(
17731 workflow_completion_protocol_version_with_message_streams(
17732 &[json!({
17733 "type": "open_condition_wait",
17734 "condition_wait_occurrence_id": "rust:condition-wait:0",
17735 "condition_key": "ready",
17736 })],
17737 true,
17738 ),
17739 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
17740 );
17741 }
17742
17743 #[test]
17744 fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
17745 let manifest = portable_worker_affinity_capability_manifest();
17746
17747 for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
17748 assert_eq!(manifest[capability]["supported"], json!(false));
17749 assert_eq!(
17750 manifest[capability]["minimum_protocol_version"],
17751 json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
17752 );
17753 assert!(manifest[capability]["reason"]
17754 .as_str()
17755 .is_some_and(|reason| !reason.is_empty()));
17756 }
17757 }
17758
17759 fn typed_fidelity_probe() -> AvroValue {
17760 AvroValue::Map(BTreeMap::from([
17761 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
17762 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
17763 (
17764 "numeric".to_string(),
17765 AvroValue::Map(BTreeMap::from([
17766 ("0".to_string(), AvroValue::String("zero".to_string())),
17767 ("1".to_string(), AvroValue::String("one".to_string())),
17768 ])),
17769 ),
17770 (
17771 "nested".to_string(),
17772 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
17773 "enabled".to_string(),
17774 AvroValue::Boolean(true),
17775 )]))]),
17776 ),
17777 (
17778 "projection_collisions".to_string(),
17779 AvroValue::Array(projection_collision_probe()),
17780 ),
17781 ]))
17782 }
17783
17784 fn projection_collision_probe() -> Vec<AvroValue> {
17785 vec![
17786 AvroValue::Map(BTreeMap::from([
17787 ("$type".to_string(), AvroValue::String("bytes".to_string())),
17788 (
17789 "base64".to_string(),
17790 AvroValue::String("ordinary user text".to_string()),
17791 ),
17792 ])),
17793 AvroValue::Map(BTreeMap::from([
17794 ("$type".to_string(), AvroValue::String("map".to_string())),
17795 (
17796 "entries".to_string(),
17797 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
17798 ("key".to_string(), AvroValue::String("ordinary".to_string())),
17799 (
17800 "value".to_string(),
17801 AvroValue::String("user map".to_string()),
17802 ),
17803 ]))]),
17804 ),
17805 ])),
17806 ]
17807 }
17808
17809 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
17810 struct TypedContract {
17811 nested: TypedNested,
17812 mode: TypedMode,
17813 optional: Option<String>,
17814 absent: Option<String>,
17815 items: Vec<i64>,
17816 labels: BTreeMap<String, String>,
17817 bytes: serde_bytes::ByteBuf,
17818 signed: i64,
17819 finite: f64,
17820 }
17821
17822 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
17823 struct TypedNested {
17824 enabled: bool,
17825 }
17826
17827 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
17828 enum TypedMode {
17829 Detailed { label: String },
17830 }
17831
17832 fn typed_contract() -> TypedContract {
17833 TypedContract {
17834 nested: TypedNested { enabled: true },
17835 mode: TypedMode::Detailed {
17836 label: "compiler-checked".to_string(),
17837 },
17838 optional: Some("present".to_string()),
17839 absent: None,
17840 items: vec![i64::MIN, 0, i64::MAX],
17841 labels: BTreeMap::from([
17842 ("language".to_string(), "rust".to_string()),
17843 ("wire".to_string(), "avro".to_string()),
17844 ]),
17845 bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
17846 signed: -9_223_372_036_854_775_000,
17847 finite: 12.5,
17848 }
17849 }
17850
17851 #[derive(Clone, Debug, Default, PartialEq)]
17852 struct ReplayCounterState {
17853 loaded: Option<String>,
17854 count: i64,
17855 finished: bool,
17856 }
17857
17858 fn replay_counter_worker() -> Worker {
17859 let client = Client::new("http://127.0.0.1:8080").expect("client");
17860 let mut worker = Worker::new(client, "rust-workers");
17861 worker.register_replayed_workflow(
17862 "replay-counter",
17863 ReplayCounterState::default,
17864 |ctx, _input, state| async move {
17865 let loaded = ctx.activity("load-counter", json!([])).await?;
17866 state.update(|current| {
17867 current.loaded = loaded.as_str().map(str::to_string);
17868 })?;
17869 for _ in 0..2 {
17870 let signal = ctx.wait_signal("increment").await?;
17871 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
17872 state.update(|current| current.count += amount)?;
17873 }
17874 state.update(|current| current.finished = true)?;
17875 state.read(|current| Ok(json!(current.count)))?
17876 },
17877 );
17878 worker.register_replayed_query::<ReplayCounterState, _, _>(
17879 "replay-counter",
17880 "current",
17881 |_ctx, state, _args| async move {
17882 Ok(json!({
17883 "loaded": state.loaded,
17884 "count": state.count,
17885 "finished": state.finished,
17886 }))
17887 },
17888 );
17889 worker.register_replayed_query::<ReplayCounterState, _, _>(
17890 "replay-counter",
17891 "detached-mutation",
17892 |_ctx, state, _args| async move {
17893 let mut detached = (*state).clone();
17894 detached.count = 999;
17895 Ok(json!(detached.count))
17896 },
17897 );
17898 worker.register_replayed_query::<ReplayCounterState, _, _>(
17899 "replay-counter",
17900 "failed-mutation",
17901 |_ctx, state, _args| async move {
17902 let mut detached = (*state).clone();
17903 detached.count = 999;
17904 Err(Error::WorkerLoop("query refused".to_string()))
17905 },
17906 );
17907 worker
17908 }
17909
17910 fn replay_counter_query(
17911 query_name: &str,
17912 history_events: Value,
17913 run_status: &str,
17914 ) -> QueryTask {
17915 let arguments = fixture_envelope(json!([]));
17916 serde_json::from_value(json!({
17917 "query_task_id": format!("query-{query_name}"),
17918 "workflow_type": "replay-counter",
17919 "query_name": query_name,
17920 "payload_codec": DEFAULT_CODEC,
17921 "workflow_arguments": arguments.clone(),
17922 "query_arguments": arguments,
17923 "history_events": history_events,
17924 "run_status": run_status,
17925 }))
17926 .expect("query task")
17927 }
17928
17929 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
17930 workflow_context_with_codec(history, DEFAULT_CODEC)
17931 }
17932
17933 fn workflow_context_with_codec(
17934 history: Vec<HistoryEvent>,
17935 payload_codec: &str,
17936 ) -> WorkflowContext {
17937 WorkflowContext {
17938 cancellation_scope_id: "root".into(),
17939 state: Arc::new(Mutex::new(
17940 WorkflowState::new_with_identity(
17941 history,
17942 None,
17943 None,
17944 "rust-workers".to_string(),
17945 payload_codec.to_string(),
17946 None,
17947 )
17948 .expect("valid workflow history"),
17949 )),
17950 }
17951 }
17952
17953 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
17954 HistoryEvent {
17955 event_type: event_type.to_string(),
17956 payload,
17957 raw: HashMap::new(),
17958 }
17959 }
17960
17961 fn parallel_path_entry(
17962 kind: &str,
17963 base: u64,
17964 size: usize,
17965 index: usize,
17966 ) -> ParallelGroupMetadata {
17967 parallel_group_entry(base, size, index, kind)
17968 }
17969
17970 fn parallel_history_event(
17971 event_type: &str,
17972 sequence: u64,
17973 identity_field: &str,
17974 identity: &str,
17975 path: Vec<ParallelGroupMetadata>,
17976 result: Option<Value>,
17977 ) -> HistoryEvent {
17978 let mut payload = serde_json::Map::from_iter([
17979 ("sequence".to_string(), json!(sequence)),
17980 (identity_field.to_string(), json!(identity)),
17981 ]);
17982 let inner = path.last().expect("parallel history path");
17983 apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
17984 payload.insert("parallel_group_path".to_string(), json!(path));
17985 if let Some(result) = result {
17986 let field = if event_type == "ChildRunCompleted" {
17987 "result"
17988 } else {
17989 "result"
17990 };
17991 payload.insert(field.to_string(), fixture_envelope(result));
17992 payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
17993 }
17994 history_event(event_type, Value::Object(payload))
17995 }
17996
17997 fn nested_parallel_operations() -> Vec<ParallelOperation> {
17998 vec![
17999 ParallelOperation::activity("first", json!([])),
18000 ParallelOperation::group(vec![
18001 ParallelOperation::child_workflow(
18002 "second",
18003 ChildWorkflowOptions::new("child-workers"),
18004 json!([]),
18005 ),
18006 ParallelOperation::activity("third", json!([])),
18007 ]),
18008 ]
18009 }
18010
18011 fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
18012 let outer = [
18013 parallel_path_entry("mixed", 1, 3, 0),
18014 parallel_path_entry("mixed", 1, 3, 1),
18015 parallel_path_entry("mixed", 1, 3, 2),
18016 ];
18017 [
18018 vec![outer[0].clone()],
18019 vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
18020 vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
18021 ]
18022 }
18023
18024 #[test]
18025 fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
18026 let ctx = workflow_context(Vec::new());
18027 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
18028 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18029
18030 assert!(matches!(
18031 call.as_mut().poll(&mut task_context),
18032 Poll::Pending
18033 ));
18034 let commands = ctx.take_commands().expect("parallel commands");
18035 assert_eq!(
18036 commands
18037 .iter()
18038 .map(|command| command["type"].as_str().unwrap_or_default())
18039 .collect::<Vec<_>>(),
18040 [
18041 "schedule_activity",
18042 "start_child_workflow",
18043 "schedule_activity"
18044 ]
18045 );
18046 let paths = nested_parallel_paths();
18047 for (command, path) in commands.iter().zip(paths) {
18048 assert_eq!(command["parallel_group_path"], json!(path));
18049 assert_eq!(
18050 command["parallel_group_id"],
18051 json!(path.last().expect("inner group").parallel_group_id)
18052 );
18053 }
18054 }
18055
18056 fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
18057 let paths = nested_parallel_paths();
18058 let third = parallel_history_event(
18059 "ActivityCompleted",
18060 3,
18061 "activity_type",
18062 "third",
18063 paths[2].clone(),
18064 Some(json!("three")),
18065 );
18066 vec![
18067 parallel_history_event(
18068 "ActivityCompleted",
18069 1,
18070 "activity_type",
18071 "first",
18072 paths[0].clone(),
18073 Some(json!("one")),
18074 ),
18075 parallel_history_event(
18076 "ChildWorkflowScheduled",
18077 2,
18078 "child_workflow_type",
18079 "second",
18080 paths[1].clone(),
18081 None,
18082 ),
18083 parallel_history_event(
18084 "ChildRunCompleted",
18085 2,
18086 "child_workflow_type",
18087 "second",
18088 paths[1].clone(),
18089 Some(json!("two")),
18090 ),
18091 third.clone(),
18092 third,
18093 ]
18094 }
18095
18096 #[test]
18097 fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
18098 for _restart_or_completed_replay in 0..2 {
18099 let ctx = workflow_context(completed_nested_parallel_history());
18100 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
18101 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18102 let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
18103 panic!("completed nested parallel history must replay");
18104 };
18105 assert_eq!(
18106 results,
18107 vec![
18108 ParallelResult::Activity(json!("one")),
18109 ParallelResult::Group(vec![
18110 ParallelResult::ChildWorkflow(ChildWorkflowResult {
18111 parent: WorkflowIdentity {
18112 workflow_id: None,
18113 run_id: None,
18114 },
18115 child: WorkflowIdentity {
18116 workflow_id: None,
18117 run_id: None,
18118 },
18119 child_workflow_type: Some("second".to_string()),
18120 result: json!("two"),
18121 }),
18122 ParallelResult::Activity(json!("three")),
18123 ]),
18124 ]
18125 );
18126 assert!(ctx.take_commands().expect("commands").is_empty());
18127 ctx.ensure_history_consumed().expect("history consumed");
18128 }
18129 }
18130
18131 #[test]
18132 fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
18133 let paths = nested_parallel_paths();
18134 let history = vec![
18135 parallel_history_event(
18136 "ActivityCompleted",
18137 1,
18138 "activity_type",
18139 "first",
18140 paths[0].clone(),
18141 Some(json!("one")),
18142 ),
18143 parallel_history_event(
18144 "ChildWorkflowScheduled",
18145 2,
18146 "child_workflow_type",
18147 "second",
18148 paths[1].clone(),
18149 None,
18150 ),
18151 parallel_history_event(
18152 "ChildRunFailed",
18153 2,
18154 "child_workflow_type",
18155 "second",
18156 paths[1].clone(),
18157 None,
18158 ),
18159 parallel_history_event(
18160 "ActivityCompleted",
18161 3,
18162 "activity_type",
18163 "third",
18164 paths[2].clone(),
18165 Some(json!("late")),
18166 ),
18167 ];
18168 let ctx = workflow_context(history);
18169 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
18170 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18171 let outcome = call.as_mut().poll(&mut task_context);
18172 let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
18173 panic!("one failed child must return a typed partial failure: {outcome:?}");
18174 };
18175 assert_eq!(failure.member_path, [1, 0]);
18176 assert_eq!(failure.group_id, "parallel-calls:1:3");
18177 assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
18178 assert_eq!(
18179 failure
18180 .completed
18181 .iter()
18182 .map(|completion| completion.member_path.clone())
18183 .collect::<Vec<_>>(),
18184 [vec![0], vec![1, 1]]
18185 );
18186 }
18187
18188 #[test]
18189 fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
18190 let paths = nested_parallel_paths();
18191 let history = vec![
18192 parallel_history_event(
18193 "ActivityScheduled",
18194 1,
18195 "activity_type",
18196 "first",
18197 paths[0].clone(),
18198 None,
18199 ),
18200 parallel_history_event(
18201 "ChildWorkflowScheduled",
18202 2,
18203 "child_workflow_type",
18204 "second",
18205 paths[1].clone(),
18206 None,
18207 ),
18208 parallel_history_event(
18209 "ActivityScheduled",
18210 3,
18211 "activity_type",
18212 "third",
18213 paths[2].clone(),
18214 None,
18215 ),
18216 ];
18217 for _restart in 0..2 {
18218 let ctx = workflow_context(history.clone());
18219 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
18220 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18221 let outcome = call.as_mut().poll(&mut task_context);
18222 assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
18223 assert!(ctx.take_commands().expect("commands").is_empty());
18224 }
18225 }
18226
18227 fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
18228 vec![selection_group_entry(
18229 1,
18230 2,
18231 index,
18232 "activity",
18233 &SelectionMemberMetadata {
18234 key: SelectionKey::Name(key.to_string()),
18235 index,
18236 base_sequence: index as u64 + 1,
18237 size: 1,
18238 kind: "activity".to_string(),
18239 },
18240 )]
18241 }
18242
18243 fn selection_activity_event(
18244 event_type: &str,
18245 index: usize,
18246 key: &str,
18247 result: Option<Value>,
18248 ) -> HistoryEvent {
18249 let sequence = index as u64 + 1;
18250 let mut event = parallel_history_event(
18251 event_type,
18252 sequence,
18253 "activity_type",
18254 &format!("{key}-activity"),
18255 selection_path(index, key),
18256 result,
18257 );
18258 event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
18259 event.raw.insert(
18260 "id".to_string(),
18261 json!(if event_type == "ActivityCompleted" {
18262 format!("event-{key}")
18263 } else {
18264 format!("{event_type}-{key}")
18265 }),
18266 );
18267 event
18268 }
18269
18270 fn selection_winner_marker() -> HistoryEvent {
18271 history_event(
18272 "SelectionResolved",
18273 json!({
18274 "selection_group_id": "select-calls:1:2",
18275 "selection_group_base_sequence": 1,
18276 "selection_group_size": 2,
18277 "member_key": "fast",
18278 "member_index": 1,
18279 "member_base_sequence": 2,
18280 "member_size": 1,
18281 "operation_kind": "activity",
18282 "operation_identity": "activity-fast",
18283 "outcome": "completed",
18284 "resolution_event_id": "event-fast",
18285 "resolution_event_type": "ActivityCompleted",
18286 }),
18287 )
18288 }
18289
18290 fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
18291 ctx.select_keyed(vec![
18292 (
18293 "slow",
18294 ParallelOperation::activity_with_options(
18295 "slow-activity",
18296 ActivityOptions::new().task_queue("default"),
18297 json!([]),
18298 ),
18299 ),
18300 (
18301 "fast",
18302 ParallelOperation::activity_with_options(
18303 "fast-activity",
18304 ActivityOptions::new().task_queue("default"),
18305 json!([]),
18306 ),
18307 ),
18308 ])
18309 }
18310
18311 fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
18312 let ctx = workflow_context(history);
18313 let mut call = Box::pin(keyed_activity_selection(&ctx));
18314 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18315 let selected = match call.as_mut().poll(&mut task_context) {
18316 Poll::Ready(Ok(selected)) => selected,
18317 Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
18318 Poll::Pending => panic!("persisted selection winner must replay without pending"),
18319 };
18320 assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
18321 assert_eq!(
18322 selected.value,
18323 Some(ParallelResult::Activity(json!("winner-value")))
18324 );
18325 let slow = selected
18326 .handle(&SelectionKey::Name("slow".to_string()))
18327 .expect("slow handle")
18328 .clone();
18329 let mut await_slow = Box::pin(slow.await_result());
18330 assert!(matches!(
18331 await_slow.as_mut().poll(&mut task_context),
18332 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
18333 ));
18334 assert!(ctx.take_commands().expect("commands").is_empty());
18335 }
18336
18337 const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
18338
18339 fn canonical_selection_history() -> Vec<HistoryEvent> {
18340 const FIXTURE: &[u8] =
18341 include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
18342 assert_eq!(
18343 format!("{:x}", Sha256::digest(FIXTURE)),
18344 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
18345 );
18346 let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
18347
18348 serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
18349 }
18350
18351 #[test]
18352 fn selection_fresh_process_entrypoint() {
18353 let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
18354 return;
18355 };
18356 let persisted = fs::read(path).expect("persisted selection history");
18357 assert_eq!(
18358 format!("{:x}", Sha256::digest(&persisted)),
18359 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
18360 );
18361 let fixture: Value =
18362 serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
18363 let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
18364 .expect("valid persisted selection history");
18365
18366 assert_persisted_selection_replay(history);
18367 }
18368
18369 #[test]
18370 fn selection_starts_every_member_with_stable_keys_and_group_identity() {
18371 let ctx = workflow_context(Vec::new());
18372 let mut call = Box::pin(keyed_activity_selection(&ctx));
18373 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18374
18375 assert!(matches!(
18376 call.as_mut().poll(&mut task_context),
18377 Poll::Pending
18378 ));
18379 let commands = ctx.take_commands().expect("selection commands");
18380 assert_eq!(commands.len(), 2);
18381 assert_eq!(commands[0]["selection_member_key"], json!("slow"));
18382 assert_eq!(commands[1]["selection_member_key"], json!("fast"));
18383 assert!(commands.iter().all(|command| {
18384 command["parallel_group_id"] == json!("select-calls:1:2")
18385 && command["parallel_group_mode"] == json!("select")
18386 }));
18387 }
18388
18389 #[test]
18390 fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
18391 let ctx = workflow_context(Vec::new());
18392 let mut invalid = Box::pin(ctx.select_keyed(vec![(
18393 "",
18394 ParallelOperation::activity("invalid", json!([])),
18395 )]));
18396 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18397 assert!(matches!(
18398 invalid.as_mut().poll(&mut task_context),
18399 Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
18400 reason: "selection_key_invalid",
18401 ..
18402 })))
18403 ));
18404
18405 for invalid_key in [json!(""), json!(-1)] {
18406 let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
18407 event.payload["selection_member_key"] = invalid_key.clone();
18408 event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
18409 assert!(matches!(
18410 WorkflowState::new_with_identity(
18411 vec![event],
18412 None,
18413 None,
18414 "rust-workers".to_string(),
18415 DEFAULT_CODEC.to_string(),
18416 None,
18417 ),
18418 Err(Error::NonDeterministicReplay(_))
18419 ));
18420 }
18421 }
18422
18423 #[test]
18424 fn selection_preserves_valid_named_and_numeric_keys() {
18425 let ctx = workflow_context(Vec::new());
18426 let mut selection = Box::pin(ctx.select_keyed(vec![
18427 (
18428 SelectionKey::Index(0),
18429 ParallelOperation::activity("numeric", json!([])),
18430 ),
18431 (
18432 SelectionKey::Name("named".to_string()),
18433 ParallelOperation::timer(Duration::from_secs(1)),
18434 ),
18435 ]));
18436 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18437
18438 assert!(matches!(
18439 selection.as_mut().poll(&mut task_context),
18440 Poll::Pending
18441 ));
18442 let commands = ctx.take_commands().expect("selection commands");
18443 assert_eq!(commands[0]["selection_member_key"], json!(0));
18444 assert_eq!(commands[1]["selection_member_key"], json!("named"));
18445 }
18446
18447 #[test]
18448 fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
18449 let history = canonical_selection_history();
18450 assert_persisted_selection_replay(history.clone());
18451
18452 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
18453 .join("tests/fixtures/durable_selection_runtime_history.json");
18454 let output =
18455 ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
18456 .args([
18457 "--exact",
18458 "tests::selection_fresh_process_entrypoint",
18459 "--nocapture",
18460 ])
18461 .env(SELECTION_COLD_REPLAY_HISTORY, &path)
18462 .output()
18463 .expect("run fresh selection replay process");
18464
18465 assert!(
18466 output.status.success(),
18467 "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
18468 String::from_utf8_lossy(&output.stdout),
18469 String::from_utf8_lossy(&output.stderr),
18470 );
18471 }
18472
18473 #[test]
18474 fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
18475 let mut history = canonical_selection_history();
18476 history.retain(|event| event.event_type != "SelectionResolved");
18477 let ctx = workflow_context(history);
18478 let mut selection = Box::pin(keyed_activity_selection(&ctx));
18479 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18480
18481 assert!(matches!(
18482 selection.as_mut().poll(&mut task_context),
18483 Poll::Pending
18484 ));
18485 assert!(ctx.take_commands().expect("commands").is_empty());
18486 assert!(
18487 ctx.matched_recorded_pending()
18488 .expect("selection pending state"),
18489 "terminal member history must keep the workflow durably pending until SelectionResolved commits"
18490 );
18491 }
18492
18493 #[test]
18494 fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
18495 for (terminal_event, predicate_satisfied, timeout_seconds) in [
18496 ("ConditionWaitSatisfied", true, None),
18497 ("ConditionWaitTimedOut", false, Some(0)),
18498 ] {
18499 let member = SelectionMemberMetadata {
18500 key: SelectionKey::Name("condition".to_string()),
18501 index: 0,
18502 base_sequence: 1,
18503 size: 1,
18504 kind: "condition".to_string(),
18505 };
18506 let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
18507 let mut payload = json!({
18508 "sequence": 1,
18509 "condition_wait_id": "condition-1",
18510 "condition_wait_occurrence_id": "rust:condition-wait:0",
18511 "condition_key": "ready",
18512 "condition_definition_fingerprint": "sha256:ready-v1",
18513 "parallel_group_path": path,
18514 });
18515 payload
18516 .as_object_mut()
18517 .expect("condition history payload")
18518 .extend(
18519 serde_json::to_value(&path[0])
18520 .expect("condition selection metadata")
18521 .as_object()
18522 .expect("condition selection metadata object")
18523 .clone(),
18524 );
18525 if let Some(timeout_seconds) = timeout_seconds {
18526 payload["timeout_seconds"] = json!(timeout_seconds);
18527 }
18528 let history = vec![
18529 history_event("ConditionWaitOpened", payload.clone()),
18530 history_event(terminal_event, payload),
18531 ];
18532 let ctx = workflow_context(history);
18533 let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
18534 if timeout_seconds.is_some() {
18535 options = options.timeout(Duration::ZERO);
18536 }
18537 let mut selection = Box::pin(ctx.select_keyed(vec![(
18538 "condition",
18539 ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
18540 )]));
18541 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18542
18543 assert!(matches!(
18544 selection.as_mut().poll(&mut task_context),
18545 Poll::Pending
18546 ));
18547 assert!(ctx.take_commands().expect("commands").is_empty());
18548 assert!(
18549 ctx.matched_recorded_pending()
18550 .expect("condition selection pending state"),
18551 "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
18552 );
18553 }
18554 }
18555
18556 #[test]
18557 fn selection_immediate_condition_members_open_a_durable_wait() {
18558 for predicate_satisfied in [true, false] {
18559 let ctx = workflow_context(Vec::new());
18560 let mut selection = Box::pin(ctx.select_keyed(vec![(
18561 "condition",
18562 ParallelOperation::condition(
18563 ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
18564 move || Ok(predicate_satisfied),
18565 ),
18566 )]));
18567 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18568
18569 assert!(matches!(
18570 selection.as_mut().poll(&mut task_context),
18571 Poll::Pending
18572 ));
18573 let commands = ctx.take_commands().expect("condition selection command");
18574 assert_eq!(commands.len(), 1);
18575 assert_eq!(commands[0]["type"], json!("open_condition_wait"));
18576 assert_eq!(commands[0]["timeout_seconds"], json!(0));
18577 assert_eq!(
18578 commands[0]["parallel_group_path"][0]["parallel_group_mode"],
18579 json!("select")
18580 );
18581 }
18582 }
18583
18584 #[test]
18585 fn selection_loser_cancellation_is_explicit_and_idempotent() {
18586 let history = vec![
18587 selection_activity_event("ActivityScheduled", 0, "slow", None),
18588 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
18589 selection_winner_marker(),
18590 ];
18591 let ctx = workflow_context(history.clone());
18592 let mut call = Box::pin(keyed_activity_selection(&ctx));
18593 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18594 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
18595 panic!("winner must replay");
18596 };
18597 let slow = selected
18598 .handle(&SelectionKey::Name("slow".to_string()))
18599 .expect("slow handle")
18600 .clone();
18601 let mut cancel = Box::pin(slow.cancel());
18602 assert!(matches!(
18603 cancel.as_mut().poll(&mut task_context),
18604 Poll::Pending
18605 ));
18606 assert!(matches!(
18607 cancel.as_mut().poll(&mut task_context),
18608 Poll::Pending
18609 ));
18610 let commands = ctx.take_commands().expect("cancel command");
18611 assert_eq!(commands.len(), 1);
18612 assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
18613 assert_eq!(commands[0]["member_key"], json!("slow"));
18614
18615 let mut cancelled_history = history;
18616 cancelled_history.push(history_event(
18617 "SelectionOperationCancelled",
18618 json!({
18619 "selection_group_id": "select-calls:1:2",
18620 "member_key": "slow",
18621 "member_index": 0,
18622 "member_base_sequence": 1,
18623 "member_size": 1,
18624 "operation_kind": "activity",
18625 "operation_identity": "activity-slow",
18626 "cancelled_at": "2026-08-27T00:00:00Z",
18627 }),
18628 ));
18629 let replayed = workflow_context(cancelled_history);
18630 let mut call = Box::pin(keyed_activity_selection(&replayed));
18631 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
18632 panic!("winner must replay after cancellation");
18633 };
18634 let slow = selected
18635 .handle(&SelectionKey::Name("slow".to_string()))
18636 .expect("slow handle")
18637 .clone();
18638 let mut cancel = Box::pin(slow.cancel());
18639 assert!(matches!(
18640 cancel.as_mut().poll(&mut task_context),
18641 Poll::Ready(Ok(()))
18642 ));
18643 assert!(replayed.take_commands().expect("commands").is_empty());
18644 }
18645
18646 #[test]
18647 fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
18648 let base_history = vec![
18649 selection_activity_event("ActivityScheduled", 0, "slow", None),
18650 selection_activity_event("ActivityScheduled", 1, "fast", None),
18651 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
18652 selection_winner_marker(),
18653 ];
18654 for (field, corrupt) in [
18655 ("member_key", json!("fast")),
18656 ("member_index", json!(1)),
18657 ("member_base_sequence", json!(3)),
18658 ("member_size", json!(2)),
18659 ("operation_kind", json!("timer")),
18660 ("operation_identity", json!("forged")),
18661 ] {
18662 let mut cancellation = json!({
18663 "selection_group_id": "select-calls:1:2",
18664 "member_key": "slow",
18665 "member_index": 0,
18666 "member_base_sequence": 1,
18667 "member_size": 1,
18668 "operation_kind": "activity",
18669 "operation_identity": "activity-slow",
18670 });
18671 cancellation[field] = corrupt;
18672 let mut history = base_history.clone();
18673 history.push(history_event("SelectionOperationCancelled", cancellation));
18674 let ctx = workflow_context(history);
18675 let mut selection = Box::pin(keyed_activity_selection(&ctx));
18676 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18677
18678 assert!(matches!(
18679 selection.as_mut().poll(&mut task_context),
18680 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
18681 ));
18682 }
18683 }
18684
18685 #[test]
18686 fn selection_child_identity_prefers_the_durable_run_id() {
18687 let ctx = workflow_context(vec![history_event(
18688 "ChildWorkflowScheduled",
18689 json!({
18690 "sequence": 1,
18691 "child_workflow_type": "child",
18692 "child_workflow_instance_id": "child-instance",
18693 "child_workflow_run_id": "child-run",
18694 }),
18695 )]);
18696 let state = ctx.state.lock().expect("workflow state");
18697
18698 assert_eq!(
18699 selection_operation_identity(&state, "child", 1, 1),
18700 "child-run"
18701 );
18702 }
18703
18704 #[test]
18705 fn selection_activity_identity_requires_canonical_execution_id() {
18706 let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
18707 let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
18708 let mut fast_completed =
18709 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
18710 for event in [&mut fast_open, &mut fast_completed] {
18711 event
18712 .payload
18713 .as_object_mut()
18714 .expect("activity payload")
18715 .remove("activity_execution_id");
18716 event.payload["activity_id"] = json!("forged-activity-id");
18717 }
18718 let mut marker = selection_winner_marker();
18719 marker.payload["operation_identity"] = json!("forged-activity-id");
18720 let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
18721 let mut selection = Box::pin(keyed_activity_selection(&ctx));
18722 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18723
18724 assert!(matches!(
18725 selection.as_mut().poll(&mut task_context),
18726 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
18727 ));
18728 }
18729
18730 #[test]
18731 fn selection_completion_before_cancellation_remains_awaitable() {
18732 let history = vec![
18733 selection_activity_event("ActivityScheduled", 0, "slow", None),
18734 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
18735 selection_winner_marker(),
18736 selection_activity_event(
18737 "ActivityCompleted",
18738 0,
18739 "slow",
18740 Some(json!("completed-first")),
18741 ),
18742 ];
18743 let ctx = workflow_context(history);
18744 let mut selection = Box::pin(keyed_activity_selection(&ctx));
18745 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18746 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
18747 panic!("winner must replay");
18748 };
18749 let slow = selected
18750 .handle(&SelectionKey::Name("slow".to_string()))
18751 .expect("slow handle")
18752 .clone();
18753 let mut cancel = Box::pin(slow.cancel());
18754 assert!(matches!(
18755 cancel.as_mut().poll(&mut task_context),
18756 Poll::Ready(Ok(()))
18757 ));
18758 let mut await_slow = Box::pin(slow.await_result());
18759 assert!(matches!(
18760 await_slow.as_mut().poll(&mut task_context),
18761 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
18762 ));
18763 let commands = ctx.take_commands().expect("commands");
18764 assert!(commands.is_empty());
18765 }
18766
18767 #[test]
18768 fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
18769 let nested_member = SelectionMemberMetadata {
18770 key: SelectionKey::Name("nested".to_string()),
18771 index: 0,
18772 base_sequence: 1,
18773 size: 2,
18774 kind: "group".to_string(),
18775 };
18776 let deadline_member = SelectionMemberMetadata {
18777 key: SelectionKey::Name("deadline".to_string()),
18778 index: 1,
18779 base_sequence: 3,
18780 size: 1,
18781 kind: "timer".to_string(),
18782 };
18783 let nested_paths = [
18784 vec![
18785 selection_group_entry(1, 3, 0, "mixed", &nested_member),
18786 parallel_group_entry(1, 2, 0, "activity"),
18787 ],
18788 vec![
18789 selection_group_entry(1, 3, 1, "mixed", &nested_member),
18790 parallel_group_entry(1, 2, 1, "activity"),
18791 ],
18792 ];
18793 let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
18794 let mut timer_fired = parallel_history_event(
18795 "TimerFired",
18796 3,
18797 "timer_id",
18798 "timer-3",
18799 deadline_path.clone(),
18800 None,
18801 );
18802 timer_fired.payload["delay_seconds"] = json!(0);
18803 timer_fired
18804 .raw
18805 .insert("id".to_string(), json!("timer-fired"));
18806 let mut timer_scheduled = parallel_history_event(
18807 "TimerScheduled",
18808 3,
18809 "timer_id",
18810 "timer-3",
18811 deadline_path,
18812 None,
18813 );
18814 timer_scheduled.payload["delay_seconds"] = json!(0);
18815 let history = vec![
18816 parallel_history_event(
18817 "ActivityScheduled",
18818 1,
18819 "activity_type",
18820 "nested-first",
18821 nested_paths[0].clone(),
18822 None,
18823 ),
18824 parallel_history_event(
18825 "ActivityScheduled",
18826 2,
18827 "activity_type",
18828 "nested-second",
18829 nested_paths[1].clone(),
18830 None,
18831 ),
18832 timer_scheduled,
18833 timer_fired,
18834 history_event(
18835 "SelectionResolved",
18836 json!({
18837 "selection_group_id": "select-calls:1:3",
18838 "selection_group_base_sequence": 1,
18839 "selection_group_size": 3,
18840 "member_key": "deadline",
18841 "member_index": 1,
18842 "member_base_sequence": 3,
18843 "member_size": 1,
18844 "operation_kind": "timer",
18845 "operation_identity": "timer-3",
18846 "outcome": "completed",
18847 "resolution_event_id": "timer-fired",
18848 "resolution_event_type": "TimerFired",
18849 }),
18850 ),
18851 parallel_history_event(
18852 "ActivityFailed",
18853 2,
18854 "activity_type",
18855 "nested-second",
18856 nested_paths[1].clone(),
18857 None,
18858 ),
18859 ];
18860 let ctx = workflow_context(history);
18861 let mut selection = Box::pin(ctx.select_keyed(vec![
18862 (
18863 "nested",
18864 ParallelOperation::group(vec![
18865 ParallelOperation::activity("nested-first", json!([])),
18866 ParallelOperation::activity("nested-second", json!([])),
18867 ]),
18868 ),
18869 ("deadline", ParallelOperation::timer(Duration::ZERO)),
18870 ]));
18871 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18872 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
18873 panic!("deadline winner must replay");
18874 };
18875 let nested = selected
18876 .handle(&SelectionKey::Name("nested".to_string()))
18877 .expect("nested handle")
18878 .clone();
18879 let mut cancel = Box::pin(nested.cancel());
18880 assert!(matches!(
18881 cancel.as_mut().poll(&mut task_context),
18882 Poll::Ready(Ok(()))
18883 ));
18884 let mut await_nested = Box::pin(nested.await_result());
18885
18886 assert!(matches!(
18887 await_nested.as_mut().poll(&mut task_context),
18888 Poll::Ready(Err(Error::ActivityFailed(_)))
18889 ));
18890 assert!(ctx.take_commands().expect("commands").is_empty());
18891 }
18892
18893 #[test]
18894 fn selection_supports_child_timer_signal_condition_and_nested_groups() {
18895 let ctx = workflow_context(Vec::new());
18896 let mut call = Box::pin(ctx.select(vec![
18897 ParallelOperation::child_workflow(
18898 "child",
18899 ChildWorkflowOptions::new("children"),
18900 json!([]),
18901 ),
18902 ParallelOperation::timer(Duration::from_secs(30)),
18903 ParallelOperation::signal("approval"),
18904 ParallelOperation::condition(
18905 ConditionWaitOptions::new("ready", "sha256:ready"),
18906 || Ok(false),
18907 ),
18908 ParallelOperation::group(vec![
18909 ParallelOperation::activity("nested-one", json!([])),
18910 ParallelOperation::activity("nested-two", json!([])),
18911 ]),
18912 ]));
18913 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18914 assert!(matches!(
18915 call.as_mut().poll(&mut task_context),
18916 Poll::Pending
18917 ));
18918 let commands = ctx.take_commands().expect("selection commands");
18919 assert_eq!(
18920 commands
18921 .iter()
18922 .map(|command| command["type"].as_str().unwrap_or_default())
18923 .collect::<Vec<_>>(),
18924 [
18925 "start_child_workflow",
18926 "start_timer",
18927 "open_signal_wait",
18928 "open_condition_wait",
18929 "schedule_activity",
18930 "schedule_activity",
18931 ]
18932 );
18933 assert!(commands.iter().all(|command| {
18934 command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
18935 }));
18936 assert_eq!(
18937 commands[4]["parallel_group_path"].as_array().map(Vec::len),
18938 Some(2)
18939 );
18940 assert_eq!(
18941 commands[4]["parallel_group_path"][0]["selection_member_kind"],
18942 json!("group")
18943 );
18944 assert_eq!(
18945 commands[5]["parallel_group_path"][0]["selection_member_kind"],
18946 json!("group")
18947 );
18948
18949 let one_leaf_ctx = workflow_context(Vec::new());
18950 let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
18951 ParallelOperation::activity("nested-only", json!([])),
18952 ])]));
18953 assert!(matches!(
18954 one_leaf.as_mut().poll(&mut task_context),
18955 Poll::Pending
18956 ));
18957 let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
18958 assert_eq!(one_leaf_commands.len(), 1);
18959 assert_eq!(
18960 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
18961 json!("group")
18962 );
18963 assert_eq!(
18964 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
18965 json!(1)
18966 );
18967 }
18968
18969 async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
18970 let mut saga = ctx.saga();
18971 let outcome = async {
18972 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
18973 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
18974 let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
18975 saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
18976 ctx.activity("trip.charge", json!([])).await?;
18977 Ok(json!({"status": "booked"}))
18978 }
18979 .await;
18980 saga.finish(outcome).await
18981 }
18982
18983 fn saga_activity(
18984 event_type: &str,
18985 sequence: u64,
18986 activity_type: &str,
18987 result: Option<Value>,
18988 ) -> HistoryEvent {
18989 let mut payload = json!({
18990 "sequence": sequence,
18991 "activity_type": activity_type,
18992 "message": format!("{activity_type} failed"),
18993 "exception_type": "PlannedFailure",
18994 "non_retryable": true,
18995 });
18996 if let Some(result) = result {
18997 payload["result"] = fixture_envelope(result);
18998 }
18999 history_event(event_type, payload)
19000 }
19001
19002 #[test]
19003 fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
19004 let completed_hotel_compensation = saga_activity(
19005 "ActivityCompleted",
19006 4,
19007 "trip.cancel-hotel",
19008 Some(Value::Null),
19009 );
19010 let history = vec![
19011 saga_activity(
19012 "ActivityCompleted",
19013 1,
19014 "trip.reserve-flight",
19015 Some(json!("flight-1")),
19016 ),
19017 saga_activity(
19018 "ActivityCompleted",
19019 2,
19020 "trip.reserve-hotel",
19021 Some(json!("hotel-1")),
19022 ),
19023 saga_activity("ActivityFailed", 3, "trip.charge", None),
19024 completed_hotel_compensation.clone(),
19025 completed_hotel_compensation,
19026 ];
19027
19028 for _restart in 0..2 {
19029 let ctx = workflow_context(history.clone());
19030 let mut future = Box::pin(trip_saga(ctx.clone()));
19031 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19032 assert!(matches!(
19033 future.as_mut().poll(&mut task_context),
19034 Poll::Pending
19035 ));
19036 let commands = ctx.take_commands().expect("compensation command");
19037 assert_eq!(commands.len(), 1);
19038 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
19039 }
19040 }
19041
19042 #[test]
19043 fn saga_compensation_failure_preserves_both_typed_failures() {
19044 let history = vec![
19045 saga_activity(
19046 "ActivityCompleted",
19047 1,
19048 "trip.reserve-flight",
19049 Some(json!("flight-1")),
19050 ),
19051 saga_activity(
19052 "ActivityCompleted",
19053 2,
19054 "trip.reserve-hotel",
19055 Some(json!("hotel-1")),
19056 ),
19057 saga_activity("ActivityFailed", 3, "trip.charge", None),
19058 saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
19059 ];
19060 let ctx = workflow_context(history);
19061 let mut future = Box::pin(trip_saga(ctx));
19062 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19063 let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
19064 future.as_mut().poll(&mut task_context)
19065 else {
19066 panic!("compensation failure must remain structured");
19067 };
19068 assert!(matches!(
19069 *failure.initiating_failure,
19070 Error::ActivityFailed(_)
19071 ));
19072 assert!(matches!(
19073 *failure.compensation_failure,
19074 Error::ActivityFailed(_)
19075 ));
19076 assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
19077 assert_eq!(failure.compensation_registration_order, 2);
19078 }
19079
19080 #[test]
19081 fn saga_compensates_cooperative_cancellation() {
19082 let ctx = workflow_context(vec![saga_activity(
19083 "ActivityCompleted",
19084 1,
19085 "trip.reserve-flight",
19086 Some(json!("flight-1")),
19087 )]);
19088 ctx.state.lock().expect("state").cancel_requested = true;
19089 let run = {
19090 let ctx = ctx.clone();
19091 async move {
19092 let mut saga = ctx.saga();
19093 let outcome = async {
19094 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
19095 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
19096 ctx.throw_if_cancellation_requested()?;
19097 Ok(json!("unexpected"))
19098 }
19099 .await;
19100 saga.finish(outcome).await
19101 }
19102 };
19103 let mut future = Box::pin(run);
19104 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19105 assert!(matches!(
19106 future.as_mut().poll(&mut task_context),
19107 Poll::Pending
19108 ));
19109 let commands = ctx.take_commands().expect("cancellation compensation");
19110 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
19111 }
19112
19113 fn workflow_task(
19114 workflow_type: &str,
19115 history_events: Vec<HistoryEvent>,
19116 payload_codec: &str,
19117 ) -> WorkflowTask {
19118 WorkflowTask {
19119 task_id: format!("wft-{workflow_type}"),
19120 workflow_command_id: None,
19121 workflow_id: Some(format!("wf-{workflow_type}")),
19122 run_id: Some(format!("run-{workflow_type}")),
19123 workflow_type: workflow_type.to_string(),
19124 cancel_requested: false,
19125 payload_codec: payload_codec.to_string(),
19126 arguments: Some(
19127 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
19128 ),
19129 total_history_events: Some(history_events.len() as u64),
19130 history_size_bytes: None,
19131 continue_as_new_recommended: None,
19132 history_budget_pressure: None,
19133 history_events,
19134 next_history_page_token: None,
19135 workflow_task_attempt: 1,
19136 workflow_signal_id: None,
19137 signal_name: None,
19138 signal_arguments: None,
19139 workflow_update_id: None,
19140 update_name: None,
19141 lease_owner: Some("rust-worker".to_string()),
19142 }
19143 }
19144
19145 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
19146 struct SideEffectProbe {
19147 request_id: String,
19148 attempt: u32,
19149 }
19150
19151 #[test]
19152 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
19153 let calls = AtomicUsize::new(0);
19154 let ctx = workflow_context(Vec::new());
19155 let value = ctx
19156 .side_effect(|| {
19157 calls.fetch_add(1, Ordering::SeqCst);
19158 SideEffectProbe {
19159 request_id: "request-42".to_string(),
19160 attempt: 3,
19161 }
19162 })
19163 .expect("first side effect");
19164 assert_eq!(value.attempt, 3);
19165 assert_eq!(calls.load(Ordering::SeqCst), 1);
19166 let commands = ctx.take_commands().expect("commands");
19167 assert_eq!(commands.len(), 1);
19168 assert_eq!(commands[0]["type"], "record_side_effect");
19169 assert_eq!(
19170 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
19171 serde_json::to_value(&value).expect("value")
19172 );
19173
19174 let replay = workflow_context(vec![history_event(
19175 "SideEffectRecorded",
19176 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
19177 )]);
19178 let replayed: SideEffectProbe = replay
19179 .side_effect(|| {
19180 calls.fetch_add(1, Ordering::SeqCst);
19181 panic!("committed side-effect callbacks must not run during replay")
19182 })
19183 .expect("replayed side effect");
19184 assert_eq!(replayed, value);
19185 assert_eq!(calls.load(Ordering::SeqCst), 1);
19186 assert!(replay.take_commands().expect("commands").is_empty());
19187 replay.ensure_history_consumed().expect("history consumed");
19188 }
19189
19190 #[test]
19191 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
19192 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
19193 let value = ctx
19194 .side_effect(|| SideEffectProbe {
19195 request_id: "avro-request".to_string(),
19196 attempt: 1,
19197 })
19198 .expect("Avro side effect");
19199 let uuid = ctx.uuid_v4().expect("deterministic UUID");
19200 let commands = ctx.take_commands().expect("commands");
19201 assert_eq!(commands.len(), 2);
19202 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
19203 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
19204 assert_eq!(
19205 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
19206 serde_json::to_value(&value).expect("value")
19207 );
19208
19209 let replay = workflow_context_with_codec(
19210 vec![
19211 history_event(
19212 "SideEffectRecorded",
19213 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
19214 ),
19215 history_event(
19216 "SideEffectRecorded",
19217 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
19218 ),
19219 ],
19220 DEFAULT_CODEC,
19221 );
19222 let replayed: SideEffectProbe = replay
19223 .side_effect(|| panic!("Avro callback must not run"))
19224 .expect("replayed Avro value");
19225 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
19226 assert_eq!(replayed, value);
19227 assert_eq!(replayed_uuid, uuid);
19228 assert!(replay.take_commands().expect("commands").is_empty());
19229 }
19230
19231 #[test]
19232 fn typed_side_effect_replay_preserves_bytes_and_maps() {
19233 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
19234 let value = ctx
19235 .side_effect_avro_value(typed_fidelity_probe)
19236 .expect("typed side effect");
19237 let commands = ctx.take_commands().expect("side-effect command");
19238 assert_eq!(
19239 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
19240 .expect("recorded side effect"),
19241 value
19242 );
19243
19244 let replay = workflow_context_with_codec(
19245 vec![history_event(
19246 "SideEffectRecorded",
19247 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
19248 )],
19249 DEFAULT_CODEC,
19250 );
19251 assert_eq!(
19252 replay
19253 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
19254 .expect("replayed typed side effect"),
19255 value
19256 );
19257 }
19258
19259 #[test]
19260 fn ordered_side_effects_share_the_durable_command_stream() {
19261 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
19262 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
19263 let ctx = workflow_context(vec![
19264 history_event(
19265 "SideEffectRecorded",
19266 json!({"sequence": 1, "result": first}),
19267 ),
19268 history_event(
19269 "SideEffectRecorded",
19270 json!({"sequence": 2, "result": second}),
19271 ),
19272 ]);
19273 let first: String = ctx
19274 .side_effect(|| panic!("first callback must not run"))
19275 .expect("first replay");
19276 let second: i32 = ctx
19277 .side_effect(|| panic!("second callback must not run"))
19278 .expect("second replay");
19279 assert_eq!(first, "first");
19280 assert_eq!(second, 29);
19281 ctx.ensure_history_consumed().expect("ordered history");
19282
19283 let reordered = workflow_context(vec![history_event(
19284 "VersionMarkerRecorded",
19285 json!({
19286 "sequence": 1,
19287 "change_id": "before-side-effect",
19288 "version": 1,
19289 "min_supported": 1,
19290 "max_supported": 1,
19291 }),
19292 )]);
19293 let error = reordered
19294 .side_effect(|| "new".to_string())
19295 .expect_err("command reordering must fail");
19296 assert!(matches!(
19297 error,
19298 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19299 if reason == "recorded_command_mismatch"
19300 ));
19301 }
19302
19303 #[test]
19304 fn legacy_patch_preserves_pending_and_completed_activity_history() {
19305 for completed in [false, true] {
19306 let mut history = vec![history_event(
19307 "ActivityScheduled",
19308 json!({"sequence": 1, "activity_type": "old"}),
19309 )];
19310 if completed {
19311 history.push(history_event(
19312 "ActivityCompleted",
19313 json!({
19314 "sequence": 1,
19315 "activity_type": "old",
19316 "result": encode_value_envelope(&json!(41), DEFAULT_CODEC).unwrap(),
19317 }),
19318 ));
19319 }
19320 for _cold_replay in 0..2 {
19321 let ctx = workflow_context(history.clone());
19322 assert!(!ctx.patched("added-step").expect("legacy patch decision"));
19323 assert!(!ctx.patched("added-step").expect("same frozen decision"));
19324 ctx.deprecate_patch("added-step")
19325 .expect("retained boundary");
19326 let mut activity = Box::pin(ctx.activity("old", json!([])));
19327 let mut context = TaskContext::from_waker(noop_waker_ref());
19328 match activity.as_mut().poll(&mut context) {
19329 Poll::Ready(result) => {
19330 assert!(completed);
19331 assert_eq!(result.expect("recorded result"), json!(41));
19332 ctx.ensure_history_consumed().expect("old command retained");
19333 }
19334 Poll::Pending => assert!(!completed),
19335 }
19336 assert!(ctx.take_commands().expect("commands").is_empty());
19337 }
19338 }
19339 }
19340
19341 #[test]
19342 fn legacy_patch_preserves_an_unmarked_completed_workflow() {
19343 let ctx = workflow_context(vec![history_event("WorkflowCompleted", json!({}))]);
19344 assert!(!ctx.patched("added-step").expect("legacy completed run"));
19345 ctx.deprecate_patch("added-step")
19346 .expect("same legacy decision");
19347 assert!(ctx.take_commands().expect("commands").is_empty());
19348 ctx.ensure_history_consumed().expect("completed history");
19349 }
19350
19351 #[test]
19352 fn legacy_patch_rejects_a_range_without_the_original_branch() {
19353 let ctx = workflow_context(vec![history_event(
19354 "ActivityScheduled",
19355 json!({"sequence": 1, "activity_type": "old"}),
19356 )]);
19357 let error = ctx
19358 .get_version("added-step", 0, 1)
19359 .expect_err("legacy is unsupported");
19360 assert!(matches!(
19361 error,
19362 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19363 if reason == "version_marker_incompatible_range"
19364 ));
19365 assert!(!ctx
19366 .patched("supported-step")
19367 .expect("failed check did not consume history"));
19368 assert!(ctx.take_commands().expect("commands").is_empty());
19369 }
19370
19371 fn retained_patch_marker(sequence: u64, version: i32, time: &str) -> HistoryEvent {
19372 let mut event = history_event(
19373 "VersionMarkerRecorded",
19374 json!({
19375 "sequence": sequence,
19376 "change_id": "retained-patch",
19377 "version": version,
19378 "min_supported": -1,
19379 "max_supported": 1,
19380 }),
19381 );
19382 event.raw.insert("timestamp".into(), json!(time));
19383 event
19384 }
19385
19386 #[test]
19387 fn retained_patch_markers_preserve_each_original_boundary() {
19388 for completed in [false, true] {
19389 let mut history = vec![
19390 retained_patch_marker(1, 1, "2026-10-10T21:47:29.120756Z"),
19391 retained_patch_marker(2, 1, "2026-10-10T21:47:29.123604Z"),
19392 history_event(
19393 "ActivityScheduled",
19394 json!({"sequence": 3, "activity_type": "old"}),
19395 ),
19396 ];
19397 if completed {
19398 history.push(history_event(
19399 "ActivityCompleted",
19400 json!({
19401 "sequence": 3,
19402 "activity_type": "old",
19403 "result": fixture_envelope(json!(41)),
19404 }),
19405 ));
19406 }
19407 for _cold_replay in 0..2 {
19408 let ctx = workflow_context(history.clone());
19409 assert!(ctx.patched("retained-patch").unwrap());
19410 assert_eq!(ctx.state.lock().unwrap().command_cursor, 1);
19411 assert!(ctx.patched("retained-patch").unwrap());
19412 assert_eq!(ctx.state.lock().unwrap().command_cursor, 2);
19413 ctx.deprecate_patch("retained-patch").unwrap();
19414 assert_eq!(ctx.state.lock().unwrap().command_cursor, 2);
19415 let mut activity = Box::pin(ctx.activity("old", json!([])));
19416 let mut context = TaskContext::from_waker(noop_waker_ref());
19417 match activity.as_mut().poll(&mut context) {
19418 Poll::Ready(result) => {
19419 assert!(completed);
19420 assert_eq!(result.unwrap(), json!(41));
19421 ctx.ensure_history_consumed().unwrap();
19422 }
19423 Poll::Pending => assert!(!completed),
19424 }
19425 assert!(ctx.take_commands().unwrap().is_empty());
19426 let state = ctx.state.lock().unwrap();
19427 for (actual, original) in state.history_events.iter().zip(&history) {
19428 assert_eq!(actual.payload, original.payload);
19429 assert_eq!(actual.raw, original.raw);
19430 }
19431 }
19432 }
19433 }
19434
19435 #[test]
19436 fn retained_patch_aliases_do_not_skip_intervening_operations() {
19437 let ctx = workflow_context(vec![
19438 retained_patch_marker(1, 1, "2026-10-10T21:47:29.120756Z"),
19439 history_event(
19440 "SideEffectRecorded",
19441 json!({
19442 "sequence": 2, "result": fixture_envelope(json!("between")),
19443 }),
19444 ),
19445 retained_patch_marker(3, 1, "2026-10-10T21:47:29.123604Z"),
19446 history_event(
19447 "VersionMarkerRecorded",
19448 json!({
19449 "sequence": 4, "change_id": "other-patch", "version": 1,
19450 "min_supported": -1, "max_supported": 1,
19451 }),
19452 ),
19453 ]);
19454 assert!(ctx.patched("retained-patch").unwrap());
19455 assert!(ctx.patched("retained-patch").unwrap());
19456 assert_eq!(ctx.state.lock().unwrap().command_cursor, 1);
19457 assert_eq!(
19458 ctx.side_effect(|| "must not execute".to_string()).unwrap(),
19459 "between"
19460 );
19461 ctx.deprecate_patch("retained-patch").unwrap();
19462 assert_eq!(ctx.state.lock().unwrap().command_cursor, 3);
19463 ctx.deprecate_patch("retained-patch").unwrap();
19464 assert_eq!(ctx.state.lock().unwrap().command_cursor, 3);
19465 assert!(ctx.patched("other-patch").unwrap());
19466 ctx.ensure_history_consumed().unwrap();
19467 assert!(ctx.take_commands().unwrap().is_empty());
19468 }
19469
19470 #[test]
19471 fn retained_patch_alias_range_validation_leaves_the_cursor_unchanged() {
19472 let mut alias = retained_patch_marker(2, 1, "2026-10-10T21:47:29.123604Z");
19473 alias.payload["max_supported"] = json!(3);
19474 let ctx = workflow_context(vec![
19475 retained_patch_marker(1, 1, "2026-10-10T21:47:29.120756Z"),
19476 alias,
19477 ]);
19478 assert_eq!(ctx.get_version("retained-patch", -1, 2).unwrap(), 1);
19479 let error = ctx.get_version("retained-patch", 2, 3).unwrap_err();
19480 assert!(matches!(error,
19481 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19482 if reason == "version_marker_incompatible_range"));
19483 assert_eq!(ctx.state.lock().unwrap().command_cursor, 1);
19484 assert_eq!(ctx.get_version("retained-patch", -1, 3).unwrap(), 1);
19485 ctx.ensure_history_consumed().unwrap();
19486 assert!(ctx.take_commands().unwrap().is_empty());
19487 }
19488
19489 #[test]
19490 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
19491 let ctx = workflow_context(Vec::new());
19492 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
19493 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
19494 assert!(ctx.patched("new-search").expect("patch"));
19495 ctx.deprecate_patch("new-search").expect("deprecate patch");
19496 let commands = ctx.take_commands().expect("commands");
19497 assert_eq!(commands.len(), 2);
19498 assert_eq!(commands[0]["type"], "record_version_marker");
19499 assert_eq!(commands[0]["version"], 2);
19500 assert_eq!(commands[1]["change_id"], "new-search");
19501
19502 let replay = workflow_context(vec![history_event(
19503 "VersionMarkerRecorded",
19504 json!({
19505 "sequence": 1,
19506 "change_id": "checkout-v2",
19507 "version": 2,
19508 "min_supported": 1,
19509 "max_supported": 2,
19510 }),
19511 )]);
19512 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
19513 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
19514 assert!(replay.take_commands().expect("commands").is_empty());
19515 replay.ensure_history_consumed().expect("history consumed");
19516 }
19517
19518 #[test]
19519 fn version_markers_reject_incompatible_or_malformed_history() {
19520 let incompatible = workflow_context(vec![history_event(
19521 "VersionMarkerRecorded",
19522 json!({
19523 "sequence": 1,
19524 "change_id": "checkout-v2",
19525 "version": 1,
19526 "min_supported": 1,
19527 "max_supported": 2,
19528 }),
19529 )]);
19530 let error = incompatible
19531 .get_version("checkout-v2", 2, 3)
19532 .expect_err("old version is unsupported");
19533 assert!(matches!(
19534 error,
19535 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19536 if reason == "version_marker_incompatible_range"
19537 ));
19538
19539 for (history, reason) in [
19540 (
19541 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
19542 "side_effect_result_missing",
19543 ),
19544 (
19545 vec![history_event(
19546 "SideEffectRecorded",
19547 json!({
19548 "sequence": 1,
19549 "result": {"codec": "avro", "blob": "not-base64"},
19550 }),
19551 )],
19552 "side_effect_payload_incompatible",
19553 ),
19554 (
19555 vec![history_event(
19556 "SideEffectRecorded",
19557 json!({"sequence": 1, "result": {"unwrapped": true}}),
19558 )],
19559 "side_effect_payload_malformed",
19560 ),
19561 (
19562 vec![history_event(
19563 "VersionMarkerRecorded",
19564 json!({
19565 "sequence": 1,
19566 "change_id": "change",
19567 "version": 1,
19568 "min_supported": 2,
19569 "max_supported": 1,
19570 }),
19571 )],
19572 "version_marker_history_range_invalid",
19573 ),
19574 ] {
19575 let error = WorkflowState::new(
19576 history,
19577 "rust-workers".to_string(),
19578 DEFAULT_CODEC.to_string(),
19579 None,
19580 )
19581 .expect_err("malformed history must fail");
19582 assert!(matches!(
19583 error,
19584 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
19585 if actual == reason
19586 ));
19587 }
19588 }
19589
19590 #[test]
19591 fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
19592 let history = vec![history_event(
19593 "SearchAttributesUpserted",
19594 json!({
19595 "sequence": 1,
19596 "attributes": {"customer_tier": "gold"},
19597 "attribute_types": {"customer_tier": "keyword"},
19598 "merged": {"customer_tier": "gold"}
19599 }),
19600 )];
19601
19602 let matching = workflow_context(history.clone());
19603 matching
19604 .upsert_search_attributes(
19605 SearchAttributeUpdate::new()
19606 .keyword("customer_tier", "gold")
19607 .expect("keyword update"),
19608 )
19609 .expect("matching typed update must replay");
19610 matching
19611 .ensure_history_consumed()
19612 .expect("history consumed");
19613
19614 let changed_type = workflow_context(history.clone());
19615 let error = changed_type
19616 .upsert_search_attributes(
19617 SearchAttributeUpdate::new()
19618 .string("customer_tier", "gold")
19619 .expect("string update"),
19620 )
19621 .expect_err("same JSON value with a different declaration must be nondeterministic");
19622 let Error::NonDeterministicReplay(failure) = error else {
19623 panic!("typed identity drift must be a replay failure");
19624 };
19625 assert_eq!(failure.reason, "search_attribute_type_mismatch");
19626 assert_eq!(failure.sequence, Some(1));
19627
19628 let changed_value = workflow_context(history);
19629 let error = changed_value
19630 .upsert_search_attributes(
19631 SearchAttributeUpdate::new()
19632 .keyword("customer_tier", "platinum")
19633 .expect("keyword update"),
19634 )
19635 .expect_err("changed values must be nondeterministic");
19636 let Error::NonDeterministicReplay(failure) = error else {
19637 panic!("value drift must be a replay failure");
19638 };
19639 assert_eq!(failure.reason, "search_attribute_value_mismatch");
19640 }
19641
19642 #[test]
19643 fn legacy_search_attribute_history_keeps_type_identity_unknown() {
19644 let history = vec![history_event(
19645 "SearchAttributesUpserted",
19646 json!({
19647 "sequence": 1,
19648 "attributes": {"customer_tier": "gold"},
19649 "merged": {"customer_tier": "gold"}
19650 }),
19651 )];
19652
19653 for update in [
19654 SearchAttributeUpdate::new()
19655 .keyword("customer_tier", "gold")
19656 .expect("keyword update"),
19657 SearchAttributeUpdate::new()
19658 .string("customer_tier", "gold")
19659 .expect("string update"),
19660 ] {
19661 let restarted = workflow_context(history.clone());
19662 restarted
19663 .upsert_search_attributes(update)
19664 .expect("legacy history constrains values but has unknown type identity");
19665 restarted
19666 .ensure_history_consumed()
19667 .expect("history consumed");
19668 }
19669 }
19670
19671 #[test]
19672 fn search_attribute_command_emits_canonical_types() {
19673 let ctx = workflow_context(Vec::new());
19674 ctx.upsert_search_attributes(
19675 SearchAttributeUpdate::new()
19676 .keyword("customer_tier", "gold")
19677 .expect("keyword update")
19678 .int("attempts", 3)
19679 .expect("int update")
19680 .delete("obsolete")
19681 .expect("delete update"),
19682 )
19683 .expect("valid search attributes");
19684
19685 assert_eq!(
19686 ctx.take_commands().expect("commands"),
19687 vec![json!({
19688 "type": "upsert_search_attributes",
19689 "attributes": {
19690 "attempts": 3,
19691 "customer_tier": "gold",
19692 "obsolete": null
19693 },
19694 "attribute_types": {
19695 "attempts": "int",
19696 "customer_tier": "keyword"
19697 }
19698 })]
19699 );
19700 }
19701
19702 #[test]
19703 fn duplicate_side_effects_and_conflicting_version_markers_are_rejected() {
19704 let duplicate_side_effect = WorkflowState::new(
19705 vec![
19706 history_event(
19707 "SideEffectRecorded",
19708 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
19709 ),
19710 history_event(
19711 "SideEffectRecorded",
19712 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
19713 ),
19714 ],
19715 "rust-workers".to_string(),
19716 DEFAULT_CODEC.to_string(),
19717 None,
19718 )
19719 .expect_err("duplicate side effect");
19720 assert!(matches!(
19721 duplicate_side_effect,
19722 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19723 if reason == "duplicate_side_effect_record"
19724 ));
19725
19726 let marker = |sequence, version| {
19727 history_event(
19728 "VersionMarkerRecorded",
19729 json!({
19730 "sequence": sequence,
19731 "change_id": "same-change",
19732 "version": version,
19733 "min_supported": -1,
19734 "max_supported": 1,
19735 }),
19736 )
19737 };
19738 let duplicate_marker = WorkflowState::new(
19739 vec![marker(1, 1), marker(3, -1)],
19740 "rust-workers".to_string(),
19741 DEFAULT_CODEC.to_string(),
19742 None,
19743 )
19744 .expect_err("conflicting marker");
19745 assert!(matches!(
19746 duplicate_marker,
19747 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19748 if reason == "version_marker_conflict"
19749 ));
19750 }
19751
19752 #[test]
19753 fn retained_patch_aliases_reject_duplicate_physical_records_and_invalid_ranges() {
19754 let marker = retained_patch_marker(1, 1, "2026-10-10T21:47:29.120756Z");
19755 let mut invalid_alias = retained_patch_marker(2, 1, "2026-10-10T21:47:29.123604Z");
19756 invalid_alias.payload["max_supported"] = json!(0);
19757 for (history, expected_reason) in [
19758 (
19759 vec![marker.clone(), marker.clone()],
19760 "duplicate_version_marker_record",
19761 ),
19762 (
19763 vec![marker, invalid_alias],
19764 "version_marker_history_range_invalid",
19765 ),
19766 ] {
19767 let error =
19768 WorkflowState::new(history, "rust-workers".into(), DEFAULT_CODEC.into(), None)
19769 .expect_err("invalid historical alias");
19770 assert!(matches!(error,
19771 Error::NonDeterministicReplay(ReplayFailure { reason, .. })
19772 if reason == expected_reason));
19773 }
19774 }
19775
19776 #[test]
19777 fn retained_patch_aliases_require_the_original_call_boundary() {
19778 let ctx = workflow_context(vec![
19779 retained_patch_marker(1, 1, "2026-10-10T21:47:29.120756Z"),
19780 retained_patch_marker(2, 1, "2026-10-10T21:47:29.123604Z"),
19781 ]);
19782 assert!(ctx.patched("retained-patch").unwrap());
19783 assert!(ctx.ensure_history_consumed().is_err());
19784 assert_eq!(ctx.state.lock().unwrap().command_cursor, 1);
19785 ctx.deprecate_patch("retained-patch").unwrap();
19786 ctx.ensure_history_consumed().unwrap();
19787 assert!(ctx.take_commands().unwrap().is_empty());
19788 }
19789
19790 #[test]
19791 fn retained_patch_aliases_do_not_replace_an_earlier_legacy_decision() {
19792 let ctx = workflow_context(vec![
19793 history_event(
19794 "SideEffectRecorded",
19795 json!({
19796 "sequence": 1, "result": fixture_envelope(json!("old-operation")),
19797 }),
19798 ),
19799 retained_patch_marker(2, 1, "2026-10-10T21:47:29.123604Z"),
19800 ]);
19801 assert!(!ctx.patched("retained-patch").unwrap());
19802 assert_eq!(
19803 ctx.side_effect(|| "must not execute".to_string()).unwrap(),
19804 "old-operation"
19805 );
19806 let error = ctx.patched("retained-patch").unwrap_err();
19807 assert!(matches!(error,
19808 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19809 if reason == "version_marker_conflict"));
19810 assert_eq!(ctx.state.lock().unwrap().command_cursor, 1);
19811 assert!(ctx.take_commands().unwrap().is_empty());
19812 }
19813
19814 #[test]
19815 fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
19816 let mut state = WorkflowState::new(
19817 Vec::new(),
19818 "rust-workers".to_string(),
19819 DEFAULT_CODEC.to_string(),
19820 None,
19821 )
19822 .expect("workflow state");
19823 state.workflow_command_identity = "command-7".to_string();
19824 let context = WorkflowContext {
19825 state: Arc::new(Mutex::new(state)),
19826 cancellation_scope_id: "root".into(),
19827 };
19828 let item =
19829 WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
19830
19831 context
19832 .append_workflow_stream("output", &[item], Some(10))
19833 .expect("append command");
19834 context
19835 .error_workflow_stream("output", "producer failed", None)
19836 .expect("error command");
19837 let commands = context.take_commands().expect("commands");
19838
19839 assert_eq!(commands[0]["type"], "record_side_effect");
19840 assert_eq!(
19841 commands[0]["workflow_stream"]["command_identity"],
19842 "command-7"
19843 );
19844 assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
19845 assert_eq!(
19846 commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
19847 "dw-stream:command-7:0:0"
19848 );
19849 assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
19850
19851 let recorded = history_event(
19852 "SideEffectRecorded",
19853 json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
19854 );
19855 let mut replay_state = WorkflowState::new(
19856 vec![recorded],
19857 "rust-workers".to_string(),
19858 DEFAULT_CODEC.to_string(),
19859 None,
19860 )
19861 .expect("replay state");
19862 replay_state.workflow_command_identity = "command-7".to_string();
19863 let replay_context = WorkflowContext {
19864 state: Arc::new(Mutex::new(replay_state)),
19865 cancellation_scope_id: "root".into(),
19866 };
19867 replay_context
19868 .append_workflow_stream(
19869 "output",
19870 &[WorkflowStreamAppendItem::from_reference(
19871 "s3://bucket/item.avro",
19872 )],
19873 Some(10),
19874 )
19875 .expect("replayed append");
19876 assert!(replay_context
19877 .take_commands()
19878 .expect("replayed commands")
19879 .is_empty());
19880 }
19881
19882 #[test]
19883 fn workflow_stream_authoring_requires_server_durable_command_identity() {
19884 let context = workflow_context(Vec::new());
19885 let error = context
19886 .append_workflow_stream(
19887 "output",
19888 &[WorkflowStreamAppendItem::from_reference(
19889 "s3://bucket/item.avro",
19890 )],
19891 None,
19892 )
19893 .expect_err("stream append without durable command identity must fail closed");
19894
19895 assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
19896 assert!(context.take_commands().expect("commands").is_empty());
19897 }
19898
19899 #[test]
19900 fn worker_stream_authoring_uses_task_identity_and_replays_without_output() {
19901 let client = Client::builder("http://localhost:8080").build().unwrap();
19902 let mut worker = Worker::new(client, "rust-workers");
19903 worker.register_workflow("streams.worker", |ctx, _| async move {
19904 ctx.append_workflow_stream(
19905 "output",
19906 &[WorkflowStreamAppendItem::new(json!("hello"))?],
19907 None,
19908 )?;
19909 ctx.close_workflow_stream("output", None)?;
19910 Ok(json!("done"))
19911 });
19912 for command_id in [None, Some(String::new()), Some("command-42".to_string())] {
19913 let mut task = workflow_task("streams.worker", Vec::new(), DEFAULT_CODEC);
19914 task.workflow_command_id = command_id.clone();
19915 let expected_identity = command_id
19916 .as_deref()
19917 .filter(|id| !id.is_empty())
19918 .unwrap_or(&task.task_id)
19919 .to_string();
19920 let commands = worker
19921 .execute_workflow_task(task)
19922 .expect("ordinary worker stream output");
19923 assert_eq!(commands.len(), 3);
19924 assert_eq!(
19925 commands[0]["workflow_stream"]["command_identity"],
19926 expected_identity
19927 );
19928 assert_eq!(
19929 commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
19930 format!("dw-stream:{expected_identity}:0:0")
19931 );
19932 assert_eq!(
19933 commands[1]["workflow_stream"]["command_identity"],
19934 expected_identity
19935 );
19936 assert_eq!(commands[1]["workflow_stream"]["operation"], "close");
19937 let history = commands[..2]
19938 .iter()
19939 .enumerate()
19940 .map(|(index, command)| {
19941 history_event(
19942 "SideEffectRecorded",
19943 json!({"sequence":index + 1,"result":command["result"]}),
19944 )
19945 })
19946 .collect();
19947 let mut replay = workflow_task("streams.worker", history, DEFAULT_CODEC);
19948 replay.task_id = "replacement-task".to_string();
19949 let replayed = worker
19950 .execute_workflow_task(replay)
19951 .expect("replacement worker consumes recorded stream effects");
19952 assert_eq!(replayed.len(), 1);
19953 assert_eq!(replayed[0]["type"], "complete_workflow");
19954 }
19955 let mut missing = workflow_task("streams.worker", Vec::new(), DEFAULT_CODEC);
19956 missing.task_id.clear();
19957 assert!(matches!(
19958 worker.execute_workflow_task(missing),
19959 Err(Error::MissingWorkflowCommandIdentity)
19960 ));
19961 }
19962
19963 #[test]
19964 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
19965 fn worker(calls: Arc<AtomicUsize>) -> Worker {
19966 let client = Client::new("http://127.0.0.1:8080").expect("client");
19967 let mut worker = Worker::new(client, "rust-workers");
19968 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
19969 let calls = Arc::clone(&calls);
19970 async move {
19971 let captured = ctx.side_effect(|| {
19972 calls.fetch_add(1, Ordering::SeqCst);
19973 "captured-once".to_string()
19974 })?;
19975 let version = ctx.get_version("cold-restart", 1, 2)?;
19976 Ok(json!({"captured": captured, "version": version}))
19977 }
19978 });
19979 worker
19980 }
19981
19982 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
19983 WorkflowTask {
19984 task_id: "wft-side-effect-version".to_string(),
19985 workflow_command_id: None,
19986 workflow_id: Some("wf-side-effect-version".to_string()),
19987 run_id: Some("run-side-effect-version".to_string()),
19988 workflow_type: "rust.side-effect-version".to_string(),
19989 cancel_requested: false,
19990 payload_codec: DEFAULT_CODEC.to_string(),
19991 arguments: Some(
19992 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
19993 ),
19994 history_events,
19995 total_history_events: None,
19996 history_size_bytes: None,
19997 continue_as_new_recommended: None,
19998 history_budget_pressure: None,
19999 next_history_page_token: None,
20000 workflow_task_attempt: 1,
20001 workflow_signal_id: None,
20002 signal_name: None,
20003 signal_arguments: None,
20004 workflow_update_id: None,
20005 update_name: None,
20006 lease_owner: Some("rust-worker".to_string()),
20007 }
20008 }
20009
20010 let calls = Arc::new(AtomicUsize::new(0));
20011 let initial = worker(Arc::clone(&calls))
20012 .execute_workflow_task(task(Vec::new()))
20013 .expect("initial execution");
20014 assert_eq!(
20015 initial
20016 .iter()
20017 .map(|command| &command["type"])
20018 .collect::<Vec<_>>(),
20019 vec![
20020 "record_side_effect",
20021 "record_version_marker",
20022 "complete_workflow"
20023 ]
20024 );
20025 assert_eq!(calls.load(Ordering::SeqCst), 1);
20026
20027 let restarted = worker(Arc::clone(&calls));
20028 let replayed = restarted
20029 .execute_workflow_task(task(vec![
20030 history_event(
20031 "SideEffectRecorded",
20032 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
20033 ),
20034 history_event(
20035 "VersionMarkerRecorded",
20036 json!({
20037 "sequence": 2,
20038 "change_id": "cold-restart",
20039 "version": 2,
20040 "min_supported": 1,
20041 "max_supported": 2,
20042 }),
20043 ),
20044 ]))
20045 .expect("cold replay");
20046 assert_eq!(replayed.len(), 1);
20047 assert_eq!(replayed[0]["type"], "complete_workflow");
20048 assert_eq!(calls.load(Ordering::SeqCst), 1);
20049 }
20050
20051 #[test]
20052 fn side_effect_replay_rejects_changed_rust_value_type() {
20053 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
20054 let ctx = workflow_context(vec![history_event(
20055 "SideEffectRecorded",
20056 json!({"sequence": 1, "result": result}),
20057 )]);
20058 let error = ctx
20059 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
20060 .expect_err("changed type must fail replay");
20061 assert!(matches!(
20062 error,
20063 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20064 if reason == "side_effect_type_mismatch"
20065 ));
20066 }
20067
20068 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
20069 vec![
20070 history_event(
20071 "ActivityScheduled",
20072 json!({
20073 "sequence": 1,
20074 "activity_type": "flaky",
20075 "activity_execution_id": "act-1",
20076 "activity": {
20077 "id": "act-1",
20078 "sequence": 1,
20079 "type": "flaky",
20080 "queue": "critical-activities",
20081 "execution_mode": null,
20082 "retry_policy": {
20083 "snapshot_version": 1,
20084 "max_attempts": 3,
20085 "backoff_seconds": [2, 4],
20086 "start_to_close_timeout": 30,
20087 "schedule_to_start_timeout": 5,
20088 "schedule_to_close_timeout": 90,
20089 "heartbeat_timeout": 10,
20090 "non_retryable_error_types": ["PermanentError"]
20091 }
20092 }
20093 }),
20094 ),
20095 history_event(
20096 "ActivityStarted",
20097 json!({
20098 "sequence": 1,
20099 "activity_type": "flaky",
20100 "activity_execution_id": "act-1",
20101 "activity_attempt_id": "attempt-1",
20102 "attempt_number": 1
20103 }),
20104 ),
20105 history_event(
20106 "ActivityRetryScheduled",
20107 json!({
20108 "sequence": 1,
20109 "activity_type": "flaky",
20110 "activity_execution_id": "act-1",
20111 "activity_attempt_id": "attempt-1",
20112 "attempt_number": 1,
20113 "retry_after_attempt": 1,
20114 "retry_backoff_seconds": 2,
20115 "failure_category": "activity",
20116 "exception_type": "TransientError"
20117 }),
20118 ),
20119 history_event(
20120 "ActivityStarted",
20121 json!({
20122 "sequence": 1,
20123 "activity_type": "flaky",
20124 "activity_execution_id": "act-1",
20125 "activity_attempt_id": "attempt-2",
20126 "attempt_number": 2
20127 }),
20128 ),
20129 history_event(
20130 "ActivityCompleted",
20131 json!({
20132 "sequence": 1,
20133 "activity_type": "flaky",
20134 "activity_execution_id": "act-1",
20135 "activity_attempt_id": "attempt-2",
20136 "attempt_number": 2,
20137 "payload_codec": DEFAULT_CODEC,
20138 "result": fixture_envelope(json!({"status":"recovered"}))
20139 }),
20140 ),
20141 ]
20142 }
20143
20144 fn retry_activity_options() -> ActivityOptions {
20145 ActivityOptions::new()
20146 .task_queue("critical-activities")
20147 .retry_policy(
20148 ActivityRetryPolicy::new(3)
20149 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
20150 .non_retryable_error_type("PermanentError"),
20151 )
20152 .start_to_close_timeout(Duration::from_secs(30))
20153 .schedule_to_start_timeout(Duration::from_secs(5))
20154 .schedule_to_close_timeout(Duration::from_secs(90))
20155 .heartbeat_timeout(Duration::from_secs(10))
20156 }
20157
20158 #[test]
20159 fn fixed_avro_value_round_trips_json_values() {
20160 let value = json!({"greeting": "hello", "count": 3, "ok": true});
20161 let envelope = PayloadEnvelope::avro(&value).expect("encode");
20162 assert_eq!(envelope.codec, DEFAULT_CODEC);
20163 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
20164 }
20165
20166 #[tokio::test]
20167 async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
20168 let client = Client::new("http://127.0.0.1:8080").expect("client");
20169 let mut worker = Worker::new(client, "rust-workers");
20170 worker.register_typed_workflow(
20171 "typed.contract.workflow",
20172 |_ctx, input: TypedContract| async move { Ok(input) },
20173 );
20174 worker.register_typed_activity(
20175 "typed.contract.activity",
20176 |_ctx, input: TypedContract| async move { Ok(input) },
20177 );
20178
20179 let expected = typed_contract();
20180 let arguments = AvroValue::Array(vec![
20181 AvroValue::from_serialize(&expected).expect("typed request")
20182 ]);
20183 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
20184 let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
20185 workflow.arguments = Some(envelope.clone());
20186 let commands = worker
20187 .execute_workflow_task(workflow)
20188 .expect("typed workflow task");
20189 let workflow_result: TypedContract =
20190 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
20191 .expect("workflow result envelope")
20192 .deserialize()
20193 .expect("workflow result type");
20194 assert_eq!(workflow_result, expected);
20195
20196 let activity = ActivityTask {
20197 task_id: "typed-contract-activity".to_string(),
20198 activity_attempt_id: Some("typed-contract-attempt".to_string()),
20199 attempt_id: None,
20200 activity_type: "typed.contract.activity".to_string(),
20201 payload_codec: DEFAULT_CODEC.to_string(),
20202 arguments: Some(envelope),
20203 attempt_number: 1,
20204 lease_owner: Some("rust-worker".to_string()),
20205 };
20206 let activity_result: TypedContract = worker
20207 .execute_activity_task(activity)
20208 .await
20209 .expect("typed activity task")
20210 .deserialize()
20211 .expect("activity result type");
20212 assert_eq!(activity_result, expected);
20213 }
20214
20215 #[tokio::test]
20216 async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
20217 let client = Client::new("http://127.0.0.1:8080").expect("client");
20218 let mut worker = Worker::new(client, "rust-workers");
20219 worker.register_typed_workflow(
20220 "typed.shape.workflow",
20221 |_ctx, input: TypedContract| async move { Ok(input) },
20222 );
20223 worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
20224 Ok(f64::NAN)
20225 });
20226
20227 let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
20228 workflow.arguments = Some(
20229 encode_typed_envelope(
20230 &AvroValue::Array(vec![
20231 AvroValue::String("first".to_string()),
20232 AvroValue::String("second".to_string()),
20233 ]),
20234 DEFAULT_CODEC,
20235 )
20236 .expect("malformed typed arguments"),
20237 );
20238 let commands = worker
20239 .execute_workflow_task(workflow)
20240 .expect("shape mismatch becomes a workflow failure");
20241 let message = commands[0]["message"].as_str().expect("failure message");
20242 assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
20243 assert!(message.contains(type_name::<TypedContract>()));
20244 assert!(message.contains("task carried 2 arguments"));
20245
20246 let activity = ActivityTask {
20247 task_id: "typed-unsupported-activity".to_string(),
20248 activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
20249 attempt_id: None,
20250 activity_type: "typed.unsupported.activity".to_string(),
20251 payload_codec: DEFAULT_CODEC.to_string(),
20252 arguments: Some(
20253 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
20254 .expect("unit arguments"),
20255 ),
20256 attempt_number: 1,
20257 lease_owner: Some("rust-worker".to_string()),
20258 };
20259 let Error::HandlerType {
20260 handler_kind,
20261 handler_name,
20262 value_kind,
20263 rust_type,
20264 message,
20265 } = worker
20266 .execute_activity_task(activity)
20267 .await
20268 .expect_err("non-finite handler output must fail")
20269 else {
20270 panic!("expected contextual handler type failure");
20271 };
20272 assert_eq!(handler_kind, HandlerKind::Activity);
20273 assert_eq!(handler_name, "typed.unsupported.activity");
20274 assert_eq!(value_kind, HandlerValueKind::Result);
20275 assert_eq!(rust_type, type_name::<f64>());
20276 assert!(message.contains("non_finite_float"));
20277 }
20278
20279 #[tokio::test]
20280 async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
20281 #[derive(Clone, Default)]
20282 struct State {
20283 observed: Option<TypedContract>,
20284 }
20285
20286 let client = Client::new("http://127.0.0.1:8080").expect("client");
20287 let mut worker = Worker::new(client, "rust-workers");
20288 worker.register_typed_replayed_workflow(
20289 "typed.contract.replayed",
20290 State::default,
20291 |ctx, input: TypedContract, state| async move {
20292 let result: TypedContract =
20293 ctx.activity_typed("typed.contract.activity", input).await?;
20294 state.update(|current| current.observed = Some(result.clone()))?;
20295 Ok(result)
20296 },
20297 );
20298 worker.register_replayed_query::<State, _, _>(
20299 "typed.contract.replayed",
20300 "observed",
20301 |_ctx, state, _args| async move {
20302 Ok(json!(state.observed.as_ref().map(|value| value.signed)))
20303 },
20304 );
20305
20306 let expected = typed_contract();
20307 let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
20308 let workflow_arguments =
20309 encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
20310 .expect("workflow arguments");
20311 let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
20312 let task = QueryTask {
20313 query_task_id: "typed-replay-query".to_string(),
20314 query_task_attempt: 1,
20315 lease_owner: Some("rust-worker".to_string()),
20316 workflow_id: Some("typed-replay".to_string()),
20317 run_id: Some("typed-replay-run".to_string()),
20318 workflow_type: "typed.contract.replayed".to_string(),
20319 query_name: "observed".to_string(),
20320 payload_codec: DEFAULT_CODEC.to_string(),
20321 workflow_arguments: Some(workflow_arguments),
20322 query_arguments: Some(
20323 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
20324 .expect("query arguments"),
20325 ),
20326 history_events: vec![
20327 history_event(
20328 "ActivityScheduled",
20329 json!({
20330 "sequence": 1,
20331 "activity_type": "typed.contract.activity"
20332 }),
20333 ),
20334 history_event(
20335 "ActivityCompleted",
20336 json!({
20337 "sequence": 1,
20338 "activity_type": "typed.contract.activity",
20339 "payload_codec": DEFAULT_CODEC,
20340 "result": result
20341 }),
20342 ),
20343 ],
20344 history_export: None,
20345 run_status: Some("completed".to_string()),
20346 };
20347
20348 assert_eq!(
20349 worker
20350 .execute_query_task(task)
20351 .await
20352 .expect("typed replay query")
20353 .deserialize::<i64>()
20354 .expect("query result"),
20355 expected.signed
20356 );
20357 }
20358
20359 #[tokio::test]
20360 async fn update_context_preserves_workflow_input_and_committed_signals() {
20361 let client = Client::new("http://127.0.0.1:8080").expect("client");
20362 let mut worker = Worker::new(client, "rust-workers");
20363 worker.register_workflow("snapshot", |_ctx, _input| async move { Ok(Value::Null) });
20364 let workflow_input = AvroValue::Array(vec![typed_fidelity_probe()]);
20365 let signal_arguments = vec![AvroValue::Bytes(vec![0, 255]), AvroValue::Long(7)];
20366 let expected_input = workflow_input.clone();
20367 let expected_signals = signal_arguments.clone();
20368 let handler = move |context: QueryContext, arguments: AvroValue| {
20369 let expected_input = expected_input.clone();
20370 let expected_signals = expected_signals.clone();
20371 async move {
20372 assert_eq!(context.workflow_id.as_deref(), Some("wf-snapshot"));
20373 assert_eq!(context.run_id.as_deref(), Some("run-snapshot"));
20374 assert_eq!(context.workflow_input_avro_value(), &expected_input);
20375 assert_eq!(context.workflow_input(), &expected_input.into_json()?);
20376 assert_eq!(
20377 context.signals_avro_value("changed"),
20378 vec![expected_signals.clone()]
20379 );
20380 assert_eq!(
20381 context.signals("changed"),
20382 vec![expected_signals
20383 .into_iter()
20384 .map(AvroValue::into_json)
20385 .collect::<Result<Vec<_>>>()?]
20386 );
20387 assert_eq!(
20388 context.signal_events()[0].id.as_deref(),
20389 Some("signal-original")
20390 );
20391 assert_eq!(context.signal_events()[0].workflow_sequence, Some(1));
20392 Ok(arguments)
20393 }
20394 };
20395 worker.register_query_avro_value("snapshot", "inspect", handler.clone());
20396 worker.register_update_avro_value("snapshot", "inspect", handler);
20397 let update_arguments = AvroValue::Array(vec![AvroValue::String("update-only".into())]);
20398 let update_envelope =
20399 encode_typed_envelope(&update_arguments, DEFAULT_CODEC).expect("update input");
20400 let mut task = workflow_task(
20401 "snapshot",
20402 vec![
20403 history_event(
20404 "SignalReceived",
20405 json!({
20406 "signal_id": "signal-original", "signal_name": "changed", "workflow_sequence": 1,
20407 "arguments": encode_typed_envelope(&AvroValue::Array(signal_arguments), DEFAULT_CODEC).expect("signal input"),
20408 }),
20409 ),
20410 history_event(
20411 "UpdateAccepted",
20412 json!({
20413 "update_id": "update-original", "update_name": "inspect", "arguments": update_envelope.clone(),
20414 }),
20415 ),
20416 ],
20417 DEFAULT_CODEC,
20418 );
20419 task.arguments =
20420 Some(encode_typed_envelope(&workflow_input, DEFAULT_CODEC).expect("workflow input"));
20421 task.workflow_update_id = Some("update-original".into());
20422 task.update_name = Some("inspect".into());
20423 let query = QueryTask {
20424 query_task_id: "query-snapshot".into(),
20425 query_task_attempt: 1,
20426 lease_owner: None,
20427 workflow_id: task.workflow_id.clone(),
20428 run_id: task.run_id.clone(),
20429 workflow_type: task.workflow_type.clone(),
20430 query_name: "inspect".into(),
20431 payload_codec: DEFAULT_CODEC.into(),
20432 workflow_arguments: task.arguments.clone(),
20433 query_arguments: Some(update_envelope),
20434 history_events: task.history_events.clone(),
20435 history_export: None,
20436 run_status: Some("running".into()),
20437 };
20438 assert_eq!(
20439 worker
20440 .execute_query_task(query)
20441 .await
20442 .expect("query snapshot"),
20443 update_arguments
20444 );
20445 for _ in 0..2 {
20446 let commands = worker
20447 .execute_workflow_task(task.clone())
20448 .expect("cold update snapshot");
20449 assert_eq!(commands[0]["type"], "complete_update");
20450 assert_eq!(commands[0]["update_id"], "update-original");
20451 assert_eq!(
20452 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
20453 .expect("update result"),
20454 update_arguments
20455 );
20456 }
20457 }
20458
20459 #[test]
20460 fn committed_signal_snapshot_counts_identities_once() {
20461 let arguments = encode_value_envelope(&json!([7]), DEFAULT_CODEC).expect("signal input");
20462 let received = history_event(
20463 "SignalReceived",
20464 json!({
20465 "signal_id": "signal-first", "workflow_command_id": "command-first",
20466 "signal_name": "changed", "arguments": arguments.clone(),
20467 }),
20468 );
20469 let applied = history_event(
20470 "SignalApplied",
20471 json!({
20472 "signal_id": "signal-first", "workflow_command_id": "command-first",
20473 "signal_name": "changed", "value": encode_value_envelope(&json!(7), DEFAULT_CODEC).expect("applied value"), "workflow_sequence": 1,
20474 }),
20475 );
20476 let distinct = history_event(
20477 "SignalReceived",
20478 json!({
20479 "signal_id": "signal-second", "workflow_command_id": "command-second",
20480 "signal_name": "changed", "arguments": arguments,
20481 }),
20482 );
20483 let history = vec![received.clone(), applied.clone(), distinct];
20484 let signals =
20485 history_signal_events(&history, None, DEFAULT_CODEC).expect("signal snapshot");
20486 assert_eq!(signals.len(), 2);
20487 assert_eq!(signals[0].id.as_deref(), Some("signal-first"));
20488 assert_eq!(signals[1].id.as_deref(), Some("signal-second"));
20489 assert_eq!(signals[0].workflow_sequence, Some(1));
20490 assert_eq!(signals[0].arguments, signals[1].arguments);
20491 let mut command_only = received.clone();
20492 command_only
20493 .payload
20494 .as_object_mut()
20495 .expect("payload")
20496 .remove("signal_id");
20497 let signals = history_signal_events(&[command_only, applied.clone()], None, DEFAULT_CODEC)
20498 .expect("command identity");
20499 assert_eq!(signals.len(), 1);
20500 assert_eq!(signals[0].id.as_deref(), Some("signal-first"));
20501 let mut conflicting = applied;
20502 conflicting.payload["value"] =
20503 encode_value_envelope(&json!(8), DEFAULT_CODEC).expect("different input");
20504 assert!(history_signal_events(&[received, conflicting], None, DEFAULT_CODEC).is_err());
20505 }
20506
20507 #[test]
20508 fn committed_signal_snapshot_preserves_argument_vectors_after_application() {
20509 for arguments in [
20510 vec![],
20511 vec![AvroValue::Array(vec![
20512 AvroValue::Long(1),
20513 AvroValue::Long(2),
20514 ])],
20515 vec![AvroValue::Bytes(vec![0, 255])],
20516 vec![AvroValue::Null],
20517 vec![AvroValue::Boolean(false)],
20518 vec![
20519 AvroValue::Long(1),
20520 AvroValue::Array(vec![AvroValue::Long(2)]),
20521 ],
20522 ] {
20523 let applied = match arguments.as_slice() {
20524 [] => AvroValue::Boolean(true),
20525 [value] => value.clone(),
20526 values => AvroValue::Array(values.to_vec()),
20527 };
20528 for applied in [applied, AvroValue::Array(arguments.clone())] {
20529 let events = vec![
20530 history_event(
20531 "SignalReceived",
20532 json!({
20533 "signal_id": "original", "signal_name": "changed",
20534 "arguments": encode_typed_envelope(&AvroValue::Array(arguments.clone()), DEFAULT_CODEC).expect("signal arguments"),
20535 }),
20536 ),
20537 history_event(
20538 "SignalApplied",
20539 json!({
20540 "signal_id": "original", "signal_name": "changed",
20541 "value": encode_typed_envelope(&applied, DEFAULT_CODEC).expect("application value"),
20542 }),
20543 ),
20544 ];
20545 let signals = history_signal_events(&events, None, DEFAULT_CODEC)
20546 .expect("canonical signal snapshot");
20547 assert_eq!(signals.len(), 1);
20548 assert_eq!(signals[0].avro_arguments, arguments);
20549 }
20550 }
20551 }
20552
20553 #[tokio::test]
20554 async fn update_context_rejects_malformed_snapshot_before_handler() {
20555 let client = Client::new("http://127.0.0.1:8080").expect("client");
20556 let mut worker = Worker::new(client, "rust-workers");
20557 worker.register_workflow("snapshot", |_ctx, _input| async move { Ok(Value::Null) });
20558 let calls = Arc::new(AtomicUsize::new(0));
20559 let observed = Arc::clone(&calls);
20560 worker.register_update("snapshot", "inspect", move |_ctx, _arguments| {
20561 observed.fetch_add(1, Ordering::SeqCst);
20562 async { Ok(Value::Null) }
20563 });
20564 let mut task = workflow_task(
20565 "snapshot",
20566 vec![history_event(
20567 "UpdateAccepted",
20568 json!({
20569 "update_id": "update-original", "update_name": "inspect",
20570 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("update input"),
20571 }),
20572 )],
20573 DEFAULT_CODEC,
20574 );
20575 task.workflow_update_id = Some("update-original".into());
20576 task.update_name = Some("inspect".into());
20577 let mut bad_input = task.clone();
20578 bad_input.arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "invalid"}));
20579 assert!(worker.execute_workflow_task(bad_input).is_err());
20580 task.history_events.insert(0, history_event("SignalReceived", json!({
20581 "signal_name": "changed", "arguments": {"codec": DEFAULT_CODEC, "blob": "invalid"},
20582 })));
20583 assert!(worker.execute_workflow_task(task).is_err());
20584 assert_eq!(calls.load(Ordering::SeqCst), 0);
20585 }
20586
20587 #[tokio::test]
20588 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
20589 let client = Client::new("http://127.0.0.1:8080").expect("client");
20590 let mut worker = Worker::new(client, "rust-workers");
20591 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
20592 worker
20593 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
20594 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
20595 Ok(input)
20596 });
20597 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
20598 Ok(input)
20599 });
20600 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
20601 Ok(AvroValue::Array(
20602 ctx.wait_signal_avro_value("changed").await?,
20603 ))
20604 });
20605
20606 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
20607 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
20608
20609 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
20610 workflow.arguments = Some(envelope.clone());
20611 let commands = worker
20612 .execute_workflow_task(workflow)
20613 .expect("typed workflow task");
20614 assert_eq!(commands[0]["type"], "complete_workflow");
20615 assert_eq!(
20616 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
20617 .expect("typed workflow result"),
20618 arguments
20619 );
20620
20621 let activity = ActivityTask {
20622 task_id: "activity-typed".to_string(),
20623 activity_attempt_id: Some("attempt-typed".to_string()),
20624 attempt_id: None,
20625 activity_type: "typed.activity".to_string(),
20626 payload_codec: DEFAULT_CODEC.to_string(),
20627 arguments: Some(envelope.clone()),
20628 attempt_number: 1,
20629 lease_owner: Some("rust-worker".to_string()),
20630 };
20631 assert_eq!(
20632 worker
20633 .execute_activity_task(activity)
20634 .await
20635 .expect("typed activity result"),
20636 arguments
20637 );
20638
20639 let query = QueryTask {
20640 query_task_id: "query-typed".to_string(),
20641 query_task_attempt: 1,
20642 lease_owner: Some("rust-worker".to_string()),
20643 workflow_id: Some("typed-1".to_string()),
20644 run_id: Some("run-typed".to_string()),
20645 workflow_type: "typed.echo".to_string(),
20646 query_name: "inspect".to_string(),
20647 payload_codec: DEFAULT_CODEC.to_string(),
20648 workflow_arguments: Some(
20649 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
20650 .expect("workflow input"),
20651 ),
20652 query_arguments: Some(envelope.clone()),
20653 history_events: Vec::new(),
20654 history_export: None,
20655 run_status: Some("running".to_string()),
20656 };
20657 assert_eq!(
20658 worker
20659 .execute_query_task(query)
20660 .await
20661 .expect("typed query result"),
20662 arguments
20663 );
20664
20665 let mut update = workflow_task(
20666 "typed.echo",
20667 vec![history_event(
20668 "UpdateAccepted",
20669 json!({
20670 "update_id": "update-typed",
20671 "update_name": "replace",
20672 "arguments": envelope.clone(),
20673 }),
20674 )],
20675 DEFAULT_CODEC,
20676 );
20677 update.workflow_update_id = Some("update-typed".to_string());
20678 update.update_name = Some("replace".to_string());
20679 let commands = worker
20680 .execute_workflow_task(update)
20681 .expect("typed update task");
20682 assert_eq!(commands[0]["type"], "complete_update");
20683 assert_eq!(
20684 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
20685 .expect("typed update result"),
20686 arguments
20687 );
20688
20689 let mut signal = workflow_task(
20690 "typed.signal",
20691 vec![history_event(
20692 "SignalReceived",
20693 json!({
20694 "signal_id": "signal-typed",
20695 "signal_name": "changed",
20696 "arguments": envelope.clone(),
20697 }),
20698 )],
20699 DEFAULT_CODEC,
20700 );
20701 signal.workflow_signal_id = Some("signal-typed".to_string());
20702 signal.signal_name = Some("changed".to_string());
20703 signal.signal_arguments = Some(envelope);
20704 let commands = worker
20705 .execute_workflow_task(signal)
20706 .expect("typed signal resume");
20707 assert_eq!(
20708 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
20709 .expect("typed signal result"),
20710 arguments
20711 );
20712 }
20713
20714 #[tokio::test]
20715 async fn typed_helpers_never_parse_json_inspection_projection() {
20716 let collision_values = projection_collision_probe();
20717 let expected = AvroValue::Array(collision_values.clone());
20718 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
20719
20720 let activity_context = workflow_context_with_codec(
20721 vec![history_event(
20722 "ActivityCompleted",
20723 json!({
20724 "sequence": 1,
20725 "activity_type": "collision.activity",
20726 "payload_codec": DEFAULT_CODEC,
20727 "result": envelope.clone(),
20728 }),
20729 )],
20730 DEFAULT_CODEC,
20731 );
20732 assert_eq!(
20733 activity_context
20734 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
20735 .await
20736 .expect("typed activity collision result"),
20737 expected
20738 );
20739
20740 let signal_context = workflow_context_with_codec(
20741 vec![
20742 history_event(
20743 "SignalWaitOpened",
20744 json!({"sequence": 1, "signal_name": "collision"}),
20745 ),
20746 history_event(
20747 "SignalApplied",
20748 json!({
20749 "sequence": 1,
20750 "signal_name": "collision",
20751 "payload_codec": DEFAULT_CODEC,
20752 "value": envelope.clone(),
20753 }),
20754 ),
20755 ],
20756 DEFAULT_CODEC,
20757 );
20758 assert_eq!(
20759 signal_context
20760 .wait_signal_avro_value("collision")
20761 .await
20762 .expect("typed signal collision arguments"),
20763 collision_values
20764 );
20765
20766 let child_context = workflow_context_with_codec(
20767 vec![
20768 history_event(
20769 "ChildWorkflowScheduled",
20770 json!({
20771 "sequence": 1,
20772 "child_workflow_instance_id": "collision-child",
20773 "child_workflow_run_id": "collision-run",
20774 "child_workflow_type": "collision.child",
20775 }),
20776 ),
20777 history_event(
20778 "ChildRunCompleted",
20779 json!({
20780 "sequence": 1,
20781 "child_workflow_instance_id": "collision-child",
20782 "child_workflow_run_id": "collision-run",
20783 "child_workflow_type": "collision.child",
20784 "payload_codec": DEFAULT_CODEC,
20785 "result": envelope,
20786 }),
20787 ),
20788 ],
20789 DEFAULT_CODEC,
20790 );
20791 let child = child_context
20792 .start_child_workflow_avro_value(
20793 "collision.child",
20794 ChildWorkflowOptions::new("collision-workers"),
20795 AvroValue::Array(Vec::new()),
20796 )
20797 .await
20798 .expect("typed child collision result");
20799 assert_eq!(child.result, expected);
20800 }
20801
20802 #[tokio::test]
20803 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
20804 let client = Client::new("http://127.0.0.1:8080").expect("client");
20805 let mut worker = Worker::new(client, "rust-workers");
20806 worker.register_replayed_workflow_avro_value(
20807 "typed.replayed",
20808 || (),
20809 |_ctx, input, _state| async move { Ok(input) },
20810 );
20811 worker.register_replayed_query_avro_value::<(), _, _>(
20812 "typed.replayed",
20813 "inspect",
20814 |ctx, _state, args| async move {
20815 let mut signals = ctx.signals_avro_value("collision");
20816 let signal = signals
20817 .pop()
20818 .map(AvroValue::Array)
20819 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
20820 Ok(AvroValue::Array(vec![
20821 ctx.workflow_input_avro_value().clone(),
20822 signal,
20823 args,
20824 ]))
20825 },
20826 );
20827 let arguments = AvroValue::Array(projection_collision_probe());
20828 let signal_arguments =
20829 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
20830 let task = QueryTask {
20831 query_task_id: "query-typed-replay".to_string(),
20832 query_task_attempt: 1,
20833 lease_owner: Some("rust-worker".to_string()),
20834 workflow_id: Some("typed-replay".to_string()),
20835 run_id: Some("run-typed-replay".to_string()),
20836 workflow_type: "typed.replayed".to_string(),
20837 query_name: "inspect".to_string(),
20838 payload_codec: DEFAULT_CODEC.to_string(),
20839 workflow_arguments: Some(
20840 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
20841 ),
20842 query_arguments: Some(
20843 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
20844 ),
20845 history_events: vec![history_event(
20846 "SignalReceived",
20847 json!({
20848 "signal_id": "collision-signal",
20849 "signal_name": "collision",
20850 "workflow_sequence": 1,
20851 "payload_codec": DEFAULT_CODEC,
20852 "arguments": signal_arguments,
20853 }),
20854 )],
20855 history_export: None,
20856 run_status: Some("completed".to_string()),
20857 };
20858
20859 assert_eq!(
20860 worker
20861 .execute_query_task(task)
20862 .await
20863 .expect("typed replay query"),
20864 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
20865 );
20866 }
20867
20868 #[test]
20869 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
20870 let value = BTreeMap::from([(1_i32, "integer key")]);
20871 let error = PayloadEnvelope::avro(&value)
20872 .expect_err("integer map keys must fail")
20873 .to_string();
20874
20875 assert!(error.contains("invalid_map_key"));
20876 }
20877
20878 #[test]
20879 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
20880 let envelope = PayloadEnvelope {
20881 codec: "json".to_string(),
20882 blob: r#"{"greeting":"hello"}"#.to_string(),
20883 };
20884
20885 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
20886 let diagnostic = error.to_string();
20887 assert!(diagnostic.contains("unsupported_payload_codec"));
20888 assert!(diagnostic.contains("codec=\"avro\""));
20889 assert!(diagnostic.contains("HTTP document transport"));
20890 }
20891
20892 #[test]
20893 fn untagged_json_payload_value_fails_closed() {
20894 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
20895 .expect_err("untagged JSON payload values must fail");
20896 let diagnostic = error.to_string();
20897 assert!(diagnostic.contains("unsupported_payload_codec"));
20898 assert!(diagnostic.contains("untagged durable payload"));
20899 assert!(diagnostic.contains("HTTP document transport"));
20900 }
20901
20902 #[test]
20903 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
20904 let envelope = PayloadEnvelope {
20905 codec: DEFAULT_CODEC.to_string(),
20906 blob: BASE64.encode([0x01]),
20907 };
20908
20909 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
20910 assert!(error.to_string().contains("invalid_payload_framing"));
20911 }
20912
20913 #[tokio::test]
20914 async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
20915 let server = MockWorkerServer::start();
20916 let client = Client::builder(server.base_url())
20917 .timeout(Duration::from_secs(2))
20918 .build()
20919 .expect("client");
20920 let invalid_commands = [
20921 json!({
20922 "type": "complete_workflow",
20923 "result": {"codec": "json", "blob": null}
20924 }),
20925 json!({
20926 "type": "schedule_activity",
20927 "arguments": {"codec": "yaml", "blob": "ignored"}
20928 }),
20929 json!({
20930 "type": "start_child_workflow",
20931 "arguments": {"codec": DEFAULT_CODEC, "blob": null}
20932 }),
20933 json!({"type": "continue_as_new", "arguments": []}),
20934 json!({"type": "complete_update"}),
20935 json!({"type": "record_side_effect", "result": null}),
20936 json!({
20937 "type": "start_service_operation",
20938 "payload_codec": DEFAULT_CODEC,
20939 "request_payload": "raw-avro-bytes"
20940 }),
20941 ];
20942
20943 for command in invalid_commands {
20944 let error = client
20945 .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
20946 .await
20947 .expect_err("invalid durable payload must fail locally");
20948 let diagnostic = error.to_string();
20949 assert!(
20950 diagnostic.contains("unsupported_payload_codec")
20951 || diagnostic.contains("invalid_payload_envelope")
20952 || diagnostic.contains("untagged durable payload"),
20953 "unexpected validation diagnostic: {diagnostic}"
20954 );
20955 }
20956
20957 assert_eq!(
20958 server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
20959 0,
20960 "invalid command payloads must not reach HTTP transport"
20961 );
20962 }
20963
20964 #[test]
20965 fn workflow_completion_validates_only_protocol_owned_payload_slots() {
20966 let envelope = fixture_envelope(json!({"codec": "customer-value"}));
20967 let commands = [
20968 json!({"type": "complete_workflow", "result": envelope.clone()}),
20969 json!({"type": "schedule_activity", "arguments": envelope.clone()}),
20970 json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
20971 json!({"type": "continue_as_new", "arguments": envelope.clone()}),
20972 json!({"type": "complete_update", "result": envelope.clone()}),
20973 json!({"type": "record_side_effect", "result": envelope.clone()}),
20974 json!({
20975 "type": "start_service_operation",
20976 "payload_codec": DEFAULT_CODEC,
20977 "request_payload": envelope.clone()
20978 }),
20979 json!({
20980 "type": "complete_workflow",
20981 "result": envelope,
20982 "metadata": {
20983 "codec": "json",
20984 "payload_codec": "customer-codec",
20985 "result": {"codec": "yaml", "blob": null}
20986 }
20987 }),
20988 ];
20989
20990 validate_workflow_task_commands(&commands)
20991 .expect("customer metadata must not become a protocol codec declaration");
20992 }
20993
20994 #[test]
20995 fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
20996 assert_eq!(
20997 decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
20998 AvroValue::Array(Vec::new())
20999 );
21000 assert_eq!(
21001 decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
21002 AvroValue::Array(Vec::new())
21003 );
21004
21005 let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
21006 signal.signal_name = Some("empty-signal".to_string());
21007 signal.signal_arguments = None;
21008 let decoded = decode_resume_signal(&signal)
21009 .expect("valid Avro signal")
21010 .expect("named signal resumes the workflow");
21011 assert!(decoded.arguments.is_empty());
21012 }
21013
21014 #[tokio::test]
21015 async fn malformed_task_level_codecs_become_pre_handler_failures() {
21016 let client = Client::new("http://127.0.0.1:8080").expect("client");
21017 let mut worker = Worker::new(client, "rust-workers");
21018 let handler_calls = Arc::new(AtomicUsize::new(0));
21019
21020 let calls = Arc::clone(&handler_calls);
21021 worker.register_workflow("codec.workflow", move |_ctx, _args| {
21022 calls.fetch_add(1, Ordering::SeqCst);
21023 async move { Ok(Value::Null) }
21024 });
21025 let calls = Arc::clone(&handler_calls);
21026 worker.register_activity("codec.activity", move |_ctx, _args| {
21027 calls.fetch_add(1, Ordering::SeqCst);
21028 async move { Ok(Value::Null) }
21029 });
21030 let calls = Arc::clone(&handler_calls);
21031 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
21032 calls.fetch_add(1, Ordering::SeqCst);
21033 async move { Ok(Value::Null) }
21034 });
21035
21036 let mut failures = Vec::new();
21037 for codec_case in [
21038 InvalidTaskPayloadCodec::Missing,
21039 InvalidTaskPayloadCodec::Null,
21040 InvalidTaskPayloadCodec::NonString,
21041 ] {
21042 let mut workflow = json!({
21043 "task_id": format!("workflow-{}", codec_case.label()),
21044 "workflow_type": "codec.workflow"
21045 });
21046 codec_case.apply(&mut workflow);
21047 match serde_json::from_value::<WorkflowTask>(workflow) {
21048 Ok(task) => match worker.execute_workflow_task(task) {
21049 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
21050 outcome => failures.push(format!(
21051 "workflow {} codec returned {outcome:?}",
21052 codec_case.label()
21053 )),
21054 },
21055 Err(error) => failures.push(format!(
21056 "workflow {} codec failed transport deserialization: {error}",
21057 codec_case.label()
21058 )),
21059 }
21060
21061 let mut activity = json!({
21062 "task_id": format!("activity-{}", codec_case.label()),
21063 "activity_attempt_id": format!("attempt-{}", codec_case.label()),
21064 "activity_type": "codec.activity",
21065 "attempt_number": 1
21066 });
21067 codec_case.apply(&mut activity);
21068 match serde_json::from_value::<ActivityTask>(activity) {
21069 Ok(task) => match worker.execute_activity_task(task).await {
21070 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
21071 outcome => failures.push(format!(
21072 "activity {} codec returned {outcome:?}",
21073 codec_case.label()
21074 )),
21075 },
21076 Err(error) => failures.push(format!(
21077 "activity {} codec failed transport deserialization: {error}",
21078 codec_case.label()
21079 )),
21080 }
21081
21082 let mut query = json!({
21083 "query_task_id": format!("query-{}", codec_case.label()),
21084 "workflow_type": "codec.workflow",
21085 "query_name": "known"
21086 });
21087 codec_case.apply(&mut query);
21088 match serde_json::from_value::<QueryTask>(query) {
21089 Ok(task) => match worker.execute_query_task(task).await {
21090 Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
21091 outcome => failures.push(format!(
21092 "query {} codec returned {outcome:?}",
21093 codec_case.label()
21094 )),
21095 },
21096 Err(error) => failures.push(format!(
21097 "query {} codec failed transport deserialization: {error}",
21098 codec_case.label()
21099 )),
21100 }
21101 }
21102
21103 assert!(failures.is_empty(), "{}", failures.join("\n"));
21104 assert_eq!(
21105 handler_calls.load(Ordering::SeqCst),
21106 0,
21107 "invalid task codecs must not invoke a handler"
21108 );
21109 }
21110
21111 #[tokio::test]
21112 async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
21113 for codec_case in [
21114 InvalidTaskPayloadCodec::Missing,
21115 InvalidTaskPayloadCodec::Null,
21116 InvalidTaskPayloadCodec::NonString,
21117 ] {
21118 let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
21119 let client = Client::builder(server.base_url())
21120 .timeout(Duration::from_secs(2))
21121 .build()
21122 .expect("client");
21123 let mut worker = Worker::new(client, "rust-workers")
21124 .worker_id("codec-worker")
21125 .poll_timeout(Duration::from_millis(10));
21126 let handler_calls = Arc::new(AtomicUsize::new(0));
21127
21128 let calls = Arc::clone(&handler_calls);
21129 worker.register_workflow("codec.workflow", move |_ctx, _args| {
21130 calls.fetch_add(1, Ordering::SeqCst);
21131 async move { Ok(Value::Null) }
21132 });
21133 let calls = Arc::clone(&handler_calls);
21134 worker.register_activity("codec.activity", move |_ctx, _args| {
21135 calls.fetch_add(1, Ordering::SeqCst);
21136 async move { Ok(Value::Null) }
21137 });
21138 let calls = Arc::clone(&handler_calls);
21139 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
21140 calls.fetch_add(1, Ordering::SeqCst);
21141 async move { Ok(Value::Null) }
21142 });
21143
21144 assert_eq!(
21145 worker.run_once().await.expect("invalid tasks are settled"),
21146 3,
21147 "all {} codec tasks must be handled",
21148 codec_case.label()
21149 );
21150 assert_eq!(
21151 handler_calls.load(Ordering::SeqCst),
21152 0,
21153 "{} task codecs must fail before every handler",
21154 codec_case.label()
21155 );
21156
21157 for path in [
21158 "/api/worker/workflow-tasks/codec-workflow/fail",
21159 "/api/worker/activity-tasks/codec-activity/fail",
21160 "/api/worker/query-tasks/codec-query/fail",
21161 ] {
21162 let body = server.request_body(path);
21163 assert!(
21164 body["failure"]["message"]
21165 .as_str()
21166 .is_some_and(|message| message.contains("unsupported_payload_codec")),
21167 "{path} must receive the stable codec diagnostic for the {} case: {body}",
21168 codec_case.label()
21169 );
21170 }
21171 assert_eq!(
21172 server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
21173 ["reason"],
21174 "query_payload_decode_failed"
21175 );
21176 for path in [
21177 "/api/worker/workflow-tasks/codec-workflow/complete",
21178 "/api/worker/activity-tasks/codec-activity/complete",
21179 "/api/worker/query-tasks/codec-query/complete",
21180 ] {
21181 assert_eq!(
21182 server.request_count(path),
21183 0,
21184 "invalid {} codec task reached {path}",
21185 codec_case.label()
21186 );
21187 }
21188 }
21189 }
21190
21191 #[tokio::test]
21192 async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
21193 let client = Client::new("http://127.0.0.1:8080").expect("client");
21194 let mut worker = Worker::new(client, "rust-workers");
21195 let handler_calls = Arc::new(AtomicUsize::new(0));
21196
21197 let calls = Arc::clone(&handler_calls);
21198 worker.register_workflow("codec.workflow", move |_ctx, _args| {
21199 calls.fetch_add(1, Ordering::SeqCst);
21200 async move { Ok(Value::Null) }
21201 });
21202 let calls = Arc::clone(&handler_calls);
21203 worker.register_activity("codec.activity", move |_ctx, _args| {
21204 calls.fetch_add(1, Ordering::SeqCst);
21205 async move { Ok(Value::Null) }
21206 });
21207 let calls = Arc::clone(&handler_calls);
21208 worker.register_update("codec.workflow", "known", move |_ctx, _args| {
21209 calls.fetch_add(1, Ordering::SeqCst);
21210 async move { Ok(Value::Null) }
21211 });
21212 let calls = Arc::clone(&handler_calls);
21213 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
21214 calls.fetch_add(1, Ordering::SeqCst);
21215 async move { Ok(Value::Null) }
21216 });
21217
21218 let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
21219 workflow.payload_codec = "json".to_string();
21220 workflow.arguments = None;
21221 let error = worker
21222 .execute_workflow_task(workflow)
21223 .expect_err("task codec must be checked before workflow invocation");
21224 assert!(error.to_string().contains("unsupported_payload_codec"));
21225
21226 let activity = ActivityTask {
21227 task_id: "activity-invalid-codec".to_string(),
21228 activity_attempt_id: None,
21229 attempt_id: None,
21230 activity_type: "codec.activity".to_string(),
21231 payload_codec: "unknown".to_string(),
21232 arguments: None,
21233 attempt_number: 1,
21234 lease_owner: None,
21235 };
21236 let error = worker
21237 .execute_activity_task(activity)
21238 .await
21239 .expect_err("task codec must be checked before activity invocation");
21240 assert!(error.to_string().contains("unsupported_payload_codec"));
21241
21242 let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
21243 update.workflow_update_id = Some("update-invalid-codec".to_string());
21244 update.update_name = Some("known".to_string());
21245 update.history_events.push(history_event(
21246 "UpdateAccepted",
21247 json!({
21248 "update_id": "update-invalid-codec",
21249 "update_name": "known",
21250 "arguments": {"codec": "json", "blob": null}
21251 }),
21252 ));
21253 let error = worker
21254 .execute_workflow_task(update)
21255 .expect_err("nested update codec must be checked before handler lookup");
21256 assert!(error.to_string().contains("unsupported_payload_codec"));
21257
21258 let query: QueryTask = serde_json::from_value(json!({
21259 "query_task_id": "query-invalid-codec",
21260 "workflow_type": "codec.workflow",
21261 "query_name": "known",
21262 "payload_codec": DEFAULT_CODEC,
21263 "workflow_arguments": null,
21264 "query_arguments": null,
21265 "history_export": {
21266 "payloads": {"codec": DEFAULT_CODEC},
21267 "signals": [{
21268 "name": "empty",
21269 "payload_codec": "json",
21270 "arguments": null
21271 }]
21272 }
21273 }))
21274 .expect("query task");
21275 let failure = worker
21276 .execute_query_task(query)
21277 .await
21278 .expect_err("exported signal codec must be checked before query invocation");
21279 assert_eq!(failure.reason, "query_payload_decode_failed");
21280 assert!(failure.message.contains("unsupported_payload_codec"));
21281
21282 let exported_history: QueryTask = serde_json::from_value(json!({
21283 "query_task_id": "query-invalid-history-codec",
21284 "workflow_type": "codec.workflow",
21285 "query_name": "known",
21286 "payload_codec": DEFAULT_CODEC,
21287 "history_export": {
21288 "payloads": {"codec": DEFAULT_CODEC},
21289 "history_events": [{
21290 "type": "ActivityCompleted",
21291 "payload": {"payload_codec": "unknown", "result": null}
21292 }]
21293 }
21294 }))
21295 .expect("query task");
21296 let failure = worker
21297 .execute_query_task(exported_history)
21298 .await
21299 .expect_err("exported history codec must be checked before query invocation");
21300 assert_eq!(failure.reason, "query_payload_decode_failed");
21301 assert!(failure.message.contains("unsupported_payload_codec"));
21302 assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
21303
21304 let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
21305 unknown_workflow.arguments = None;
21306 unknown_workflow.history_events.push(history_event(
21307 "SignalReceived",
21308 json!({
21309 "signal_name": "empty",
21310 "payload_codec": "json",
21311 "arguments": null
21312 }),
21313 ));
21314 let error = worker
21315 .execute_workflow_task(unknown_workflow)
21316 .expect_err("history codec must precede unknown workflow outcome");
21317 assert!(error.to_string().contains("unsupported_payload_codec"));
21318
21319 let unknown_activity = ActivityTask {
21320 task_id: "activity-unknown".to_string(),
21321 activity_attempt_id: None,
21322 attempt_id: None,
21323 activity_type: "missing".to_string(),
21324 payload_codec: "json".to_string(),
21325 arguments: None,
21326 attempt_number: 1,
21327 lease_owner: None,
21328 };
21329 let error = worker
21330 .execute_activity_task(unknown_activity)
21331 .await
21332 .expect_err("codec must precede unknown activity outcome");
21333 assert!(error.to_string().contains("unsupported_payload_codec"));
21334
21335 let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
21336 unknown_update.payload_codec = "json".to_string();
21337 unknown_update.arguments = None;
21338 unknown_update.workflow_update_id = Some("update-unknown".to_string());
21339 unknown_update.update_name = Some("missing".to_string());
21340 let error = worker
21341 .execute_workflow_task(unknown_update)
21342 .expect_err("codec must precede fail_update shortcut");
21343 assert!(error.to_string().contains("unsupported_payload_codec"));
21344
21345 let unknown_query: QueryTask = serde_json::from_value(json!({
21346 "query_task_id": "query-unknown",
21347 "workflow_type": "missing",
21348 "query_name": "missing",
21349 "payload_codec": "json",
21350 "workflow_arguments": null,
21351 "query_arguments": null
21352 }))
21353 .expect("query task");
21354 let failure = worker
21355 .execute_query_task(unknown_query)
21356 .await
21357 .expect_err("codec must precede unknown query outcome");
21358 assert_eq!(failure.reason, "query_payload_decode_failed");
21359 assert!(failure.message.contains("unsupported_payload_codec"));
21360 }
21361
21362 #[tokio::test]
21363 async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
21364 let client = Client::new("http://127.0.0.1:8080").expect("client");
21365 let worker = Worker::new(client, "rust-workers");
21366
21367 for event_type in ["SignalReceived", "SignalApplied"] {
21368 for (payload_field, codec) in [
21369 ("value", "json"),
21370 ("input", "unknown"),
21371 ("arguments", "json"),
21372 ] {
21373 let payload = json!({
21374 "signal_name": "empty",
21375 payload_field: {"codec": codec, "blob": null}
21376 });
21377 let workflow = workflow_task(
21378 "missing",
21379 vec![history_event(event_type, payload.clone())],
21380 DEFAULT_CODEC,
21381 );
21382 let error = worker
21383 .execute_workflow_task(workflow)
21384 .expect_err("signal payload codec must precede unknown workflow outcome");
21385 assert!(
21386 error.to_string().contains("unsupported_payload_codec"),
21387 "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
21388 );
21389
21390 let query: QueryTask = serde_json::from_value(json!({
21391 "query_task_id": format!("query-{event_type}-{payload_field}"),
21392 "workflow_type": "missing",
21393 "query_name": "missing",
21394 "payload_codec": DEFAULT_CODEC,
21395 "workflow_arguments": null,
21396 "query_arguments": null,
21397 "history_events": [{
21398 "event_type": event_type,
21399 "payload": payload
21400 }]
21401 }))
21402 .expect("query task");
21403 let failure = worker
21404 .execute_query_task(query)
21405 .await
21406 .expect_err("signal payload codec must precede unknown query outcome");
21407 assert_eq!(
21408 failure.reason, "query_payload_decode_failed",
21409 "{event_type}.{payload_field} returned an unrelated query outcome"
21410 );
21411 assert!(
21412 failure.message.contains("unsupported_payload_codec"),
21413 "{event_type}.{payload_field} returned an unrelated query error: {}",
21414 failure.message
21415 );
21416 }
21417 }
21418 }
21419
21420 #[test]
21421 fn workflow_context_schedules_activity_until_completion_is_in_history() {
21422 let ctx = WorkflowContext {
21423 cancellation_scope_id: "root".into(),
21424 state: Arc::new(Mutex::new(
21425 WorkflowState::new_with_identity(
21426 Vec::new(),
21427 Some("wf-parent".to_string()),
21428 Some("run-parent".to_string()),
21429 "rust-workers".to_string(),
21430 DEFAULT_CODEC.to_string(),
21431 None,
21432 )
21433 .expect("workflow state"),
21434 )),
21435 };
21436
21437 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
21438 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21439 assert!(matches!(
21440 call.as_mut().poll(&mut task_context),
21441 Poll::Pending
21442 ));
21443
21444 let commands = ctx.take_commands().expect("commands");
21445 assert_eq!(commands[0]["type"], "schedule_activity");
21446 assert_eq!(commands[0]["activity_type"], "hello.activity");
21447 }
21448
21449 #[test]
21450 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
21451 let ctx = workflow_context(Vec::new());
21452 let options = ActivityOptions::new()
21453 .task_queue("payments")
21454 .retry_policy(
21455 ActivityRetryPolicy::new(4)
21456 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
21457 .non_retryable_error_type("ValidationError"),
21458 )
21459 .start_to_close_timeout(Duration::from_secs(120))
21460 .schedule_to_start_timeout(Duration::from_secs(10))
21461 .schedule_to_close_timeout(Duration::from_secs(300))
21462 .heartbeat_timeout(Duration::from_secs(15));
21463 let mut call = Box::pin(ctx.activity_with_options(
21464 "charge-card",
21465 options,
21466 json!([{"order_id": "o-1"}]),
21467 ));
21468 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21469
21470 assert!(matches!(
21471 call.as_mut().poll(&mut task_context),
21472 Poll::Pending
21473 ));
21474 assert!(matches!(
21475 call.as_mut().poll(&mut task_context),
21476 Poll::Pending
21477 ));
21478
21479 let commands = ctx.take_commands().expect("activity command");
21480 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
21481 assert_eq!(commands[0]["queue"], "payments");
21482 assert_eq!(
21483 commands[0]["retry_policy"],
21484 json!({
21485 "max_attempts": 4,
21486 "backoff_seconds": [1, 3, 9],
21487 "non_retryable_error_types": ["ValidationError"],
21488 })
21489 );
21490 assert_eq!(commands[0]["start_to_close_timeout"], 120);
21491 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
21492 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
21493 assert_eq!(commands[0]["heartbeat_timeout"], 15);
21494 }
21495
21496 fn completed_worker_session_history(session: &WorkerSessionOptions) -> Vec<HistoryEvent> {
21497 vec![
21498 history_event(
21499 "ActivityScheduled",
21500 json!({"sequence":1,"activity_type":"render",
21501 "activity":{"queue":"gpu-workers","worker_session":session.to_wire().unwrap()}}),
21502 ),
21503 history_event(
21504 "ActivityCompleted",
21505 json!({"sequence":1,"activity_type":"render",
21506 "payload_codec":DEFAULT_CODEC,
21507 "result":encode_typed_envelope(&AvroValue::Long(42),DEFAULT_CODEC).unwrap()}),
21508 ),
21509 ]
21510 }
21511
21512 #[test]
21513 fn worker_session_activity_routes_to_session_queue_once() {
21514 let ctx = workflow_context(Vec::new());
21515 let session = WorkerSessionOptions::new("render-1").queue("gpu-workers");
21516 let mut call = Box::pin(
21517 ctx.activity("render", json!([]))
21518 .in_worker_session(session.clone()),
21519 );
21520 let mut cx = TaskContext::from_waker(noop_waker_ref());
21521 assert!(call.as_mut().poll(&mut cx).is_pending());
21522 assert!(call.as_mut().poll(&mut cx).is_pending());
21523 let commands = ctx.take_commands().unwrap();
21524 assert_eq!(commands.len(), 1);
21525 assert_eq!(commands[0]["queue"], "gpu-workers");
21526 assert_eq!(commands[0]["worker_session"], session.to_wire().unwrap());
21527 }
21528
21529 #[tokio::test]
21530 async fn worker_session_cold_replay_uses_recorded_avro_result() {
21531 let session = WorkerSessionOptions::new("render-1").queue("gpu-workers");
21532 for _replacement in 0..2 {
21533 let ctx = workflow_context(completed_worker_session_history(&session));
21534 let result: i64 = ctx
21535 .activity("render", json!([]))
21536 .in_worker_session(session.clone())
21537 .typed()
21538 .await
21539 .unwrap();
21540 assert_eq!(result, 42);
21541 assert!(ctx.take_commands().unwrap().is_empty());
21542 }
21543 }
21544
21545 #[test]
21546 fn worker_session_cold_replay_rejects_changed_identity_and_lifetime() {
21547 let original = WorkerSessionOptions::new("render-1").queue("gpu-workers");
21548 for changed in [
21549 Some(WorkerSessionOptions::new("render-2").queue("gpu-workers")),
21550 Some(original.clone().ttl_seconds(60)),
21551 Some(original.clone().requirements(["gpu:l4"])),
21552 None,
21553 ] {
21554 let ctx = workflow_context(completed_worker_session_history(&original));
21555 let mut call = ctx.activity_with_options(
21556 "render",
21557 ActivityOptions::new().task_queue("gpu-workers"),
21558 json!([]),
21559 );
21560 if let Some(changed) = changed {
21561 call = call.in_worker_session(changed);
21562 }
21563 let mut call = Box::pin(call);
21564 let mut cx = TaskContext::from_waker(noop_waker_ref());
21565 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
21566 call.as_mut().poll(&mut cx)
21567 else {
21568 panic!("changed session must fail replay");
21569 };
21570 assert_eq!(failure.reason, "activity_worker_session_mismatch");
21571 assert!(ctx.take_commands().unwrap().is_empty());
21572 }
21573 }
21574
21575 #[test]
21576 fn worker_session_local_and_contradictory_queue_routing_emit_no_commands() {
21577 let session = WorkerSessionOptions::new("render-1").queue("gpu-workers");
21578 let ctx = workflow_context(Vec::new());
21579 let mut local = Box::pin(
21580 ctx.local_activity("render", json!([]))
21581 .in_worker_session(session.clone()),
21582 );
21583 let mut cx = TaskContext::from_waker(noop_waker_ref());
21584 assert!(matches!(
21585 local.as_mut().poll(&mut cx),
21586 Poll::Ready(Err(Error::WorkerLoop(_)))
21587 ));
21588 assert!(ctx.take_commands().unwrap().is_empty());
21589 let mut remote = Box::pin(
21590 ctx.activity_with_options(
21591 "render",
21592 ActivityOptions::new().task_queue("other"),
21593 json!([]),
21594 )
21595 .in_worker_session(session),
21596 );
21597 assert!(matches!(
21598 remote.as_mut().poll(&mut cx),
21599 Poll::Ready(Err(Error::WorkerLoop(_)))
21600 ));
21601 assert!(ctx.take_commands().unwrap().is_empty());
21602 }
21603
21604 #[test]
21605 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
21606 let ctx = workflow_context(Vec::new());
21607 let options = ActivityOptions::new().retry_policy(
21608 ActivityRetryPolicy::new(3)
21609 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
21610 );
21611 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
21612 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21613
21614 assert!(matches!(
21615 call.as_mut().poll(&mut task_context),
21616 Poll::Pending
21617 ));
21618 assert_eq!(
21619 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
21620 json!([1, 2])
21621 );
21622 }
21623
21624 #[test]
21625 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
21626 let cases = [
21627 (
21628 ActivityOptions::new().task_queue(" "),
21629 ActivityOptionsErrorKind::EmptyTaskQueue,
21630 ),
21631 (
21632 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
21633 ActivityOptionsErrorKind::EmptyRetryPolicy,
21634 ),
21635 (
21636 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
21637 ActivityOptionsErrorKind::InvalidMaxAttempts,
21638 ),
21639 (
21640 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
21641 max_attempts: None,
21642 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
21643 non_retryable_error_types: Vec::new(),
21644 }),
21645 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
21646 ),
21647 (
21648 ActivityOptions::new().retry_policy(
21649 ActivityRetryPolicy::new(2)
21650 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
21651 ),
21652 ActivityOptionsErrorKind::TooManyBackoffIntervals,
21653 ),
21654 (
21655 ActivityOptions::new().retry_policy(
21656 ActivityRetryPolicy::new(2).exponential_backoff(
21657 Duration::from_secs(1),
21658 0,
21659 None,
21660 ),
21661 ),
21662 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
21663 ),
21664 (
21665 ActivityOptions::new()
21666 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
21667 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
21668 ),
21669 (
21670 ActivityOptions::new().retry_policy(
21671 ActivityRetryPolicy::new(10_002).exponential_backoff(
21672 Duration::from_secs(1),
21673 1,
21674 None,
21675 ),
21676 ),
21677 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
21678 ),
21679 (
21680 ActivityOptions::new().retry_policy(
21681 ActivityRetryPolicy::new(2)
21682 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
21683 ),
21684 ActivityOptionsErrorKind::BackoffOverflow,
21685 ),
21686 ];
21687
21688 for (options, expected_kind) in cases {
21689 let ctx = workflow_context(Vec::new());
21690 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
21691 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21692 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
21693 call.as_mut().poll(&mut task_context)
21694 else {
21695 panic!("expected typed activity validation error");
21696 };
21697 assert_eq!(error.kind, expected_kind);
21698 assert!(ctx.take_commands().expect("commands").is_empty());
21699 }
21700 }
21701
21702 #[test]
21703 fn activity_options_validate_positive_and_ordered_timeouts() {
21704 let zero_timeout_cases = [
21705 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
21706 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
21707 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
21708 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
21709 ];
21710 for options in zero_timeout_cases {
21711 assert_eq!(
21712 options.validate().expect_err("zero timeout").kind,
21713 ActivityOptionsErrorKind::TimeoutNotPositive
21714 );
21715 }
21716
21717 let ordering_cases = [
21718 ActivityOptions::new()
21719 .heartbeat_timeout(Duration::from_secs(11))
21720 .start_to_close_timeout(Duration::from_secs(10)),
21721 ActivityOptions::new()
21722 .start_to_close_timeout(Duration::from_secs(31))
21723 .schedule_to_close_timeout(Duration::from_secs(30)),
21724 ActivityOptions::new()
21725 .schedule_to_start_timeout(Duration::from_secs(31))
21726 .schedule_to_close_timeout(Duration::from_secs(30)),
21727 ];
21728 for options in ordering_cases {
21729 assert_eq!(
21730 options.validate().expect_err("timeout order").kind,
21731 ActivityOptionsErrorKind::TimeoutOrder
21732 );
21733 }
21734
21735 assert_eq!(
21736 ActivityOptions::new()
21737 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
21738 .validate()
21739 .expect_err("protocol integer overflow")
21740 .kind,
21741 ActivityOptionsErrorKind::TimeoutOverflow
21742 );
21743 }
21744
21745 #[test]
21746 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
21747 let ctx = workflow_context(completed_retry_activity_history());
21748 let mut call =
21749 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
21750 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21751
21752 assert!(matches!(
21753 call.as_mut().poll(&mut task_context),
21754 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
21755 ));
21756 assert!(ctx.take_commands().expect("commands").is_empty());
21757 ctx.ensure_history_consumed().expect("history consumed");
21758 }
21759
21760 #[test]
21761 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
21762 let mut options = retry_activity_options();
21763 options
21764 .retry_policy
21765 .as_mut()
21766 .expect("retry policy")
21767 .non_retryable_error_types
21768 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
21769
21770 let new_ctx = workflow_context(Vec::new());
21771 let mut new_call =
21772 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
21773 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21774 assert!(matches!(
21775 new_call.as_mut().poll(&mut task_context),
21776 Poll::Pending
21777 ));
21778 let commands = new_ctx.take_commands().expect("commands");
21779 assert_eq!(commands.len(), 1);
21780 assert_eq!(
21781 commands[0]["retry_policy"]["non_retryable_error_types"],
21782 json!(["PermanentError"])
21783 );
21784
21785 let replay_ctx = workflow_context(completed_retry_activity_history());
21786 let mut replay_call =
21787 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
21788 assert!(matches!(
21789 replay_call.as_mut().poll(&mut task_context),
21790 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
21791 ));
21792 assert!(replay_ctx.take_commands().expect("commands").is_empty());
21793 replay_ctx
21794 .ensure_history_consumed()
21795 .expect("history consumed");
21796 }
21797
21798 #[test]
21799 fn replayed_intermediate_retry_remains_pending_across_restarts() {
21800 let history = completed_retry_activity_history()
21801 .into_iter()
21802 .take(3)
21803 .collect::<Vec<_>>();
21804
21805 for _restart in 0..2 {
21806 let ctx = workflow_context(history.clone());
21807 let mut call =
21808 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
21809 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21810 assert!(matches!(
21811 call.as_mut().poll(&mut task_context),
21812 Poll::Pending
21813 ));
21814 assert!(ctx.take_commands().expect("commands").is_empty());
21815 }
21816 }
21817
21818 #[test]
21819 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
21820 let mut changed_queue = retry_activity_options();
21821 changed_queue.task_queue = Some("different-queue".to_string());
21822
21823 let mut changed_max_attempts = retry_activity_options();
21824 let retry_policy = changed_max_attempts
21825 .retry_policy
21826 .as_mut()
21827 .expect("retry policy");
21828 retry_policy.max_attempts = Some(4);
21829
21830 let mut changed_backoff = retry_activity_options();
21831 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
21832 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
21833 Duration::from_secs(3),
21834 Duration::from_secs(4),
21835 ]));
21836
21837 let mut changed_non_retryable_types = retry_activity_options();
21838 let retry_policy = changed_non_retryable_types
21839 .retry_policy
21840 .as_mut()
21841 .expect("retry policy");
21842 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
21843
21844 let mut changed_start_to_close = retry_activity_options();
21845 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
21846 let mut changed_schedule_to_start = retry_activity_options();
21847 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
21848 let mut changed_schedule_to_close = retry_activity_options();
21849 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
21850 let mut changed_heartbeat = retry_activity_options();
21851 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
21852
21853 let cases = [
21854 (changed_queue, "activity_task_queue_mismatch"),
21855 (changed_max_attempts, "activity_retry_policy_mismatch"),
21856 (changed_backoff, "activity_retry_policy_mismatch"),
21857 (
21858 changed_non_retryable_types,
21859 "activity_retry_policy_mismatch",
21860 ),
21861 (changed_start_to_close, "activity_retry_policy_mismatch"),
21862 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
21863 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
21864 (changed_heartbeat, "activity_retry_policy_mismatch"),
21865 ];
21866
21867 for (options, expected_reason) in cases {
21868 let ctx = workflow_context(completed_retry_activity_history());
21869 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
21870 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21871 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
21872 call.as_mut().poll(&mut task_context)
21873 else {
21874 panic!("changed activity options must fail replay");
21875 };
21876 assert_eq!(failure.reason, expected_reason);
21877 assert_eq!(failure.sequence, Some(1));
21878 assert!(ctx.take_commands().expect("commands").is_empty());
21879 }
21880 }
21881
21882 #[test]
21883 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
21884 let cases = [
21885 (
21886 "execution_mode",
21887 json!("local"),
21888 "activity_execution_mode_mismatch",
21889 ),
21890 (
21891 "snapshot_version",
21892 json!(2),
21893 "activity_retry_policy_mismatch",
21894 ),
21895 ];
21896
21897 for (field, value, expected_reason) in cases {
21898 let mut history = completed_retry_activity_history();
21899 let activity = history[0].payload["activity"]
21900 .as_object_mut()
21901 .expect("activity snapshot");
21902 if field == "execution_mode" {
21903 activity.insert(field.to_string(), value);
21904 } else {
21905 activity["retry_policy"]
21906 .as_object_mut()
21907 .expect("retry snapshot")
21908 .insert(field.to_string(), value);
21909 }
21910
21911 let ctx = workflow_context(history);
21912 let mut call =
21913 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
21914 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21915 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
21916 call.as_mut().poll(&mut task_context)
21917 else {
21918 panic!("changed {field} must fail replay");
21919 };
21920 assert_eq!(failure.reason, expected_reason);
21921 assert_eq!(failure.sequence, Some(1));
21922 assert!(ctx.take_commands().expect("commands").is_empty());
21923 }
21924 }
21925
21926 #[test]
21927 fn remote_activity_rejects_local_history_markers_without_option_snapshot() {
21928 for (nested, field, value, terminal_only) in [
21929 (false, "execution_mode", json!("local"), false),
21930 (false, "local_activity", json!(true), false),
21931 (true, "local_activity", json!(true), false),
21932 (false, "local_activity", json!(true), true),
21933 ] {
21934 let mut history = completed_retry_activity_history();
21935 history[0]
21936 .payload
21937 .as_object_mut()
21938 .unwrap()
21939 .remove("activity");
21940 let event = if terminal_only {
21941 history.last_mut().unwrap()
21942 } else {
21943 &mut history[0]
21944 };
21945 if nested {
21946 event.payload["activity"] = json!({(field): value});
21947 } else {
21948 event.payload[field] = value;
21949 }
21950 let ctx = workflow_context(history);
21951 let mut call =
21952 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
21953 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21954 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
21955 call.as_mut().poll(&mut task_context)
21956 else {
21957 panic!("local history must not replay as a remote activity: {nested}/{field}/{terminal_only}");
21958 };
21959 assert_eq!(failure.reason, "activity_execution_mode_mismatch");
21960 assert_eq!(failure.sequence, Some(1));
21961 assert!(ctx.take_commands().expect("commands").is_empty());
21962 }
21963 }
21964
21965 #[test]
21966 fn activity_history_rejects_conflicting_local_execution_markers() {
21967 let mut history = completed_retry_activity_history();
21968 history.last_mut().unwrap().payload["local_activity"] = json!(true);
21969 let error = WorkflowState::new(history, "queue".into(), DEFAULT_CODEC.into(), None)
21970 .expect_err("remote schedule cannot become a local completion");
21971 assert!(
21972 matches!(error, Error::NonDeterministicReplay(ReplayFailure { reason, sequence: Some(1), .. })
21973 if reason == "activity_execution_mode_conflict")
21974 );
21975 }
21976
21977 #[test]
21978 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
21979 let mut history = completed_retry_activity_history();
21980 let activity = history[0].payload["activity"]
21981 .as_object_mut()
21982 .expect("activity snapshot");
21983 activity.remove("execution_mode");
21984 activity.remove("retry_policy");
21985
21986 let mut current = retry_activity_options();
21987 current.start_to_close_timeout = Some(Duration::from_secs(45));
21988 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
21989 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
21990 current.heartbeat_timeout = Some(Duration::from_secs(12));
21991
21992 let ctx = workflow_context(history);
21993 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
21994 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21995 assert!(matches!(
21996 call.as_mut().poll(&mut task_context),
21997 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
21998 ));
21999 assert!(ctx.take_commands().expect("commands").is_empty());
22000 ctx.ensure_history_consumed().expect("history consumed");
22001 }
22002
22003 #[test]
22004 fn terminal_activity_failed_after_start_returns_typed_failure() {
22005 let history = vec![
22006 history_event(
22007 "ActivityScheduled",
22008 json!({
22009 "sequence": 1,
22010 "activity_type": "flaky",
22011 "activity_execution_id": "act-terminal",
22012 "activity": {
22013 "id": "act-terminal",
22014 "sequence": 1,
22015 "type": "flaky",
22016 "queue": "critical-activities",
22017 "retry_policy": {
22018 "snapshot_version": 1,
22019 "max_attempts": 3,
22020 "backoff_seconds": [2, 4],
22021 "non_retryable_error_types": ["PermanentError"]
22022 }
22023 }
22024 }),
22025 ),
22026 history_event(
22027 "ActivityStarted",
22028 json!({
22029 "sequence": 1,
22030 "activity_type": "flaky",
22031 "activity_execution_id": "act-terminal",
22032 "activity_attempt_id": "attempt-1",
22033 "attempt_number": 1
22034 }),
22035 ),
22036 history_event(
22037 "ActivityFailed",
22038 json!({
22039 "sequence": 1,
22040 "activity_type": "flaky",
22041 "activity_execution_id": "act-terminal",
22042 "activity_attempt_id": "attempt-1",
22043 "attempt_number": 1,
22044 "failure_id": "failure-terminal",
22045 "failure_category": "activity",
22046 "exception_type": "PermanentError",
22047 "message": "cannot retry",
22048 "non_retryable": true
22049 }),
22050 ),
22051 ];
22052 let ctx = workflow_context(history);
22053 let mut call =
22054 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
22055 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22056
22057 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
22058 call.as_mut().poll(&mut task_context)
22059 else {
22060 panic!("terminal ActivityFailed must settle the activity future");
22061 };
22062 assert_eq!(failure.kind, ActivityFailureKind::Failed);
22063 assert_eq!(
22064 failure.activity_execution_id.as_deref(),
22065 Some("act-terminal")
22066 );
22067 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
22068 assert!(failure.non_retryable);
22069 assert!(ctx.take_commands().expect("commands").is_empty());
22070 ctx.ensure_history_consumed().expect("history consumed");
22071 }
22072
22073 #[test]
22074 fn activity_terminal_events_return_machine_readable_failures() {
22075 let cases = [
22076 (
22077 "ActivityFailed",
22078 json!({
22079 "sequence": 1,
22080 "activity_type": "charge-card",
22081 "activity_execution_id": "act-1",
22082 "activity_attempt_id": "attempt-2",
22083 "attempt_number": 2,
22084 "failure_id": "failure-1",
22085 "failure_category": "activity",
22086 "exception_type": "PaymentDeclined",
22087 "exception_class": "payments.PaymentDeclined",
22088 "message": "card declined",
22089 "non_retryable": true
22090 }),
22091 ActivityFailureKind::Failed,
22092 "activity",
22093 ),
22094 (
22095 "ActivityCancelled",
22096 json!({
22097 "sequence": 1,
22098 "activity_type": "charge-card",
22099 "activity_execution_id": "act-1",
22100 "activity_attempt_id": "attempt-1"
22101 }),
22102 ActivityFailureKind::Cancelled,
22103 "cancelled",
22104 ),
22105 ];
22106
22107 for (event_type, payload, expected_kind, expected_reason) in cases {
22108 let ctx = workflow_context(vec![history_event(event_type, payload)]);
22109 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
22110 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22111 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
22112 call.as_mut().poll(&mut task_context)
22113 else {
22114 panic!("expected terminal activity failure");
22115 };
22116 assert_eq!(failure.kind, expected_kind);
22117 assert_eq!(failure.reason, expected_reason);
22118 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
22119 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
22120 }
22121 }
22122
22123 #[test]
22124 fn every_activity_timeout_class_is_typed() {
22125 for timeout_kind in [
22126 "start_to_close",
22127 "schedule_to_start",
22128 "schedule_to_close",
22129 "heartbeat",
22130 ] {
22131 let ctx = workflow_context(vec![history_event(
22132 "ActivityTimedOut",
22133 json!({
22134 "sequence": 1,
22135 "activity_type": "slow",
22136 "activity_execution_id": "act-timeout",
22137 "activity_attempt_id": "attempt-timeout",
22138 "failure_category": "timeout",
22139 "timeout_kind": timeout_kind,
22140 "message": "deadline expired"
22141 }),
22142 )]);
22143 let mut call = Box::pin(ctx.activity("slow", json!([])));
22144 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22145 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
22146 call.as_mut().poll(&mut task_context)
22147 else {
22148 panic!("expected timeout failure");
22149 };
22150 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
22151 assert_eq!(failure.reason, timeout_kind);
22152 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
22153 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
22154 }
22155 }
22156
22157 #[test]
22158 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
22159 let ctx = workflow_context(Vec::new());
22160 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
22161 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22162
22163 assert!(matches!(
22164 sleep.as_mut().poll(&mut task_context),
22165 Poll::Pending
22166 ));
22167 assert!(matches!(
22168 sleep.as_mut().poll(&mut task_context),
22169 Poll::Pending
22170 ));
22171
22172 let commands = ctx.take_commands().expect("timer command");
22173 assert_eq!(
22174 commands,
22175 vec![json!({
22176 "type": "start_timer",
22177 "delay_seconds": 2,
22178 })]
22179 );
22180 }
22181
22182 #[test]
22183 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
22184 let history = vec![
22185 history_event(
22186 "TimerScheduled",
22187 json!({
22188 "sequence": 1,
22189 "timer_id": "timer-1",
22190 "delay_seconds": 5,
22191 "fire_at": "2026-07-11T12:00:05Z",
22192 }),
22193 ),
22194 history_event(
22195 "TimerFired",
22196 json!({
22197 "sequence": 1,
22198 "timer_id": "timer-1",
22199 "delay_seconds": 5,
22200 "fire_at": "2026-07-11T12:00:05Z",
22201 "fired_at": "2026-07-11T12:00:05Z",
22202 }),
22203 ),
22204 ];
22205
22206 for _restart in 0..2 {
22207 let ctx = workflow_context(history.clone());
22208 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
22209 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22210 assert!(matches!(
22211 sleep.as_mut().poll(&mut task_context),
22212 Poll::Ready(Ok(()))
22213 ));
22214 assert!(ctx.take_commands().expect("commands").is_empty());
22215 ctx.ensure_history_consumed().expect("history consumed");
22216 }
22217 }
22218
22219 #[test]
22220 fn workflow_sleep_rejects_changed_delay_during_replay() {
22221 let ctx = workflow_context(vec![
22222 history_event(
22223 "TimerScheduled",
22224 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
22225 ),
22226 history_event(
22227 "TimerFired",
22228 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
22229 ),
22230 ]);
22231 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
22232 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22233
22234 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
22235 sleep.as_mut().poll(&mut task_context)
22236 else {
22237 panic!("changed timer delay must be rejected");
22238 };
22239 assert_eq!(failure.reason, "timer_delay_mismatch");
22240 assert_eq!(failure.sequence, Some(1));
22241 }
22242
22243 #[test]
22244 fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
22245 let ctx = workflow_context(Vec::new());
22246 let mut wait = Box::pin(
22247 ctx.wait_condition(
22248 ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
22249 .timeout(Duration::from_millis(60_001)),
22250 || Ok(false),
22251 ),
22252 );
22253 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22254
22255 assert!(matches!(
22256 wait.as_mut().poll(&mut task_context),
22257 Poll::Pending
22258 ));
22259 assert!(matches!(
22260 wait.as_mut().poll(&mut task_context),
22261 Poll::Pending
22262 ));
22263 assert_eq!(
22264 ctx.take_commands().expect("condition command"),
22265 vec![json!({
22266 "type": "open_condition_wait",
22267 "condition_wait_occurrence_id": "rust:condition-wait:0",
22268 "condition_key": "approval.ready",
22269 "condition_definition_fingerprint": "sha256:approval-v1",
22270 "timeout_seconds": 61,
22271 })]
22272 );
22273 }
22274
22275 #[test]
22276 fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
22277 let ctx = workflow_context(Vec::new());
22278 let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
22279 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22280 assert!(matches!(
22281 satisfied.as_mut().poll(&mut task_context),
22282 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22283 ));
22284
22285 let mut timed_out = Box::pin(wait_condition!(
22286 ctx,
22287 "no-wait",
22288 timeout: Duration::ZERO,
22289 || Ok(false),
22290 ));
22291 assert!(matches!(
22292 timed_out.as_mut().poll(&mut task_context),
22293 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
22294 ));
22295 assert!(ctx.take_commands().expect("commands").is_empty());
22296 }
22297
22298 #[test]
22299 fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
22300 let signal_history = vec![
22301 history_event(
22302 "ConditionWaitOpened",
22303 json!({
22304 "sequence": 4,
22305 "condition_wait_id": "condition:4",
22306 "condition_wait_occurrence_id": "rust:condition-wait:0",
22307 "condition_key": "approval",
22308 "condition_definition_fingerprint": "sha256:approval-v1",
22309 "timeout_seconds": 30,
22310 }),
22311 ),
22312 history_event(
22313 "SignalReceived",
22314 json!({
22315 "workflow_sequence": 4,
22316 "signal_name": "approve",
22317 "arguments": fixture_envelope(json!(["Ada"])),
22318 }),
22319 ),
22320 ];
22321 for _worker_before_or_after_restart in 0..2 {
22322 let ctx = workflow_context(signal_history.clone());
22323 let predicate_ctx = ctx.clone();
22324 let mut wait = Box::pin(
22325 ctx.wait_condition(
22326 ConditionWaitOptions::new("approval", "sha256:approval-v1")
22327 .timeout(Duration::from_secs(30)),
22328 move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
22329 ),
22330 );
22331 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22332 assert!(matches!(
22333 wait.as_mut().poll(&mut task_context),
22334 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22335 ));
22336 assert!(ctx.take_commands().expect("commands").is_empty());
22337 ctx.ensure_history_consumed().expect("condition consumed");
22338 }
22339
22340 let update_history = vec![
22341 history_event(
22342 "ConditionWaitOpened",
22343 json!({
22344 "sequence": 7,
22345 "condition_wait_id": "condition:7",
22346 "condition_wait_occurrence_id": "rust:condition-wait:0",
22347 "condition_key": "update-approval",
22348 "condition_definition_fingerprint": "sha256:update-approval-v1",
22349 }),
22350 ),
22351 history_event(
22352 "UpdateApplied",
22353 json!({
22354 "sequence": 7,
22355 "update_id": "update-1",
22356 "update_name": "approve",
22357 "arguments": fixture_envelope(json!([true])),
22358 }),
22359 ),
22360 ];
22361 let ctx = workflow_context(update_history);
22362 let predicate_ctx = ctx.clone();
22363 let mut wait = Box::pin(ctx.wait_condition(
22364 ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
22365 move || {
22366 Ok(predicate_ctx
22367 .updates("approve")?
22368 .first()
22369 .and_then(|arguments| arguments.first())
22370 .and_then(Value::as_bool)
22371 == Some(true))
22372 },
22373 ));
22374 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22375 assert!(matches!(
22376 wait.as_mut().poll(&mut task_context),
22377 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22378 ));
22379 assert!(ctx.take_commands().expect("commands").is_empty());
22380 ctx.ensure_history_consumed().expect("condition consumed");
22381 }
22382
22383 #[test]
22384 fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
22385 let open_history = vec![
22386 history_event(
22387 "ConditionWaitOpened",
22388 json!({
22389 "sequence": 3,
22390 "condition_wait_id": "condition:3",
22391 "condition_wait_occurrence_id": "rust:condition-wait:0",
22392 "condition_key": "two-votes",
22393 "condition_definition_fingerprint": "sha256:two-votes-v1",
22394 "timeout_seconds": 120,
22395 }),
22396 ),
22397 history_event(
22398 "SignalReceived",
22399 json!({
22400 "workflow_sequence": 3,
22401 "signal_name": "vote",
22402 "arguments": fixture_envelope(json!(["first"])),
22403 }),
22404 ),
22405 ];
22406 for _worker_before_or_after_restart in 0..2 {
22407 let ctx = workflow_context(open_history.clone());
22408 let predicate_ctx = ctx.clone();
22409 let mut wait = Box::pin(
22410 ctx.wait_condition(
22411 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
22412 .timeout(Duration::from_secs(120)),
22413 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
22414 ),
22415 );
22416 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22417 assert!(matches!(
22418 wait.as_mut().poll(&mut task_context),
22419 Poll::Pending
22420 ));
22421 assert_eq!(
22422 ctx.take_commands().expect("reopened condition"),
22423 vec![json!({
22424 "type": "open_condition_wait",
22425 "condition_wait_occurrence_id": "rust:condition-wait:0",
22426 "condition_key": "two-votes",
22427 "condition_definition_fingerprint": "sha256:two-votes-v1",
22428 "timeout_seconds": 120,
22429 })]
22430 );
22431 }
22432
22433 let satisfied_ctx = workflow_context(vec![
22434 history_event(
22435 "ConditionWaitOpened",
22436 json!({
22437 "sequence": 5,
22438 "condition_wait_id": "condition:5",
22439 "condition_wait_occurrence_id": "rust:condition-wait:0",
22440 "condition_key": "approval",
22441 "condition_definition_fingerprint": "sha256:approval-v1",
22442 }),
22443 ),
22444 history_event(
22445 "ConditionWaitSatisfied",
22446 json!({
22447 "sequence": 5,
22448 "condition_wait_id": "condition:5",
22449 "condition_wait_occurrence_id": "rust:condition-wait:0",
22450 "condition_key": "approval",
22451 "condition_definition_fingerprint": "sha256:approval-v1",
22452 }),
22453 ),
22454 ]);
22455 let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
22456 ConditionWaitOptions::new("approval", "sha256:approval-v1"),
22457 || Ok(false),
22458 ));
22459 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22460 assert!(matches!(
22461 satisfied.as_mut().poll(&mut task_context),
22462 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22463 ));
22464
22465 let timed_out_ctx = workflow_context(vec![
22466 history_event(
22467 "ConditionWaitOpened",
22468 json!({
22469 "sequence": 8,
22470 "condition_wait_id": "condition:8",
22471 "condition_wait_occurrence_id": "rust:condition-wait:0",
22472 "condition_key": "approval-timeout",
22473 "condition_definition_fingerprint": "sha256:approval-timeout-v1",
22474 "timeout_seconds": 5,
22475 }),
22476 ),
22477 history_event(
22478 "TimerScheduled",
22479 json!({
22480 "sequence": 9,
22481 "timer_id": "condition-timer:9",
22482 "timer_kind": "condition_timeout",
22483 "condition_wait_id": "condition:8",
22484 "delay_seconds": 5,
22485 }),
22486 ),
22487 history_event(
22488 "TimerFired",
22489 json!({
22490 "sequence": 9,
22491 "timer_id": "condition-timer:9",
22492 "timer_kind": "condition_timeout",
22493 "condition_wait_id": "condition:8",
22494 "delay_seconds": 5,
22495 }),
22496 ),
22497 ]);
22498 let mut timed_out = Box::pin(
22499 timed_out_ctx.wait_condition(
22500 ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
22501 .timeout(Duration::from_secs(5)),
22502 || Ok(true),
22503 ),
22504 );
22505 assert!(matches!(
22506 timed_out.as_mut().poll(&mut task_context),
22507 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
22508 ));
22509 }
22510
22511 #[test]
22512 fn satisfied_reopened_selection_condition_waits_for_its_canonical_winner() {
22513 let ctx = workflow_context(reopened_selection_condition_history());
22514 {
22515 let state = ctx.state.lock().unwrap();
22516 assert!(matches!(
22517 selection_member_recorded_command(&state, 2),
22518 Some(RecordedCommand::ConditionWait {
22519 sequence: 3,
22520 result: None,
22521 ..
22522 })
22523 ));
22524 }
22525 let mut selected = Box::pin(ctx.select_keyed(vec![
22526 ("timer", ParallelOperation::timer(Duration::from_secs(300))),
22527 (
22528 "votes",
22529 ParallelOperation::condition(
22530 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
22531 || Ok(true),
22532 ),
22533 ),
22534 ]));
22535 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22536 assert!(selected.as_mut().poll(&mut task_context).is_pending());
22537 assert!(
22538 ctx.take_commands().unwrap().is_empty(),
22539 "true predicate must not reopen the recorded wait"
22540 );
22541 ctx.ensure_history_consumed()
22542 .expect("physical reopens are consumed while canonical selection is pending");
22543 }
22544
22545 #[test]
22546 fn reopened_selection_condition_replays_its_latest_physical_winner() {
22547 for (terminal_type, acknowledge_timeout) in [
22548 ("ConditionWaitSatisfied", false),
22549 ("ConditionWaitTimedOut", false),
22550 ("TimerFired", false),
22551 ("TimerFired", true),
22552 ] {
22553 let mut history = reopened_selection_condition_history();
22554 let original_identity = history[1].payload["condition_wait_id"].clone();
22555 let uses_timer = terminal_type == "TimerFired";
22556 let mut resolution_payload = history[3].payload.clone();
22557 if uses_timer {
22558 for event in &mut history[1..] {
22559 event.payload["timeout_seconds"] = json!(3);
22560 }
22561 resolution_payload = history[3].payload.clone();
22562 resolution_payload["timer_kind"] = json!("condition_timeout");
22563 resolution_payload["timer_id"] = json!("condition-timeout-3");
22564 resolution_payload["delay_seconds"] = json!(3);
22565 history.push(history_event("TimerScheduled", resolution_payload.clone()));
22566 }
22567 let mut resolution = history_event(terminal_type, resolution_payload.clone());
22568 resolution
22569 .raw
22570 .insert("id".into(), json!("latest-condition-resolution"));
22571 history.push(resolution);
22572 history.push(history_event(
22573 "SelectionResolved",
22574 json!({
22575 "selection_group_id": "select-calls:1:2",
22576 "selection_group_base_sequence": 1,
22577 "selection_group_size": 2,
22578 "member_key": "votes",
22579 "member_index": 1,
22580 "member_base_sequence": 2,
22581 "member_size": 1,
22582 "operation_kind": "condition",
22583 "operation_identity": original_identity,
22584 "outcome": "completed",
22585 "resolution_event_id": "latest-condition-resolution",
22586 "resolution_event_type": terminal_type,
22587 }),
22588 ));
22589 if acknowledge_timeout {
22590 history.push(history_event("ConditionWaitTimedOut", resolution_payload));
22591 }
22592 let ctx = workflow_context(history);
22593 let mut selected = Box::pin(ctx.select_keyed(vec![
22594 ("timer", ParallelOperation::timer(Duration::from_secs(300))),
22595 (
22596 "votes",
22597 ParallelOperation::condition(
22598 if uses_timer {
22599 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
22600 .timeout(Duration::from_secs(3))
22601 } else {
22602 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
22603 },
22604 || Ok(true),
22605 ),
22606 ),
22607 ]));
22608 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22609 match selected.as_mut().poll(&mut task_context) {
22610 Poll::Ready(Ok(result)) => {
22611 assert_eq!(result.key, SelectionKey::Name("votes".into()));
22612 let expected = if terminal_type == "ConditionWaitSatisfied" {
22613 ConditionWaitResult::Satisfied
22614 } else {
22615 ConditionWaitResult::TimedOut
22616 };
22617 assert_eq!(result.value, Some(ParallelResult::Condition(expected)));
22618 }
22619 other => panic!("latest physical winner must replay: {other:?}"),
22620 }
22621 assert!(ctx.take_commands().unwrap().is_empty());
22622 ctx.ensure_history_consumed()
22623 .expect("winner history consumed");
22624 }
22625 }
22626
22627 fn reopened_selection_condition_history() -> Vec<HistoryEvent> {
22628 let fixture: Value = serde_json::from_str(include_str!(
22629 "../tests/fixtures/replay-regressions/grouped-condition-physical-reopen.json"
22630 ))
22631 .expect("grouped condition fixture");
22632 fixture["history"]
22633 .as_array()
22634 .unwrap()
22635 .iter()
22636 .map(|event| {
22637 let mut payload = event["payload"].clone();
22638 let index = payload["parallel_group_index"].as_u64().unwrap();
22639 payload["parallel_group_id"] = json!("select-calls:1:2");
22640 payload["parallel_group_mode"] = json!("select");
22641 payload["selection_member_key"] = json!(if index == 0 { "timer" } else { "votes" });
22642 payload["selection_member_index"] = json!(index);
22643 payload["selection_member_base_sequence"] = json!(index + 1);
22644 payload["selection_member_size"] = json!(1);
22645 payload["selection_member_kind"] =
22646 json!(if index == 0 { "timer" } else { "condition" });
22647 history_event(event["event_type"].as_str().unwrap(), payload)
22648 })
22649 .collect()
22650 }
22651
22652 #[test]
22653 fn grouped_condition_replay_rejects_unproven_physical_reopens() {
22654 let fixture: Value = serde_json::from_str(include_str!(
22655 "../tests/fixtures/replay-regressions/grouped-condition-physical-reopen.json"
22656 ))
22657 .expect("grouped condition fixture");
22658 let original = fixture["history"].as_array().expect("fixture history");
22659 let decode = |events: &[Value]| {
22660 let history = events
22661 .iter()
22662 .map(|event| {
22663 history_event(
22664 event["event_type"].as_str().unwrap(),
22665 event["payload"].clone(),
22666 )
22667 })
22668 .collect::<Vec<_>>();
22669 recorded_commands(
22670 &history,
22671 DEFAULT_CODEC,
22672 WorkflowIdentity {
22673 workflow_id: None,
22674 run_id: None,
22675 },
22676 )
22677 };
22678 assert!(decode(original).is_ok());
22679 for field in [
22680 "condition_wait_occurrence_id",
22681 "condition_key",
22682 "condition_definition_fingerprint",
22683 "timeout_seconds",
22684 "parallel_group_id",
22685 "parallel_group_size",
22686 "parallel_group_index",
22687 ] {
22688 let mut changed = original.clone();
22689 changed[3]["payload"][field] = match field {
22690 "timeout_seconds" | "parallel_group_size" => json!(30),
22691 "parallel_group_index" => json!(0),
22692 _ => json!("changed"),
22693 };
22694 assert!(
22695 decode(&changed).is_err(),
22696 "reopen must reject changed {field}"
22697 );
22698 }
22699 let mut unsettled = original.clone();
22700 unsettled.remove(2);
22701 assert!(
22702 decode(&unsettled).is_err(),
22703 "physical predecessor must be resolved"
22704 );
22705 let mut timed_out = original.clone();
22706 timed_out[2]["event_type"] = json!("ConditionWaitTimedOut");
22707 assert!(decode(&timed_out).is_err(), "timeout is terminal");
22708 let mut late_resolution = original.clone();
22709 late_resolution.swap(2, 3);
22710 assert!(
22711 decode(&late_resolution).is_err(),
22712 "later history cannot authorize a reopen"
22713 );
22714 }
22715
22716 #[test]
22717 fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
22718 let history = vec![
22719 history_event(
22720 "ConditionWaitOpened",
22721 json!({
22722 "sequence": 3,
22723 "condition_wait_id": "condition:3",
22724 "condition_wait_occurrence_id": "rust:condition-wait:0",
22725 "condition_key": "two-votes",
22726 "condition_definition_fingerprint": "sha256:two-votes-v1",
22727 }),
22728 ),
22729 history_event(
22730 "SignalReceived",
22731 json!({
22732 "workflow_sequence": 3,
22733 "signal_name": "vote",
22734 "arguments": fixture_envelope(json!(["first"])),
22735 }),
22736 ),
22737 history_event(
22738 "ConditionWaitSatisfied",
22739 json!({
22740 "sequence": 3,
22741 "condition_wait_id": "condition:3",
22742 "condition_wait_occurrence_id": "rust:condition-wait:0",
22743 "condition_key": "two-votes",
22744 "condition_definition_fingerprint": "sha256:two-votes-v1",
22745 }),
22746 ),
22747 history_event(
22748 "ConditionWaitOpened",
22749 json!({
22750 "sequence": 5,
22751 "condition_wait_id": "condition:5",
22752 "condition_wait_occurrence_id": "rust:condition-wait:0",
22753 "condition_key": "two-votes",
22754 "condition_definition_fingerprint": "sha256:two-votes-v1",
22755 }),
22756 ),
22757 history_event(
22758 "SignalReceived",
22759 json!({
22760 "workflow_sequence": 5,
22761 "signal_name": "vote",
22762 "arguments": fixture_envelope(json!(["second"])),
22763 }),
22764 ),
22765 history_event(
22766 "ConditionWaitSatisfied",
22767 json!({
22768 "sequence": 5,
22769 "condition_wait_id": "condition:5",
22770 "condition_wait_occurrence_id": "rust:condition-wait:0",
22771 "condition_key": "two-votes",
22772 "condition_definition_fingerprint": "sha256:two-votes-v1",
22773 }),
22774 ),
22775 ];
22776 for _cold_worker_or_restart in 0..2 {
22777 let ctx = workflow_context(history.clone());
22778 let predicate_ctx = ctx.clone();
22779 let mut wait = Box::pin(ctx.wait_condition(
22780 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
22781 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
22782 ));
22783 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22784
22785 assert!(matches!(
22786 wait.as_mut().poll(&mut task_context),
22787 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22788 ));
22789 assert!(ctx.take_commands().expect("commands").is_empty());
22790 ctx.ensure_history_consumed()
22791 .expect("every physical wait-open is consumed");
22792 }
22793 }
22794
22795 #[test]
22796 fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
22797 let history = vec![
22798 history_event(
22799 "ConditionWaitOpened",
22800 json!({
22801 "sequence": 3,
22802 "condition_wait_id": "condition:3",
22803 "condition_wait_occurrence_id": "rust:condition-wait:0",
22804 "condition_key": "approved",
22805 "condition_definition_fingerprint": "sha256:approved-v1",
22806 }),
22807 ),
22808 history_event(
22809 "UpdateApplied",
22810 json!({
22811 "sequence": 3,
22812 "update_id": "update-1",
22813 "update_name": "approve",
22814 "arguments": fixture_envelope(json!([false])),
22815 }),
22816 ),
22817 history_event(
22818 "ConditionWaitOpened",
22819 json!({
22820 "sequence": 5,
22821 "condition_wait_id": "condition:5",
22822 "condition_wait_occurrence_id": "rust:condition-wait:0",
22823 "condition_key": "approved",
22824 "condition_definition_fingerprint": "sha256:approved-v1",
22825 }),
22826 ),
22827 history_event(
22828 "UpdateApplied",
22829 json!({
22830 "sequence": 5,
22831 "update_id": "update-2",
22832 "update_name": "approve",
22833 "arguments": fixture_envelope(json!([true])),
22834 }),
22835 ),
22836 ];
22837
22838 for _cold_worker_or_restart in 0..2 {
22839 let ctx = workflow_context(history.clone());
22840 let predicate_ctx = ctx.clone();
22841 let mut wait = Box::pin(ctx.wait_condition(
22842 ConditionWaitOptions::new("approved", "sha256:approved-v1"),
22843 move || {
22844 Ok(predicate_ctx
22845 .updates("approve")?
22846 .last()
22847 .and_then(|arguments| arguments.first())
22848 .and_then(Value::as_bool)
22849 == Some(true))
22850 },
22851 ));
22852 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22853
22854 assert!(matches!(
22855 wait.as_mut().poll(&mut task_context),
22856 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22857 ));
22858 assert!(ctx.take_commands().expect("commands").is_empty());
22859 ctx.ensure_history_consumed()
22860 .expect("every update-driven reopen is consumed");
22861 }
22862 }
22863
22864 #[test]
22865 fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
22866 for (first_key, first_fingerprint, second_key, second_fingerprint) in [
22867 ("shared", "sha256:first", "shared", "sha256:second"),
22868 ("first", "sha256:shared", "second", "sha256:shared"),
22869 ("shared", "sha256:shared", "shared", "sha256:shared"),
22870 ("first", "sha256:first", "second", "sha256:second"),
22871 ] {
22872 let history = vec![
22873 history_event(
22874 "ConditionWaitOpened",
22875 json!({
22876 "sequence": 3,
22877 "condition_wait_id": "condition:3",
22878 "condition_wait_occurrence_id": "rust:condition-wait:0",
22879 "condition_key": first_key,
22880 "condition_definition_fingerprint": first_fingerprint,
22881 }),
22882 ),
22883 history_event(
22884 "ConditionWaitSatisfied",
22885 json!({
22886 "sequence": 3,
22887 "condition_wait_id": "condition:3",
22888 "condition_wait_occurrence_id": "rust:condition-wait:0",
22889 "condition_key": first_key,
22890 "condition_definition_fingerprint": first_fingerprint,
22891 }),
22892 ),
22893 history_event(
22894 "ConditionWaitOpened",
22895 json!({
22896 "sequence": 4,
22897 "condition_wait_id": "condition:4",
22898 "condition_wait_occurrence_id": "rust:condition-wait:1",
22899 "condition_key": second_key,
22900 "condition_definition_fingerprint": second_fingerprint,
22901 }),
22902 ),
22903 history_event(
22904 "ConditionWaitSatisfied",
22905 json!({
22906 "sequence": 4,
22907 "condition_wait_id": "condition:4",
22908 "condition_wait_occurrence_id": "rust:condition-wait:1",
22909 "condition_key": second_key,
22910 "condition_definition_fingerprint": second_fingerprint,
22911 }),
22912 ),
22913 ];
22914 for _cold_worker_or_restart in 0..2 {
22915 let ctx = workflow_context(history.clone());
22916 let mut task_context = TaskContext::from_waker(noop_waker_ref());
22917 let mut first = Box::pin(ctx.wait_condition(
22918 ConditionWaitOptions::new(first_key, first_fingerprint),
22919 || Ok(false),
22920 ));
22921 assert!(matches!(
22922 first.as_mut().poll(&mut task_context),
22923 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22924 ));
22925
22926 let mut second = Box::pin(ctx.wait_condition(
22927 ConditionWaitOptions::new(second_key, second_fingerprint),
22928 || Ok(false),
22929 ));
22930 assert!(matches!(
22931 second.as_mut().poll(&mut task_context),
22932 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
22933 ));
22934 assert!(ctx.take_commands().expect("commands").is_empty());
22935 ctx.ensure_history_consumed()
22936 .expect("each authored wait consumes one occurrence");
22937 }
22938 }
22939 }
22940
22941 #[test]
22942 fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
22943 fn worker() -> Worker {
22944 let client = Client::new("http://127.0.0.1:8080").expect("client");
22945 let mut worker = Worker::new(client, "rust-workers");
22946 worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
22947 let mut outcomes = Vec::new();
22948 for _ in 0..2 {
22949 outcomes.push(
22950 ctx.wait_condition(
22951 ConditionWaitOptions::new("shared", "sha256:shared"),
22952 || Ok(false),
22953 )
22954 .await?,
22955 );
22956 }
22957 Ok(json!(outcomes))
22958 });
22959 worker
22960 }
22961
22962 let task = workflow_task(
22963 "rust.condition-loop",
22964 vec![
22965 history_event(
22966 "ConditionWaitOpened",
22967 json!({
22968 "sequence": 1,
22969 "condition_wait_id": "condition:1",
22970 "condition_wait_occurrence_id": "rust:condition-wait:0",
22971 "condition_key": "shared",
22972 "condition_definition_fingerprint": "sha256:shared",
22973 }),
22974 ),
22975 history_event(
22976 "ConditionWaitSatisfied",
22977 json!({
22978 "sequence": 1,
22979 "condition_wait_id": "condition:1",
22980 "condition_wait_occurrence_id": "rust:condition-wait:0",
22981 "condition_key": "shared",
22982 "condition_definition_fingerprint": "sha256:shared",
22983 }),
22984 ),
22985 history_event(
22986 "ConditionWaitOpened",
22987 json!({
22988 "sequence": 2,
22989 "condition_wait_id": "condition:2",
22990 "condition_wait_occurrence_id": "rust:condition-wait:1",
22991 "condition_key": "shared",
22992 "condition_definition_fingerprint": "sha256:shared",
22993 }),
22994 ),
22995 history_event(
22996 "ConditionWaitSatisfied",
22997 json!({
22998 "sequence": 2,
22999 "condition_wait_id": "condition:2",
23000 "condition_wait_occurrence_id": "rust:condition-wait:1",
23001 "condition_key": "shared",
23002 "condition_definition_fingerprint": "sha256:shared",
23003 }),
23004 ),
23005 ],
23006 DEFAULT_CODEC,
23007 );
23008
23009 for _cold_worker_or_restart in 0..2 {
23010 let commands = worker()
23011 .execute_workflow_task(task.clone())
23012 .expect("adjacent loop waits replay deterministically");
23013 assert_eq!(commands.len(), 1);
23014 assert_eq!(commands[0]["type"], "complete_workflow");
23015 assert_eq!(
23016 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
23017 json!(["satisfied", "satisfied"])
23018 );
23019 }
23020 }
23021
23022 #[test]
23023 fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
23024 let history = vec![history_event(
23025 "ConditionWaitOpened",
23026 json!({
23027 "sequence": 12,
23028 "condition_wait_id": "condition:12",
23029 "condition_wait_occurrence_id": "rust:condition-wait:0",
23030 "condition_key": "approval",
23031 "condition_definition_fingerprint": "sha256:approval-v1",
23032 "timeout_seconds": 30,
23033 }),
23034 )];
23035 for (options, expected_reason) in [
23036 (
23037 ConditionWaitOptions::new("changed", "sha256:approval-v1")
23038 .timeout(Duration::from_secs(30)),
23039 "condition_wait_key_mismatch",
23040 ),
23041 (
23042 ConditionWaitOptions::new("approval", "sha256:approval-v2")
23043 .timeout(Duration::from_secs(30)),
23044 "condition_wait_predicate_mismatch",
23045 ),
23046 (
23047 ConditionWaitOptions::new("approval", "sha256:approval-v1")
23048 .timeout(Duration::from_secs(29)),
23049 "condition_wait_timeout_mismatch",
23050 ),
23051 ] {
23052 let ctx = workflow_context(history.clone());
23053 let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
23054 let mut task_context = TaskContext::from_waker(noop_waker_ref());
23055 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
23056 wait.as_mut().poll(&mut task_context)
23057 else {
23058 panic!("changed condition definition must fail replay");
23059 };
23060 assert_eq!(failure.reason, expected_reason);
23061 assert_eq!(failure.sequence, Some(12));
23062 }
23063 }
23064
23065 #[test]
23066 fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
23067 let error = WorkflowState::new(
23068 vec![history_event(
23069 "ConditionWaitOpened",
23070 json!({
23071 "sequence": 12,
23072 "condition_wait_id": "condition:12",
23073 "condition_wait_occurrence_id": "rust:condition-wait:0",
23074 "condition_key": "approval",
23075 }),
23076 )],
23077 "rust-workers".to_string(),
23078 DEFAULT_CODEC.to_string(),
23079 None,
23080 )
23081 .expect_err("condition history without a predicate fingerprint must fail");
23082
23083 assert!(matches!(
23084 error,
23085 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23086 if reason == "condition_wait_predicate_fingerprint_missing"
23087 ));
23088 }
23089
23090 #[test]
23091 fn condition_wait_history_requires_authored_occurrence_identity() {
23092 let error = WorkflowState::new(
23093 vec![history_event(
23094 "ConditionWaitOpened",
23095 json!({
23096 "sequence": 12,
23097 "condition_wait_id": "condition:12",
23098 "condition_key": "approval",
23099 "condition_definition_fingerprint": "sha256:approval-v1",
23100 }),
23101 )],
23102 "rust-workers".to_string(),
23103 DEFAULT_CODEC.to_string(),
23104 None,
23105 )
23106 .expect_err("condition history without occurrence identity must fail");
23107
23108 assert!(matches!(
23109 error,
23110 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23111 if reason == "condition_wait_occurrence_id_missing"
23112 ));
23113 }
23114
23115 #[test]
23116 fn typed_search_attribute_updates_validate_emit_and_replay() {
23117 let update = SearchAttributeUpdate::new()
23118 .keyword("OrderStatus", " waiting ")
23119 .expect("keyword")
23120 .int("Attempt", 3)
23121 .expect("int")
23122 .bool("Escalated", false)
23123 .expect("bool")
23124 .keyword_list("Regions", ["us-east", "eu-west"])
23125 .expect("list")
23126 .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
23127 .expect("datetime")
23128 .delete("LegacyStatus")
23129 .expect("delete");
23130 let ctx = workflow_context(Vec::new());
23131 ctx.upsert_search_attributes(update.clone())
23132 .expect("typed update");
23133 assert_eq!(
23134 ctx.take_commands().expect("search-attribute command"),
23135 vec![json!({
23136 "type": "upsert_search_attributes",
23137 "attributes": {
23138 "Attempt": 3,
23139 "Escalated": false,
23140 "LegacyStatus": null,
23141 "OrderStatus": "waiting",
23142 "Regions": ["us-east", "eu-west"],
23143 "UpdatedAt": "2026-08-22T04:00:00Z",
23144 },
23145 "attribute_types": {
23146 "Attempt": "int",
23147 "Escalated": "bool",
23148 "OrderStatus": "keyword",
23149 "Regions": "keyword_list",
23150 "UpdatedAt": "datetime",
23151 },
23152 })]
23153 );
23154
23155 let replay = workflow_context(vec![history_event(
23156 "SearchAttributesUpserted",
23157 json!({
23158 "sequence": 6,
23159 "attributes": {
23160 "Attempt": 3,
23161 "Escalated": false,
23162 "LegacyStatus": null,
23163 "OrderStatus": "waiting",
23164 "Regions": ["us-east", "eu-west"],
23165 "UpdatedAt": "2026-08-22T04:00:00Z",
23166 },
23167 "attribute_types": {
23168 "Attempt": "int",
23169 "Escalated": "bool",
23170 "OrderStatus": "keyword",
23171 "Regions": "keyword_list",
23172 "UpdatedAt": "datetime",
23173 },
23174 "merged": {},
23175 }),
23176 )]);
23177 replay
23178 .upsert_search_attributes(update)
23179 .expect("matching update replays");
23180 assert!(replay.take_commands().expect("commands").is_empty());
23181 replay.ensure_history_consumed().expect("history consumed");
23182
23183 let type_drift = workflow_context(vec![history_event(
23184 "SearchAttributesUpserted",
23185 json!({
23186 "sequence": 7,
23187 "attributes": {"OrderStatus": "waiting"},
23188 "attribute_types": {"OrderStatus": "keyword"},
23189 "merged": {"OrderStatus": "waiting"},
23190 }),
23191 )]);
23192 let error = type_drift
23193 .upsert_search_attributes(
23194 SearchAttributeUpdate::new()
23195 .string("OrderStatus", "waiting")
23196 .expect("string update"),
23197 )
23198 .expect_err("same JSON value with a changed type must fail replay");
23199 assert!(matches!(
23200 error,
23201 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23202 if reason == "search_attribute_type_mismatch"
23203 ));
23204
23205 let malformed_types = WorkflowState::new(
23206 vec![history_event(
23207 "SearchAttributesUpserted",
23208 json!({
23209 "sequence": 8,
23210 "attributes": {"OrderStatus": "waiting"},
23211 "attribute_types": {"OrderStatus": "unsupported"},
23212 "merged": {"OrderStatus": "waiting"},
23213 }),
23214 )],
23215 "rust-workers".to_string(),
23216 DEFAULT_CODEC.to_string(),
23217 None,
23218 )
23219 .expect_err("unsupported search-attribute type metadata must fail");
23220 assert!(matches!(
23221 malformed_types,
23222 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23223 if reason == "search_attribute_types_malformed"
23224 ));
23225
23226 assert!(matches!(
23227 SearchAttributeUpdate::new().keyword("bad key", "value"),
23228 Err(SearchAttributeUpdateError::InvalidKey(_))
23229 ));
23230 assert!(matches!(
23231 SearchAttributeUpdate::new().float("Ratio", f64::NAN),
23232 Err(SearchAttributeUpdateError::NonFiniteFloat(_))
23233 ));
23234 assert!(matches!(
23235 SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
23236 Err(SearchAttributeUpdateError::ValueTooLong { .. })
23237 ));
23238 assert!(matches!(
23239 SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
23240 Err(SearchAttributeUpdateError::InvalidDateTime(_))
23241 ));
23242 assert!(matches!(
23243 workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
23244 Err(Error::InvalidSearchAttributeUpdate(
23245 SearchAttributeUpdateError::Empty
23246 ))
23247 ));
23248 }
23249
23250 #[test]
23251 fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
23252 let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
23253 let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
23254
23255 assert!(SearchAttributeUpdate::new()
23256 .string("AsciiDescription", ascii)
23257 .is_ok());
23258 assert!(SearchAttributeUpdate::new()
23259 .string("Utf8Description", utf8)
23260 .is_ok());
23261 assert!(matches!(
23262 SearchAttributeUpdate::new().string(
23263 "TooLongDescription",
23264 "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
23265 ),
23266 Err(SearchAttributeUpdateError::ValueTooLong {
23267 kind: "string",
23268 limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
23269 ..
23270 })
23271 ));
23272 }
23273
23274 #[test]
23275 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
23276 let lone_fire = WorkflowState::new(
23277 vec![history_event(
23278 "TimerFired",
23279 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23280 )],
23281 "rust-workers".to_string(),
23282 DEFAULT_CODEC.to_string(),
23283 None,
23284 )
23285 .expect_err("TimerFired requires TimerScheduled");
23286 assert!(matches!(
23287 lone_fire,
23288 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23289 if reason == "timer_schedule_missing_or_duplicate"
23290 ));
23291
23292 let wrong_identity = WorkflowState::new(
23293 vec![
23294 history_event(
23295 "TimerScheduled",
23296 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23297 ),
23298 history_event(
23299 "TimerFired",
23300 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
23301 ),
23302 ],
23303 "rust-workers".to_string(),
23304 DEFAULT_CODEC.to_string(),
23305 None,
23306 )
23307 .expect_err("fire must match scheduled timer identity");
23308 assert!(matches!(
23309 wrong_identity,
23310 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23311 if reason == "timer_identity_mismatch"
23312 ));
23313
23314 let duplicate_fire = WorkflowState::new(
23315 vec![
23316 history_event(
23317 "TimerScheduled",
23318 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23319 ),
23320 history_event(
23321 "TimerFired",
23322 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23323 ),
23324 history_event(
23325 "TimerFired",
23326 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23327 ),
23328 ],
23329 "rust-workers".to_string(),
23330 DEFAULT_CODEC.to_string(),
23331 None,
23332 )
23333 .expect_err("a durable timer cannot fire twice");
23334 assert!(matches!(
23335 duplicate_fire,
23336 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23337 if reason == "duplicate_timer_fire"
23338 ));
23339
23340 let wrong_fired_delay = WorkflowState::new(
23341 vec![
23342 history_event(
23343 "TimerScheduled",
23344 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23345 ),
23346 history_event(
23347 "TimerFired",
23348 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
23349 ),
23350 ],
23351 "rust-workers".to_string(),
23352 DEFAULT_CODEC.to_string(),
23353 None,
23354 )
23355 .expect_err("timer schedule and fire delays must agree");
23356 assert!(matches!(
23357 wrong_fired_delay,
23358 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
23359 if reason == "timer_history_delay_mismatch"
23360 ));
23361 }
23362
23363 #[test]
23364 fn replay_rejects_activity_moved_before_recorded_timer() {
23365 let ctx = workflow_context(vec![
23366 history_event(
23367 "TimerScheduled",
23368 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23369 ),
23370 history_event(
23371 "TimerFired",
23372 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
23373 ),
23374 history_event(
23375 "ActivityCompleted",
23376 json!({
23377 "sequence": 2,
23378 "activity_type": "after-timer",
23379 "payload_codec": DEFAULT_CODEC,
23380 "result": fixture_envelope(json!("done")),
23381 }),
23382 ),
23383 ]);
23384 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
23385 let mut task_context = TaskContext::from_waker(noop_waker_ref());
23386
23387 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
23388 activity.as_mut().poll(&mut task_context)
23389 else {
23390 panic!("reordered durable command must be rejected");
23391 };
23392 assert_eq!(failure.reason, "recorded_command_mismatch");
23393 assert_eq!(failure.sequence, Some(1));
23394 assert_eq!(failure.expected.as_deref(), Some("timer"));
23395 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
23396 }
23397
23398 #[test]
23399 fn workflow_context_emits_a_typed_named_signal_wait() {
23400 let ctx = workflow_context(Vec::new());
23401 let mut signal = Box::pin(ctx.wait_signal("finish"));
23402 let mut task_context = TaskContext::from_waker(noop_waker_ref());
23403
23404 assert!(matches!(
23405 signal.as_mut().poll(&mut task_context),
23406 Poll::Pending
23407 ));
23408 assert_eq!(
23409 ctx.take_commands().expect("signal-wait command"),
23410 vec![json!({
23411 "type": "open_signal_wait",
23412 "signal_name": "finish",
23413 })]
23414 );
23415 }
23416
23417 #[test]
23418 fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
23419 let ctx = workflow_context(Vec::new());
23420 let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
23421 let mut task_context = TaskContext::from_waker(noop_waker_ref());
23422
23423 let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
23424 else {
23425 panic!("runtime-reserved signal should be rejected");
23426 };
23427 assert!(message.contains("reserved by the workflow runtime"));
23428 assert!(ctx.take_commands().expect("commands").is_empty());
23429 }
23430
23431 #[tokio::test]
23432 async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
23433 let client = Client::builder("http://127.0.0.1:9")
23434 .build()
23435 .expect("client");
23436 let error = client
23437 .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
23438 .await
23439 .expect_err("runtime-reserved signal should be rejected before transport");
23440
23441 assert!(
23442 matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
23443 );
23444 }
23445
23446 #[test]
23447 fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
23448 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
23449 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
23450 value.to_string(),
23451 )]))
23452 .expect("message payload");
23453 json!({
23454 "schema": MESSAGE_STREAM_SCHEMA,
23455 "stream_name": "orders",
23456 "message_id": message_id,
23457 "position": position,
23458 "payload_envelope": payload,
23459 })
23460 }
23461
23462 fn opened(sequence: u64) -> HistoryEvent {
23463 history_event(
23464 "SignalWaitOpened",
23465 json!({
23466 "sequence": sequence,
23467 "signal_name": MESSAGE_STREAM_SIGNAL,
23468 }),
23469 )
23470 }
23471
23472 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
23473 history_event(
23474 "SignalApplied",
23475 json!({
23476 "sequence": sequence,
23477 "signal_name": MESSAGE_STREAM_SIGNAL,
23478 "value": fixture_envelope(json!([delivery])),
23479 }),
23480 )
23481 }
23482
23483 fn received(delivery: Value) -> HistoryEvent {
23484 history_event(
23485 "SignalReceived",
23486 json!({
23487 "signal_name": MESSAGE_STREAM_SIGNAL,
23488 "arguments": fixture_envelope(json!([delivery])),
23489 "payload_codec": DEFAULT_CODEC,
23490 }),
23491 )
23492 }
23493
23494 let client = Client::new("http://127.0.0.1:8080").expect("client");
23495 let mut worker = Worker::new(client, "rust-workers");
23496 worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
23497 let messages = ctx.message_stream("orders")?.receive(2).await?;
23498 Ok(json!(messages
23499 .into_iter()
23500 .map(|message| message.message_id)
23501 .collect::<Vec<_>>()))
23502 });
23503
23504 let first = delivery("message-1", 1, "one");
23505 let second = delivery("message-2", 2, "two");
23506 let batch = worker
23507 .execute_workflow_task_decision(workflow_task(
23508 "rust.message-stream-batch",
23509 vec![
23510 opened(1),
23511 received(first.clone()),
23512 applied(1, first.clone()),
23513 received(first.clone()),
23514 received(second),
23515 ],
23516 DEFAULT_CODEC,
23517 ))
23518 .expect("worker task consumes the available batch");
23519
23520 assert_eq!(batch.commands.len(), 1);
23521 assert_eq!(batch.commands[0]["type"], "complete_workflow");
23522 assert_eq!(
23523 decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
23524 .expect("workflow result"),
23525 json!(["message-1", "message-2"])
23526 );
23527 assert_eq!(
23528 batch.message_stream_cursors,
23529 vec![json!({"stream_name": "orders", "through_position": 2})]
23530 );
23531 assert!(batch.message_stream_waits.is_empty());
23532
23533 let partial = worker
23534 .execute_workflow_task_decision(workflow_task(
23535 "rust.message-stream-batch",
23536 vec![opened(1), received(first.clone()), applied(1, first)],
23537 DEFAULT_CODEC,
23538 ))
23539 .expect("worker task returns without waiting for a missing second item");
23540 assert_eq!(partial.commands.len(), 1);
23541 assert_eq!(partial.commands[0]["type"], "complete_workflow");
23542 assert_eq!(
23543 decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
23544 .expect("workflow result"),
23545 json!(["message-1"])
23546 );
23547 assert_eq!(
23548 partial.message_stream_cursors,
23549 vec![json!({"stream_name": "orders", "through_position": 1})]
23550 );
23551 assert!(partial.message_stream_waits.is_empty());
23552 }
23553
23554 #[test]
23555 fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
23556 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
23557 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
23558 value.to_string(),
23559 )]))
23560 .expect("message payload");
23561 json!({
23562 "schema": MESSAGE_STREAM_SCHEMA,
23563 "stream_name": "orders",
23564 "message_id": message_id,
23565 "position": position,
23566 "payload_envelope": payload,
23567 })
23568 }
23569
23570 fn opened(sequence: u64) -> HistoryEvent {
23571 history_event(
23572 "SignalWaitOpened",
23573 json!({
23574 "sequence": sequence,
23575 "signal_name": MESSAGE_STREAM_SIGNAL,
23576 }),
23577 )
23578 }
23579
23580 fn received(delivery: Value) -> HistoryEvent {
23581 history_event(
23582 "SignalReceived",
23583 json!({
23584 "signal_name": MESSAGE_STREAM_SIGNAL,
23585 "arguments": fixture_envelope(json!([delivery])),
23586 "payload_codec": DEFAULT_CODEC,
23587 }),
23588 )
23589 }
23590
23591 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
23592 history_event(
23593 "SignalApplied",
23594 json!({
23595 "sequence": sequence,
23596 "signal_name": MESSAGE_STREAM_SIGNAL,
23597 "value": fixture_envelope(json!([delivery])),
23598 }),
23599 )
23600 }
23601
23602 let client = Client::new("http://127.0.0.1:8080").expect("client");
23603 let mut worker = Worker::new(client, "rust-workers");
23604 worker.register_workflow(
23605 "rust.message-stream-partial-batches",
23606 |ctx, _input| async move {
23607 let stream = ctx.message_stream("orders")?;
23608 let first = stream.receive(10).await?;
23609 let second = stream.receive(10).await?;
23610 Ok(json!([
23611 first
23612 .into_iter()
23613 .map(|message| message.message_id)
23614 .collect::<Vec<_>>(),
23615 second
23616 .into_iter()
23617 .map(|message| message.message_id)
23618 .collect::<Vec<_>>(),
23619 ]))
23620 },
23621 );
23622
23623 let first = delivery("message-1", 1, "one");
23624 let second = delivery("message-2", 2, "two");
23625 let decision = worker
23626 .execute_workflow_task_decision(workflow_task(
23627 "rust.message-stream-partial-batches",
23628 vec![
23629 opened(1),
23630 received(first.clone()),
23631 applied(1, first),
23632 opened(2),
23633 received(second.clone()),
23634 applied(2, second),
23635 ],
23636 DEFAULT_CODEC,
23637 ))
23638 .expect("cold replay preserves both authored receive boundaries");
23639
23640 assert_eq!(decision.commands.len(), 1);
23641 assert_eq!(decision.commands[0]["type"], "complete_workflow");
23642 assert_eq!(
23643 decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
23644 .expect("workflow result"),
23645 json!([["message-1"], ["message-2"]])
23646 );
23647 assert_eq!(
23648 decision.message_stream_cursors,
23649 vec![json!({"stream_name": "orders", "through_position": 2})]
23650 );
23651 assert!(decision.message_stream_waits.is_empty());
23652 }
23653
23654 #[test]
23655 fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
23656 let ctx = workflow_context(Vec::new());
23657 let stream = ctx.message_stream("orders").expect("message stream");
23658 let mut receive = Box::pin(stream.receive(10));
23659 let mut task_context = TaskContext::from_waker(noop_waker_ref());
23660
23661 assert!(matches!(
23662 receive.as_mut().poll(&mut task_context),
23663 Poll::Pending
23664 ));
23665 assert_eq!(
23666 ctx.take_commands().expect("message-stream wait command"),
23667 vec![json!({
23668 "type": "open_signal_wait",
23669 "signal_name": MESSAGE_STREAM_SIGNAL,
23670 })]
23671 );
23672 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
23673 assert!(cursors.is_empty());
23674 assert_eq!(
23675 waits,
23676 vec![json!({"stream_name": "orders", "after_position": 0})]
23677 );
23678 }
23679
23680 #[test]
23681 fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
23682 let ctx = workflow_context(vec![history_event(
23683 "SignalReceived",
23684 json!({
23685 "signal_name": MESSAGE_STREAM_SIGNAL,
23686 "arguments": fixture_envelope(json!([{
23687 "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
23688 "stream_name": "orders",
23689 "through_position": 2,
23690 }])),
23691 "payload_codec": DEFAULT_CODEC,
23692 }),
23693 )]);
23694 let stream = ctx.message_stream("orders").expect("message stream");
23695 let mut receive = Box::pin(stream.receive(10));
23696 let mut task_context = TaskContext::from_waker(noop_waker_ref());
23697
23698 assert!(matches!(
23699 receive.as_mut().poll(&mut task_context),
23700 Poll::Pending
23701 ));
23702 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
23703 assert_eq!(
23704 cursors,
23705 vec![json!({"stream_name": "orders", "through_position": 2})]
23706 );
23707 assert_eq!(
23708 waits,
23709 vec![json!({"stream_name": "orders", "after_position": 2})]
23710 );
23711 }
23712
23713 #[test]
23714 fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
23715 let mut empty_map = BTreeMap::new();
23716 let mut nested = BTreeMap::new();
23717 nested.insert(
23718 "value".to_string(),
23719 AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
23720 );
23721 let values = vec![
23722 AvroValue::Bytes(vec![0, 255]),
23723 AvroValue::Long(1),
23724 AvroValue::Double(1.0),
23725 AvroValue::Array(Vec::new()),
23726 AvroValue::Map(std::mem::take(&mut empty_map)),
23727 AvroValue::Map(nested),
23728 ];
23729 let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
23730 let transport = vec![json!({
23731 "schema": MESSAGE_STREAM_SCHEMA,
23732 "stream_name": "orders",
23733 "message_id": "message-1",
23734 "position": 1,
23735 "payload_envelope": payload,
23736 })];
23737
23738 for _ in 0..2 {
23739 let Some(MessageStreamDelivery::Message(message)) =
23740 decode_message_stream_delivery(transport.clone()).expect("delivery")
23741 else {
23742 panic!("message delivery expected");
23743 };
23744 assert_eq!(message.arguments, values);
23745 assert!(matches!(message.arguments[1], AvroValue::Long(1)));
23746 assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
23747 }
23748 }
23749
23750 #[test]
23751 fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
23752 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
23753 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
23754 value.to_string(),
23755 )]))
23756 .expect("message payload");
23757 json!({
23758 "schema": MESSAGE_STREAM_SCHEMA,
23759 "stream_name": "orders",
23760 "message_id": message_id,
23761 "position": position,
23762 "payload_envelope": payload,
23763 })
23764 }
23765
23766 fn opened(sequence: u64) -> HistoryEvent {
23767 history_event(
23768 "SignalWaitOpened",
23769 json!({
23770 "sequence": sequence,
23771 "signal_name": MESSAGE_STREAM_SIGNAL,
23772 }),
23773 )
23774 }
23775
23776 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
23777 history_event(
23778 "SignalApplied",
23779 json!({
23780 "sequence": sequence,
23781 "signal_name": MESSAGE_STREAM_SIGNAL,
23782 "value": fixture_envelope(json!([delivery])),
23783 }),
23784 )
23785 }
23786
23787 fn worker() -> Worker {
23788 let client = Client::new("http://127.0.0.1:8080").expect("client");
23789 let mut worker = Worker::new(client, "rust-workers");
23790 worker.register_workflow("rust.message-stream", |ctx, _input| async move {
23791 let stream = ctx.message_stream("orders")?;
23792 let first = stream.receive_one().await?;
23793 let second = stream.receive_one().await?;
23794 Ok(json!([first.message_id, second.message_id]))
23795 });
23796 worker
23797 }
23798
23799 fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
23800 let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
23801 task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
23802 task.signal_arguments = Some(fixture_envelope(json!([delivery])));
23803 task
23804 }
23805
23806 let waiting = worker()
23807 .execute_workflow_task_decision(workflow_task(
23808 "rust.message-stream",
23809 Vec::new(),
23810 DEFAULT_CODEC,
23811 ))
23812 .expect("first worker opens the stream wait");
23813 assert_eq!(
23814 waiting.commands,
23815 vec![json!({
23816 "type": "open_signal_wait",
23817 "signal_name": MESSAGE_STREAM_SIGNAL,
23818 })]
23819 );
23820 assert!(waiting.message_stream_cursors.is_empty());
23821 assert_eq!(
23822 waiting.message_stream_waits,
23823 vec![json!({"stream_name": "orders", "after_position": 0})]
23824 );
23825
23826 let first_delivery = delivery("message-1", 1, "one");
23827 let first_arrival = worker()
23828 .execute_workflow_task_decision(task_with_resume(
23829 vec![opened(1)],
23830 first_delivery.clone(),
23831 ))
23832 .expect("replacement worker consumes the first arrival");
23833 assert_eq!(
23834 first_arrival.commands,
23835 vec![json!({
23836 "type": "open_signal_wait",
23837 "signal_name": MESSAGE_STREAM_SIGNAL,
23838 })]
23839 );
23840 assert_eq!(
23841 first_arrival.message_stream_cursors,
23842 vec![json!({"stream_name": "orders", "through_position": 1})]
23843 );
23844 assert_eq!(
23845 first_arrival.message_stream_waits,
23846 vec![json!({"stream_name": "orders", "after_position": 1})]
23847 );
23848
23849 let second_delivery = delivery("message-2", 2, "two");
23850 let first_applied = applied(1, first_delivery);
23851 let completed = worker()
23852 .execute_workflow_task_decision(task_with_resume(
23853 vec![opened(1), first_applied.clone(), opened(2)],
23854 second_delivery.clone(),
23855 ))
23856 .expect("next replacement worker consumes the second arrival");
23857 assert_eq!(completed.commands.len(), 1);
23858 assert_eq!(completed.commands[0]["type"], "complete_workflow");
23859 assert_eq!(
23860 decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
23861 .expect("workflow result"),
23862 json!(["message-1", "message-2"])
23863 );
23864 assert_eq!(
23865 completed.message_stream_cursors,
23866 vec![json!({"stream_name": "orders", "through_position": 2})]
23867 );
23868 assert!(completed.message_stream_waits.is_empty());
23869
23870 let replay_history = vec![
23871 opened(1),
23872 first_applied,
23873 opened(2),
23874 applied(2, second_delivery),
23875 ];
23876 for _cold_worker_or_restart in 0..2 {
23877 let replayed = worker()
23878 .execute_workflow_task_decision(workflow_task(
23879 "rust.message-stream",
23880 replay_history.clone(),
23881 DEFAULT_CODEC,
23882 ))
23883 .expect("cold worker replays each logical message exactly once");
23884 assert_eq!(replayed.commands.len(), 1);
23885 assert_eq!(
23886 decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
23887 .expect("replayed workflow result"),
23888 json!(["message-1", "message-2"])
23889 );
23890 assert_eq!(
23891 replayed.message_stream_cursors,
23892 vec![json!({"stream_name": "orders", "through_position": 2})]
23893 );
23894 assert!(replayed.message_stream_waits.is_empty());
23895 }
23896 }
23897
23898 #[test]
23899 fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
23900 assert!(!worker_protocol_supports_message_streams("1.14"));
23901 assert!(worker_protocol_supports_message_streams("1.15"));
23902 assert!(worker_protocol_supports_message_streams("1.16"));
23903 assert!(worker_protocol_supports_message_streams(
23904 WORKER_PROTOCOL_VERSION
23905 ));
23906 assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
23907 }
23908
23909 #[test]
23910 fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
23911 let ctx = workflow_context(vec![
23912 history_event(
23913 "ConditionWaitOpened",
23914 json!({
23915 "sequence": 1,
23916 "condition_wait_id": "condition:1",
23917 "condition_wait_occurrence_id": "rust:condition-wait:0",
23918 "condition_key": "signal:finish",
23919 "condition_definition_fingerprint": "sha256:signal-finish-v1",
23920 }),
23921 ),
23922 history_event(
23923 "ConditionWaitSatisfied",
23924 json!({
23925 "sequence": 1,
23926 "condition_wait_id": "condition:1",
23927 "condition_wait_occurrence_id": "rust:condition-wait:0",
23928 "condition_key": "signal:finish",
23929 "condition_definition_fingerprint": "sha256:signal-finish-v1",
23930 }),
23931 ),
23932 history_event(
23933 "SignalReceived",
23934 json!({"signal_name": "finish", "arguments": []}),
23935 ),
23936 ]);
23937 let mut signal = Box::pin(ctx.wait_signal("finish"));
23938 let mut task_context = TaskContext::from_waker(noop_waker_ref());
23939
23940 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
23941 signal.as_mut().poll(&mut task_context)
23942 else {
23943 panic!("condition history must not resolve as a typed signal wait");
23944 };
23945 assert_eq!(failure.reason, "recorded_command_mismatch");
23946 assert_eq!(failure.expected.as_deref(), Some("condition wait"));
23947 }
23948
23949 #[test]
23950 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
23951 let signal_then_timer = vec![
23952 history_event(
23953 "SignalWaitOpened",
23954 json!({"sequence": 1, "signal_name": "go"}),
23955 ),
23956 history_event(
23957 "SignalApplied",
23958 json!({
23959 "sequence": 1,
23960 "signal_name": "go",
23961 "value": fixture_envelope(json!(["now"])),
23962 }),
23963 ),
23964 history_event(
23965 "TimerScheduled",
23966 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
23967 ),
23968 history_event(
23969 "TimerFired",
23970 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
23971 ),
23972 ];
23973
23974 let ctx = workflow_context(signal_then_timer.clone());
23975 let mut signal = Box::pin(ctx.wait_signal("go"));
23976 let mut task_context = TaskContext::from_waker(noop_waker_ref());
23977 assert!(matches!(
23978 signal.as_mut().poll(&mut task_context),
23979 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
23980 ));
23981 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
23982 assert!(matches!(
23983 timer.as_mut().poll(&mut task_context),
23984 Poll::Ready(Ok(()))
23985 ));
23986 ctx.ensure_history_consumed()
23987 .expect("signal and timer history consumed in order");
23988
23989 let reordered = workflow_context(signal_then_timer);
23990 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
23991 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
23992 timer_first.as_mut().poll(&mut task_context)
23993 else {
23994 panic!("timer cannot consume signal-wait-first history");
23995 };
23996 assert_eq!(failure.reason, "recorded_command_mismatch");
23997 assert_eq!(failure.sequence, Some(1));
23998 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
23999
24000 let timer_then_signal = vec![
24001 history_event(
24002 "TimerScheduled",
24003 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24004 ),
24005 history_event(
24006 "TimerFired",
24007 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24008 ),
24009 history_event(
24010 "SignalWaitOpened",
24011 json!({"sequence": 2, "signal_name": "go"}),
24012 ),
24013 history_event(
24014 "SignalApplied",
24015 json!({
24016 "sequence": 2,
24017 "signal_name": "go",
24018 "value": fixture_envelope(json!([])),
24019 }),
24020 ),
24021 ];
24022 let reordered = workflow_context(timer_then_signal);
24023 let mut signal_first = Box::pin(reordered.wait_signal("go"));
24024 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
24025 signal_first.as_mut().poll(&mut task_context)
24026 else {
24027 panic!("signal wait cannot consume timer-first history");
24028 };
24029 assert_eq!(failure.reason, "recorded_command_mismatch");
24030 assert_eq!(failure.sequence, Some(1));
24031 assert_eq!(failure.expected.as_deref(), Some("timer"));
24032 }
24033
24034 #[test]
24035 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
24036 let duplicate_timer = WorkflowState::new(
24037 vec![
24038 history_event(
24039 "TimerScheduled",
24040 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24041 ),
24042 history_event(
24043 "TimerScheduled",
24044 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
24045 ),
24046 ],
24047 "rust-workers".to_string(),
24048 DEFAULT_CODEC.to_string(),
24049 None,
24050 )
24051 .expect_err("one workflow sequence cannot schedule two timers");
24052 assert!(matches!(
24053 duplicate_timer,
24054 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
24055 if reason == "timer_schedule_missing_or_duplicate"
24056 ));
24057
24058 let colliding_kinds = WorkflowState::new(
24059 vec![
24060 history_event(
24061 "TimerScheduled",
24062 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24063 ),
24064 history_event(
24065 "ActivityCompleted",
24066 json!({"sequence": 1, "activity_type": "same-sequence"}),
24067 ),
24068 ],
24069 "rust-workers".to_string(),
24070 DEFAULT_CODEC.to_string(),
24071 None,
24072 )
24073 .expect_err("one workflow sequence cannot identify two command kinds");
24074 assert!(matches!(
24075 colliding_kinds,
24076 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
24077 if reason == "durable_command_sequence_collision"
24078 ));
24079
24080 let duplicate_signal_wait = WorkflowState::new(
24081 vec![
24082 history_event(
24083 "SignalWaitOpened",
24084 json!({"sequence": 1, "signal_name": "go"}),
24085 ),
24086 history_event(
24087 "SignalWaitOpened",
24088 json!({"sequence": 1, "signal_name": "go"}),
24089 ),
24090 ],
24091 "rust-workers".to_string(),
24092 DEFAULT_CODEC.to_string(),
24093 None,
24094 )
24095 .expect_err("one workflow sequence cannot open two signal waits");
24096 assert!(matches!(
24097 duplicate_signal_wait,
24098 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
24099 if reason == "signal_wait_open_missing_or_duplicate"
24100 ));
24101 }
24102
24103 #[test]
24104 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
24105 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
24106 .expect("side-effect result");
24107 let ctx = workflow_context(vec![history_event(
24108 "SideEffectRecorded",
24109 json!({"sequence": 99, "result": result}),
24110 )]);
24111
24112 let replayed: Value = ctx
24113 .side_effect(|| panic!("recorded side effect must not run"))
24114 .expect("positive global workflow sequence is valid");
24115 assert_eq!(replayed, json!({"captured": true}));
24116 ctx.ensure_history_consumed().expect("history consumed");
24117 }
24118
24119 #[test]
24120 fn workflow_history_rejects_zero_and_descending_command_sequences() {
24121 let result =
24122 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
24123 let zero = WorkflowState::new(
24124 vec![history_event(
24125 "SideEffectRecorded",
24126 json!({"sequence": 0, "result": result.clone()}),
24127 )],
24128 "rust-workers".to_string(),
24129 DEFAULT_CODEC.to_string(),
24130 None,
24131 )
24132 .expect_err("durable command sequences must be positive");
24133 assert!(matches!(
24134 zero,
24135 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
24136 if reason == "durable_command_sequence_invalid"
24137 ));
24138
24139 let descending = WorkflowState::new(
24140 vec![
24141 history_event(
24142 "SideEffectRecorded",
24143 json!({"sequence": 3, "result": result}),
24144 ),
24145 history_event(
24146 "VersionMarkerRecorded",
24147 json!({
24148 "sequence": 2,
24149 "change_id": "descending-marker",
24150 "version": 1,
24151 "min_supported": 1,
24152 "max_supported": 1,
24153 }),
24154 ),
24155 ],
24156 "rust-workers".to_string(),
24157 DEFAULT_CODEC.to_string(),
24158 None,
24159 )
24160 .expect_err("new durable commands must remain strictly ordered");
24161 let Error::NonDeterministicReplay(failure) = descending else {
24162 panic!("expected typed replay failure");
24163 };
24164 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
24165 assert_eq!(failure.sequence, Some(2));
24166 assert_eq!(
24167 failure.expected.as_deref(),
24168 Some("workflow sequence greater than 3")
24169 );
24170 assert_eq!(failure.actual.as_deref(), Some("2"));
24171 }
24172
24173 #[test]
24174 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
24175 fn worker() -> Worker {
24176 let client = Client::new("http://127.0.0.1:8080").expect("client");
24177 let mut worker = Worker::new(client, "rust-workers");
24178 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
24179 ctx.wait_signal("finish").await?;
24180 let marker: String =
24181 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
24182 assert_eq!(marker, "after-finish");
24183 Ok(json!("finished"))
24184 });
24185 worker
24186 }
24187
24188 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
24189 .expect("side-effect result");
24190 let task = workflow_task(
24191 "rust.finish-after-gaps",
24192 vec![
24193 history_event(
24194 "SignalWaitOpened",
24195 json!({"sequence": 1, "signal_name": "finish"}),
24196 ),
24197 history_event(
24198 "SignalReceived",
24199 json!({
24200 "signal_id": "increment-3",
24201 "signal_name": "increment",
24202 "workflow_sequence": 2,
24203 "payload_codec": DEFAULT_CODEC,
24204 "arguments": fixture_envelope(json!([3])),
24205 }),
24206 ),
24207 history_event(
24208 "SignalReceived",
24209 json!({
24210 "signal_id": "increment-5",
24211 "signal_name": "increment",
24212 "workflow_sequence": 3,
24213 "payload_codec": DEFAULT_CODEC,
24214 "arguments": fixture_envelope(json!([5])),
24215 }),
24216 ),
24217 history_event(
24218 "SignalReceived",
24219 json!({
24220 "signal_id": "finish",
24221 "signal_name": "finish",
24222 "workflow_sequence": 4,
24223 "payload_codec": DEFAULT_CODEC,
24224 "arguments": fixture_envelope(json!([])),
24225 }),
24226 ),
24227 history_event(
24228 "SignalApplied",
24229 json!({
24230 "sequence": 1,
24231 "signal_id": "finish",
24232 "signal_name": "finish",
24233 "payload_codec": DEFAULT_CODEC,
24234 "value": fixture_envelope(json!([])),
24235 }),
24236 ),
24237 history_event(
24238 "SideEffectRecorded",
24239 json!({"sequence": 5, "result": marker}),
24240 ),
24241 ],
24242 DEFAULT_CODEC,
24243 );
24244
24245 for _original_or_cold_worker in 0..2 {
24246 let commands = worker()
24247 .execute_workflow_task(task.clone())
24248 .expect("signal gaps preserve deterministic replay");
24249 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
24250 assert_eq!(commands[0]["type"], "complete_workflow");
24251 assert_eq!(
24252 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
24253 json!("finished")
24254 );
24255 }
24256 }
24257
24258 #[test]
24259 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
24260 let ctx = workflow_context(Vec::new());
24261 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
24262 let mut task_context = TaskContext::from_waker(noop_waker_ref());
24263 assert!(matches!(
24264 sleep.as_mut().poll(&mut task_context),
24265 Poll::Ready(Err(Error::TimerDurationOverflow))
24266 ));
24267 assert!(ctx.take_commands().expect("commands").is_empty());
24268 }
24269
24270 #[test]
24271 fn workflow_memo_update_emits_canonical_command_and_replays_once() {
24272 let entries = AvroValue::Map(BTreeMap::from([
24273 ("text".to_string(), AvroValue::String("same".to_string())),
24274 (
24275 "nested".to_string(),
24276 AvroValue::Map(BTreeMap::from([
24277 ("beta".to_string(), AvroValue::Long(2)),
24278 ("alpha".to_string(), AvroValue::Long(1)),
24279 ])),
24280 ),
24281 ("long".to_string(), AvroValue::Long(7)),
24282 ("double".to_string(), AvroValue::Double(7.0)),
24283 ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
24284 ]));
24285 let ctx = workflow_context(Vec::new());
24286 ctx.upsert_memo(entries.clone()).expect("valid memo update");
24287 let commands = ctx.take_commands().expect("commands");
24288
24289 assert_eq!(commands.len(), 1);
24290 assert_eq!(commands[0]["type"], "upsert_memo");
24291 let server_entries = json!({
24292 "codec": "avro",
24293 "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
24294 });
24295 assert_eq!(
24296 commands[0]["entries"]
24297 .as_object()
24298 .expect("entries envelope")
24299 .keys()
24300 .collect::<Vec<_>>(),
24301 vec!["blob", "codec"]
24302 );
24303 assert_eq!(commands[0]["entries"], server_entries);
24304 let wire_entries =
24305 decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
24306 assert_eq!(wire_entries, entries);
24307
24308 let history = vec![history_event(
24309 "MemoUpserted",
24310 json!({
24311 "sequence": 1,
24312 "entries": server_entries.clone(),
24313 "merged": server_entries,
24314 }),
24315 )];
24316 let replay = workflow_context(history.clone());
24317 replay
24318 .upsert_memo(entries.clone())
24319 .expect("matching replay identity");
24320 assert!(replay.take_commands().expect("replay commands").is_empty());
24321
24322 let changed_types = AvroValue::Map(BTreeMap::from([
24323 ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
24324 (
24325 "nested".to_string(),
24326 AvroValue::Map(BTreeMap::from([
24327 ("alpha".to_string(), AvroValue::Long(1)),
24328 ("beta".to_string(), AvroValue::Long(2)),
24329 ])),
24330 ),
24331 ("long".to_string(), AvroValue::Double(7.0)),
24332 ("double".to_string(), AvroValue::Long(7)),
24333 ("binary".to_string(), AvroValue::String("same".to_string())),
24334 ]));
24335 let error = workflow_context(history)
24336 .upsert_memo(changed_types)
24337 .expect_err("memo replay identity must preserve Avro value types");
24338 assert!(matches!(
24339 error,
24340 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
24341 ));
24342 }
24343
24344 #[test]
24345 fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
24346 let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
24347 .expect("memo envelope");
24348 let replay = workflow_context(vec![history_event(
24349 "MemoUpserted",
24350 json!({
24351 "sequence": 1,
24352 "entries": original.clone(),
24353 "merged": original
24354 }),
24355 )]);
24356 let error = replay
24357 .upsert_memo(json!({"stage": "changed"}))
24358 .expect_err("changed memo update must fail replay");
24359 assert!(matches!(
24360 error,
24361 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
24362 ));
24363
24364 let invalid = workflow_context(Vec::new())
24365 .upsert_memo(
24366 json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
24367 )
24368 .expect_err("oversized key");
24369 assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
24370 }
24371
24372 #[test]
24373 fn workflow_memo_replay_distinguishes_signed_zero_identity() {
24374 let negative_zero = AvroValue::Map(BTreeMap::from([(
24375 "reading".to_string(),
24376 AvroValue::Double(-0.0),
24377 )]));
24378 let negative_zero_envelope =
24379 encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
24380 let history = vec![history_event(
24381 "MemoUpserted",
24382 json!({
24383 "sequence": 1,
24384 "entries": negative_zero_envelope.clone(),
24385 "merged": negative_zero_envelope,
24386 }),
24387 )];
24388
24389 workflow_context(history.clone())
24390 .upsert_memo(negative_zero)
24391 .expect("matching negative-zero history identity");
24392
24393 let error = workflow_context(history)
24394 .upsert_memo(AvroValue::Map(BTreeMap::from([(
24395 "reading".to_string(),
24396 AvroValue::Double(0.0),
24397 )])))
24398 .expect_err("positive zero must not consume negative-zero memo history");
24399 assert!(matches!(
24400 error,
24401 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
24402 ));
24403 }
24404
24405 #[test]
24406 fn workflow_memo_capability_requires_flag_and_command_advertisement() {
24407 let supported = json!({
24408 "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
24409 "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
24410 });
24411 assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
24412 assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
24413 "workflow_memo_updates": {"supported": false},
24414 "supported_workflow_task_commands": ["upsert_memo"]
24415 }))));
24416 assert!(commands_use_workflow_memo_updates(&[json!({
24417 "type": "upsert_memo",
24418 "entries": {"stage": "processing"}
24419 })]));
24420 }
24421
24422 #[test]
24423 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
24424 let client = Client::new("http://127.0.0.1:8080").expect("client");
24425 let mut worker = Worker::new(client, "rust-workers");
24426 worker.register_workflow("rust.timer", |ctx, _input| async move {
24427 ctx.sleep(Duration::from_secs(5)).await?;
24428 ctx.activity("after-timer", json!([])).await
24429 });
24430
24431 let task = |history_events| WorkflowTask {
24432 task_id: "wft-rust-timer-1".to_string(),
24433 workflow_command_id: None,
24434 workflow_id: Some("wf-rust-timer".to_string()),
24435 run_id: Some("run-rust-timer".to_string()),
24436 workflow_type: "rust.timer".to_string(),
24437 cancel_requested: false,
24438 payload_codec: DEFAULT_CODEC.to_string(),
24439 arguments: Some(
24440 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
24441 ),
24442 history_events,
24443 total_history_events: None,
24444 history_size_bytes: None,
24445 continue_as_new_recommended: None,
24446 history_budget_pressure: None,
24447 next_history_page_token: None,
24448 workflow_task_attempt: 1,
24449 workflow_signal_id: None,
24450 signal_name: None,
24451 signal_arguments: None,
24452 workflow_update_id: None,
24453 update_name: None,
24454 lease_owner: Some("rust-worker".to_string()),
24455 };
24456
24457 let initial = worker
24458 .execute_workflow_task(task(Vec::new()))
24459 .expect("initial timer task");
24460 assert_eq!(
24461 initial,
24462 vec![json!({"type": "start_timer", "delay_seconds": 5})]
24463 );
24464
24465 let activity_result =
24466 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
24467 let replayed = worker
24468 .execute_workflow_task(task(vec![
24469 history_event(
24470 "TimerScheduled",
24471 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24472 ),
24473 history_event(
24474 "TimerFired",
24475 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24476 ),
24477 history_event(
24478 "ActivityCompleted",
24479 json!({
24480 "sequence": 2,
24481 "activity_type": "after-timer",
24482 "payload_codec": DEFAULT_CODEC,
24483 "result": activity_result,
24484 }),
24485 ),
24486 ]))
24487 .expect("replayed workflow task");
24488 assert_eq!(replayed.len(), 1);
24489 assert_eq!(replayed[0]["type"], "complete_workflow");
24490 assert_eq!(
24491 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
24492 json!("done")
24493 );
24494 }
24495
24496 #[test]
24497 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
24498 let client = Client::new("http://127.0.0.1:8080").expect("client");
24499 let mut worker = Worker::new(client, "rust-workers");
24500 worker.register_workflow("rust.continue", |ctx, _input| async move {
24501 ctx.continue_as_new_with_options(
24502 ContinueAsNewOptions::new()
24503 .workflow_type("rust.next")
24504 .task_queue("next-workers"),
24505 json!([2, {"cursor": "next"}]),
24506 )
24507 });
24508
24509 let commands = worker
24510 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
24511 .expect("continue-as-new command");
24512
24513 assert_eq!(commands.len(), 1);
24514 assert_eq!(commands[0]["type"], "continue_as_new");
24515 assert_eq!(commands[0]["workflow_type"], "rust.next");
24516 assert_eq!(commands[0]["queue"], "next-workers");
24517 assert_eq!(
24518 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
24519 .expect("continue-as-new arguments"),
24520 json!([2, {"cursor": "next"}])
24521 );
24522 }
24523
24524 #[test]
24525 fn continue_as_new_preserves_typed_arguments() {
24526 let client = Client::new("http://127.0.0.1:8080").expect("client");
24527 let mut worker = Worker::new(client, "rust-workers");
24528 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
24529 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
24530 unreachable!("continue-as-new returns a control-flow error")
24531 });
24532
24533 let commands = worker
24534 .execute_workflow_task(workflow_task(
24535 "rust.typed-continue",
24536 Vec::new(),
24537 DEFAULT_CODEC,
24538 ))
24539 .expect("typed continue-as-new command");
24540
24541 assert_eq!(commands[0]["type"], "continue_as_new");
24542 assert_eq!(
24543 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
24544 .expect("typed continue arguments"),
24545 AvroValue::Array(vec![typed_fidelity_probe()])
24546 );
24547 }
24548
24549 #[test]
24550 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
24551 let client = Client::new("http://127.0.0.1:8080").expect("client");
24552 let mut worker = Worker::new(client, "rust-workers");
24553 worker.register_workflow("rust.continue", |ctx, _input| async move {
24554 ctx.continue_as_new(json!([2]))
24555 });
24556 let task = workflow_task(
24557 "rust.continue",
24558 vec![history_event(
24559 "WorkflowContinuedAsNew",
24560 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
24561 )],
24562 DEFAULT_CODEC,
24563 );
24564
24565 for _worker_restart_or_redelivery in 0..2 {
24566 let commands = worker
24567 .execute_workflow_task(task.clone())
24568 .expect("recorded transition replays");
24569 assert!(
24570 commands.is_empty(),
24571 "replay must not emit another successor"
24572 );
24573 }
24574 }
24575
24576 #[test]
24577 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
24578 let ctx = workflow_context(Vec::new());
24579 let error = ctx
24580 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
24581 .expect_err("blank queue must be rejected");
24582
24583 let Error::InvalidContinueAsNewOptions(error) = error else {
24584 panic!("expected typed continue-as-new validation error");
24585 };
24586 assert_eq!(error.field, "task_queue");
24587 assert!(ctx.take_commands().expect("commands").is_empty());
24588 }
24589
24590 #[test]
24591 fn workflow_context_exposes_server_history_budget() {
24592 let client = Client::new("http://127.0.0.1:8080").expect("client");
24593 let mut worker = Worker::new(client, "rust-workers");
24594 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
24595 let budget = ctx.history_budget()?;
24596 Ok(json!({
24597 "events": budget.event_count,
24598 "bytes": budget.size_bytes,
24599 "recommended": budget.continue_as_new_recommended,
24600 "pressure": budget.pressure,
24601 }))
24602 });
24603 let task: WorkflowTask = serde_json::from_value(json!({
24604 "task_id": "task-history-budget",
24605 "workflow_type": "rust.history-budget",
24606 "payload_codec": DEFAULT_CODEC,
24607 "history_events": [],
24608 "total_history_events": 480,
24609 "history_size_bytes": 1_048_576,
24610 "continue_as_new_recommended": true,
24611 "history_budget_pressure": "continue_as_new_recommended",
24612 }))
24613 .expect("published workflow task");
24614
24615 let commands = worker
24616 .execute_workflow_task(task)
24617 .expect("history-budget workflow");
24618 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
24619 assert_eq!(result["events"], 480);
24620 assert_eq!(result["bytes"], 1_048_576);
24621 assert_eq!(result["recommended"], true);
24622 assert_eq!(result["pressure"], "continue_as_new_recommended");
24623 }
24624
24625 #[test]
24626 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
24627 let client = Client::new("http://127.0.0.1:8080").expect("client");
24628 let mut worker = Worker::new(client, "rust-workers");
24629 worker.register_workflow("rust.failing", |_ctx, _input| async move {
24630 Err(Error::Codec("rust_conformance_failure".to_string()))
24631 });
24632 let task = WorkflowTask {
24633 task_id: "wft-rust-failing-1".to_string(),
24634 workflow_command_id: None,
24635 workflow_id: Some("wf-rust-failing".to_string()),
24636 run_id: Some("run-rust-failing".to_string()),
24637 workflow_type: "rust.failing".to_string(),
24638 cancel_requested: false,
24639 payload_codec: DEFAULT_CODEC.to_string(),
24640 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
24641 history_events: Vec::new(),
24642 total_history_events: Some(0),
24643 history_size_bytes: None,
24644 continue_as_new_recommended: None,
24645 history_budget_pressure: None,
24646 next_history_page_token: None,
24647 workflow_task_attempt: 1,
24648 workflow_signal_id: None,
24649 signal_name: None,
24650 signal_arguments: None,
24651 workflow_update_id: None,
24652 update_name: None,
24653 lease_owner: Some("rust-worker".to_string()),
24654 };
24655
24656 let commands = worker
24657 .execute_workflow_task(task)
24658 .expect("handler failure becomes a workflow command");
24659
24660 assert_eq!(commands.len(), 1);
24661 assert_eq!(commands[0]["type"], "fail_workflow");
24662 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
24663 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
24664 assert_eq!(commands[0]["non_retryable"], false);
24665 assert_eq!(
24666 commands[0]["message"],
24667 "codec error: rust_conformance_failure"
24668 );
24669 assert_eq!(
24670 commands[0]["exception"]["message"],
24671 "codec error: rust_conformance_failure"
24672 );
24673 }
24674
24675 #[test]
24676 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
24677 let client = Client::new("http://127.0.0.1:8080").expect("client");
24678 let mut worker = Worker::new(client, "rust-workers");
24679 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
24680 let _: String = ctx.side_effect(|| "captured".to_string())?;
24681 Err(Error::WorkerLoop("application failure".to_string()))
24682 });
24683
24684 let commands = worker
24685 .execute_workflow_task(workflow_task(
24686 "rust.failing-after-side-effect",
24687 Vec::new(),
24688 DEFAULT_CODEC,
24689 ))
24690 .expect("ordinary failure remains a workflow decision");
24691
24692 assert_eq!(commands.len(), 2);
24693 assert_eq!(commands[0]["type"], "record_side_effect");
24694 assert_eq!(commands[1]["type"], "fail_workflow");
24695 }
24696
24697 #[test]
24698 fn uncaught_recorded_activity_failure_claims_only_its_persisted_boundary() {
24699 let client = Client::new("http://127.0.0.1:8080").expect("client");
24700 let mut worker = Worker::new(client, "rust-workers");
24701 worker.register_workflow("rust.redrive", |ctx, _input| async move {
24702 ctx.activity("greet", json!([])).await?;
24703 Ok(Value::Null)
24704 });
24705 worker.register_workflow("rust.redrive-translated", |ctx, _input| async move {
24706 match ctx.activity("greet", json!([])).await {
24707 Err(Error::ActivityFailed(_)) => Err(Error::WorkerLoop("translated".to_string())),
24708 result => result,
24709 }
24710 });
24711
24712 let failed = history_event(
24713 "ActivityFailed",
24714 json!({
24715 "sequence": 1,
24716 "activity_type": "greet",
24717 "activity_execution_id": "activity-1",
24718 "message": "failed"
24719 }),
24720 );
24721 let commands = worker
24722 .execute_workflow_task(workflow_task(
24723 "rust.redrive",
24724 vec![failed.clone()],
24725 DEFAULT_CODEC,
24726 ))
24727 .expect("recorded failure becomes a workflow command");
24728 assert_eq!(commands[0]["type"], "fail_workflow");
24729 assert_eq!(commands[0]["failed_step_sequence"], 1);
24730 assert_eq!(commands[0]["failed_activity_execution_id"], "activity-1");
24731
24732 let translated = worker
24733 .execute_workflow_task(workflow_task(
24734 "rust.redrive-translated",
24735 vec![failed],
24736 DEFAULT_CODEC,
24737 ))
24738 .expect("translated failure becomes a workflow command");
24739 assert_eq!(translated[0]["type"], "fail_workflow");
24740 assert!(translated[0].get("failed_step_sequence").is_none());
24741 assert!(translated[0].get("failed_activity_execution_id").is_none());
24742
24743 let without_identity = worker
24744 .execute_workflow_task(workflow_task(
24745 "rust.redrive",
24746 vec![history_event(
24747 "ActivityFailed",
24748 json!({"sequence": 1, "activity_type": "greet", "message": "failed"}),
24749 )],
24750 DEFAULT_CODEC,
24751 ))
24752 .expect("incomplete identity still fails the workflow");
24753 assert!(without_identity[0].get("failed_step_sequence").is_none());
24754
24755 let timed_out = worker
24756 .execute_workflow_task(workflow_task(
24757 "rust.redrive",
24758 vec![history_event(
24759 "ActivityTimedOut",
24760 json!({
24761 "sequence": 1,
24762 "activity_type": "greet",
24763 "activity_execution_id": "activity-timeout",
24764 }),
24765 )],
24766 DEFAULT_CODEC,
24767 ))
24768 .expect("timeout still fails the workflow");
24769 assert!(timed_out[0].get("failed_step_sequence").is_none());
24770 }
24771
24772 #[test]
24773 fn redriven_history_reuses_completed_prefix_and_reschedules_only_failed_step() {
24774 let client = Client::new("http://127.0.0.1:8080").expect("client");
24775 let mut worker = Worker::new(client, "rust-workers");
24776 worker.register_workflow("rust.redrive-successor", |ctx, _input| async move {
24777 let first = ctx.activity("first", json!([])).await?;
24778 let second = ctx.activity("second", json!([first.clone()])).await?;
24779 Ok(json!({"first": first, "second": second}))
24780 });
24781 let first = history_event(
24782 "ActivityCompleted",
24783 json!({
24784 "sequence": 1,
24785 "activity_type": "first",
24786 "result": encode_value_envelope(&json!("recorded"), DEFAULT_CODEC).expect("first result"),
24787 "payload_codec": DEFAULT_CODEC,
24788 "reused_from_run_id": "failed-run",
24789 "reused_activity_execution_id": "original-first",
24790 }),
24791 );
24792
24793 let retry = worker
24794 .execute_workflow_task(workflow_task(
24795 "rust.redrive-successor",
24796 vec![first.clone()],
24797 DEFAULT_CODEC,
24798 ))
24799 .expect("reused result replays before failed step");
24800 assert_eq!(retry.len(), 1);
24801 assert_eq!(retry[0]["type"], "schedule_activity");
24802 assert_eq!(retry[0]["activity_type"], "second");
24803 assert_eq!(
24804 decode_wire_value(&retry[0]["arguments"], DEFAULT_CODEC).expect("second arguments"),
24805 json!(["recorded"]),
24806 );
24807
24808 let second = history_event(
24809 "ActivityCompleted",
24810 json!({
24811 "sequence": 2,
24812 "activity_type": "second",
24813 "result": encode_value_envelope(&json!("retried"), DEFAULT_CODEC).expect("second result"),
24814 "payload_codec": DEFAULT_CODEC,
24815 }),
24816 );
24817 let completed = worker
24818 .execute_workflow_task(workflow_task(
24819 "rust.redrive-successor",
24820 vec![first, second],
24821 DEFAULT_CODEC,
24822 ))
24823 .expect("retried step completes the successor");
24824 assert_eq!(completed.len(), 1);
24825 assert_eq!(completed[0]["type"], "complete_workflow");
24826 assert_eq!(
24827 decode_wire_value(&completed[0]["result"], DEFAULT_CODEC).expect("workflow result"),
24828 json!({"first": "recorded", "second": "retried"}),
24829 );
24830 }
24831
24832 #[test]
24833 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
24834 let client = Client::new("http://127.0.0.1:8080").expect("client");
24835 let mut worker = Worker::new(client, "rust-workers");
24836 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
24837 Err(Error::WorkerLoop("application failure".to_string()))
24838 });
24839 let result =
24840 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
24841
24842 let error = worker
24843 .execute_workflow_task(workflow_task(
24844 "rust.removed-side-effect",
24845 vec![history_event(
24846 "SideEffectRecorded",
24847 json!({"sequence": 1, "result": result}),
24848 )],
24849 DEFAULT_CODEC,
24850 ))
24851 .expect_err("removed committed history must not become fail_workflow");
24852
24853 let Error::NonDeterministicReplay(failure) = error else {
24854 panic!("expected typed replay failure");
24855 };
24856 assert_eq!(failure.reason, "recorded_commands_unconsumed");
24857 assert_eq!(failure.sequence, Some(1));
24858 assert_eq!(failure.expected.as_deref(), Some("side effect"));
24859 }
24860
24861 #[test]
24862 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
24863 let client = Client::new("http://127.0.0.1:8080").expect("client");
24864 let mut worker = Worker::new(client, "rust-workers");
24865 worker.register_workflow(
24866 "rust.side-effect-before-marker-error",
24867 |ctx, _input| async move {
24868 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
24869 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
24870 ctx.get_version("restart-safe", 2, 2)?;
24871 Ok(Value::Null)
24872 },
24873 );
24874
24875 let error = worker
24876 .execute_workflow_task(workflow_task(
24877 "rust.side-effect-before-marker-error",
24878 vec![history_event(
24879 "VersionMarkerRecorded",
24880 json!({
24881 "sequence": 1,
24882 "change_id": "restart-safe",
24883 "version": 1,
24884 "min_supported": 1,
24885 "max_supported": 1,
24886 }),
24887 )],
24888 DEFAULT_CODEC,
24889 ))
24890 .expect_err("replay error must return no queued workflow commands");
24891
24892 let Error::NonDeterministicReplay(failure) = error else {
24893 panic!("expected typed replay failure");
24894 };
24895 assert_eq!(failure.reason, "version_marker_incompatible_range");
24896 assert_eq!(failure.sequence, Some(1));
24897 }
24898
24899 #[test]
24900 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
24901 let client = Client::new("http://127.0.0.1:8080").expect("client");
24902 let mut worker = Worker::new(client, "rust-workers");
24903 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
24904 ctx.sleep(Duration::from_secs(5)).await?;
24905 Ok(json!({"status": "timer fired"}))
24906 });
24907
24908 let task = WorkflowTask {
24909 task_id: "wft-rust-timer-pending".to_string(),
24910 workflow_command_id: None,
24911 workflow_id: Some("wf-rust-timer".to_string()),
24912 run_id: Some("run-rust-timer".to_string()),
24913 workflow_type: "rust.timer.pending".to_string(),
24914 cancel_requested: false,
24915 payload_codec: DEFAULT_CODEC.to_string(),
24916 arguments: Some(
24917 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
24918 ),
24919 history_events: vec![history_event(
24920 "TimerScheduled",
24921 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24922 )],
24923 total_history_events: Some(1),
24924 history_size_bytes: None,
24925 continue_as_new_recommended: None,
24926 history_budget_pressure: None,
24927 next_history_page_token: None,
24928 workflow_task_attempt: 1,
24929 workflow_signal_id: None,
24930 signal_name: None,
24931 signal_arguments: None,
24932 workflow_update_id: None,
24933 update_name: None,
24934 lease_owner: Some("rust-worker".to_string()),
24935 };
24936
24937 for _redelivery_or_restart in 0..2 {
24938 let commands = worker
24939 .execute_workflow_task(task.clone())
24940 .expect("recorded timer remains pending");
24941 assert!(
24942 commands.is_empty(),
24943 "recorded timer must not be rescheduled"
24944 );
24945 }
24946 }
24947
24948 #[test]
24949 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
24950 let client = Client::new("http://127.0.0.1:8080").expect("client");
24951 let mut worker = Worker::new(client, "rust-workers");
24952 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
24953 Ok(json!({"status": "completed"}))
24954 });
24955 let task = WorkflowTask {
24956 task_id: "wft-rust-timer-removed".to_string(),
24957 workflow_command_id: None,
24958 workflow_id: Some("wf-rust-timer".to_string()),
24959 run_id: Some("run-rust-timer".to_string()),
24960 workflow_type: "rust.timer.removed".to_string(),
24961 cancel_requested: false,
24962 payload_codec: DEFAULT_CODEC.to_string(),
24963 arguments: Some(
24964 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
24965 ),
24966 history_events: vec![
24967 history_event(
24968 "TimerScheduled",
24969 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24970 ),
24971 history_event(
24972 "TimerFired",
24973 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
24974 ),
24975 ],
24976 total_history_events: Some(2),
24977 history_size_bytes: None,
24978 continue_as_new_recommended: None,
24979 history_budget_pressure: None,
24980 next_history_page_token: None,
24981 workflow_task_attempt: 1,
24982 workflow_signal_id: None,
24983 signal_name: None,
24984 signal_arguments: None,
24985 workflow_update_id: None,
24986 update_name: None,
24987 lease_owner: Some("rust-worker".to_string()),
24988 };
24989
24990 let Error::NonDeterministicReplay(failure) = worker
24991 .execute_workflow_task(task)
24992 .expect_err("removed timer must fail replay")
24993 else {
24994 panic!("expected typed replay failure");
24995 };
24996 assert_eq!(failure.reason, "recorded_commands_unconsumed");
24997 assert_eq!(failure.sequence, Some(1));
24998 }
24999
25000 #[test]
25001 fn workflow_context_emits_explicit_child_workflow_contract() {
25002 let ctx = WorkflowContext {
25003 cancellation_scope_id: "root".into(),
25004 state: Arc::new(Mutex::new(
25005 WorkflowState::new_with_identity(
25006 Vec::new(),
25007 Some("wf-parent".to_string()),
25008 Some("run-parent".to_string()),
25009 "parent-workers".to_string(),
25010 DEFAULT_CODEC.to_string(),
25011 None,
25012 )
25013 .expect("workflow state"),
25014 )),
25015 };
25016 let options = ChildWorkflowOptions::new("python-workers")
25017 .parent_close_policy(ParentClosePolicy::RequestCancel)
25018 .retry_policy(ChildWorkflowRetryPolicy {
25019 max_attempts: Some(3),
25020 backoff_seconds: vec![1, 5],
25021 non_retryable_error_types: vec!["ValidationError".to_string()],
25022 })
25023 .execution_timeout_seconds(600)
25024 .run_timeout_seconds(120);
25025 let mut call = Box::pin(ctx.start_child_workflow(
25026 "python.fulfil-order",
25027 options,
25028 json!([{"order_id": "order-42"}]),
25029 ));
25030 let mut task_context = TaskContext::from_waker(noop_waker_ref());
25031
25032 assert!(matches!(
25033 call.as_mut().poll(&mut task_context),
25034 Poll::Pending
25035 ));
25036 let commands = ctx.take_commands().expect("commands");
25037 assert_eq!(commands.len(), 1);
25038 let command = &commands[0];
25039 assert_eq!(command["type"], "start_child_workflow");
25040 assert_eq!(command["workflow_type"], "python.fulfil-order");
25041 assert_eq!(command["queue"], "python-workers");
25042 assert_eq!(command["parent_close_policy"], "request_cancel");
25043 assert_eq!(command["retry_policy"]["max_attempts"], 3);
25044 assert_eq!(command["execution_timeout_seconds"], 600);
25045 assert_eq!(command["run_timeout_seconds"], 120);
25046 assert_eq!(
25047 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
25048 json!([{"order_id": "order-42"}])
25049 );
25050 }
25051
25052 fn child_parent_worker() -> Worker {
25053 let client = Client::new("http://127.0.0.1:8080").expect("client");
25054 let mut worker = Worker::new(client, "rust-parent-workers");
25055 worker.register_workflow("rust.parent", |ctx, _input| async move {
25056 let child = ctx
25057 .start_child_workflow(
25058 "python.child",
25059 ChildWorkflowOptions::new("python-child-workers")
25060 .parent_close_policy(ParentClosePolicy::Terminate),
25061 json!([{"codec_probe": [1, true, "rust"]}]),
25062 )
25063 .await?;
25064 Ok(json!({
25065 "parent_workflow_id": child.parent.workflow_id,
25066 "parent_run_id": child.parent.run_id,
25067 "child_workflow_id": child.child.workflow_id,
25068 "child_run_id": child.child.run_id,
25069 "child_workflow_type": child.child_workflow_type,
25070 "result": child.result,
25071 }))
25072 });
25073 worker
25074 }
25075
25076 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
25077 WorkflowTask {
25078 task_id: "wft-child-parent".to_string(),
25079 workflow_command_id: None,
25080 workflow_id: Some("wf-parent".to_string()),
25081 run_id: Some("run-parent".to_string()),
25082 workflow_type: "rust.parent".to_string(),
25083 cancel_requested: false,
25084 payload_codec: DEFAULT_CODEC.to_string(),
25085 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
25086 history_events: vec![
25087 HistoryEvent {
25088 event_type: "ChildWorkflowScheduled".to_string(),
25089 payload: json!({
25090 "sequence": 1,
25091 "child_call_id": "call-child",
25092 "child_workflow_instance_id": "wf-child",
25093 "child_workflow_run_id": "run-child",
25094 "child_workflow_type": "python.child",
25095 "parent_close_policy": "terminate",
25096 }),
25097 raw: HashMap::new(),
25098 },
25099 HistoryEvent {
25100 event_type: event_type.to_string(),
25101 payload,
25102 raw: HashMap::new(),
25103 },
25104 ],
25105 total_history_events: Some(2),
25106 history_size_bytes: None,
25107 continue_as_new_recommended: None,
25108 history_budget_pressure: None,
25109 next_history_page_token: None,
25110 workflow_task_attempt: 1,
25111 workflow_signal_id: None,
25112 signal_name: None,
25113 signal_arguments: None,
25114 workflow_update_id: None,
25115 update_name: None,
25116 lease_owner: Some("rust-worker".to_string()),
25117 }
25118 }
25119
25120 #[test]
25121 fn committed_child_result_replays_without_starting_a_duplicate() {
25122 let worker = child_parent_worker();
25123 let task = child_parent_task(
25124 "ChildRunCompleted",
25125 json!({
25126 "sequence": 1,
25127 "child_call_id": "call-child",
25128 "child_workflow_instance_id": "wf-child",
25129 "child_workflow_run_id": "run-child",
25130 "child_workflow_type": "python.child",
25131 "payload_codec": DEFAULT_CODEC,
25132 "result": fixture_envelope(json!({"from":"python","ok":true})),
25133 }),
25134 );
25135
25136 for _restart in 0..2 {
25137 let commands = worker
25138 .execute_workflow_task(task.clone())
25139 .expect("replayed parent task");
25140 assert_eq!(commands.len(), 1);
25141 assert_eq!(commands[0]["type"], "complete_workflow");
25142 assert!(!commands
25143 .iter()
25144 .any(|command| command["type"] == "start_child_workflow"));
25145 let output =
25146 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
25147 assert_eq!(output["parent_workflow_id"], "wf-parent");
25148 assert_eq!(output["parent_run_id"], "run-parent");
25149 assert_eq!(output["child_workflow_id"], "wf-child");
25150 assert_eq!(output["child_run_id"], "run-child");
25151 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
25152 }
25153 }
25154
25155 #[test]
25156 fn typed_child_arguments_and_results_survive_replay() {
25157 let client = Client::new("http://127.0.0.1:8080").expect("client");
25158 let mut worker = Worker::new(client, "rust-parent-workers");
25159 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
25160 let child = ctx
25161 .start_child_workflow_avro_value(
25162 "python.typed-child",
25163 ChildWorkflowOptions::new("python-workers"),
25164 AvroValue::Array(vec![typed_fidelity_probe()]),
25165 )
25166 .await?;
25167 Ok(child.result)
25168 });
25169
25170 let initial = worker
25171 .execute_workflow_task(workflow_task(
25172 "rust.typed-parent",
25173 Vec::new(),
25174 DEFAULT_CODEC,
25175 ))
25176 .expect("typed child start");
25177 assert_eq!(initial[0]["type"], "start_child_workflow");
25178 assert_eq!(
25179 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
25180 .expect("typed child arguments"),
25181 AvroValue::Array(vec![typed_fidelity_probe()])
25182 );
25183
25184 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25185 .expect("typed child result");
25186 let task = workflow_task(
25187 "rust.typed-parent",
25188 vec![
25189 history_event(
25190 "ChildWorkflowScheduled",
25191 json!({
25192 "sequence": 1,
25193 "child_call_id": "call-typed",
25194 "child_workflow_instance_id": "wf-child",
25195 "child_workflow_run_id": "run-child",
25196 "child_workflow_type": "python.typed-child",
25197 }),
25198 ),
25199 history_event(
25200 "ChildRunCompleted",
25201 json!({
25202 "sequence": 1,
25203 "child_call_id": "call-typed",
25204 "child_workflow_instance_id": "wf-child",
25205 "child_workflow_run_id": "run-child",
25206 "child_workflow_type": "python.typed-child",
25207 "payload_codec": DEFAULT_CODEC,
25208 "result": result,
25209 }),
25210 ),
25211 ],
25212 DEFAULT_CODEC,
25213 );
25214
25215 let commands = worker
25216 .execute_workflow_task(task)
25217 .expect("typed child replay");
25218 assert_eq!(commands[0]["type"], "complete_workflow");
25219 assert_eq!(
25220 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
25221 .expect("typed parent result"),
25222 typed_fidelity_probe()
25223 );
25224 }
25225
25226 #[test]
25227 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
25228 let worker = child_parent_worker();
25229 let mut task = child_parent_task("unused", Value::Null);
25230 task.history_events.truncate(1);
25231 task.total_history_events = Some(1);
25232
25233 for _redelivery_or_restart in 0..2 {
25234 let commands = worker
25235 .execute_workflow_task(task.clone())
25236 .expect("recorded child remains pending");
25237 assert!(
25238 commands.is_empty(),
25239 "recorded pending child must not be started again"
25240 );
25241 }
25242 }
25243
25244 #[test]
25245 fn child_cancellation_becomes_stable_parent_failure_command() {
25246 let worker = child_parent_worker();
25247 let task = child_parent_task(
25248 "ChildRunCancelled",
25249 json!({
25250 "sequence": 1,
25251 "child_workflow_instance_id": "wf-child",
25252 "child_workflow_run_id": "run-child",
25253 "child_workflow_type": "python.child",
25254 "failure_id": "failure-child",
25255 "failure_category": "cancelled",
25256 "message": "cancelled by parent-close policy",
25257 }),
25258 );
25259
25260 let commands = worker
25261 .execute_workflow_task(task)
25262 .expect("parent settlement");
25263 assert_eq!(commands.len(), 1);
25264 assert_eq!(commands[0]["type"], "fail_workflow");
25265 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
25266 assert_eq!(
25267 commands[0]["exception"]["properties"]["reason"],
25268 "cancelled"
25269 );
25270 assert_eq!(
25271 commands[0]["exception"]["properties"]["child_workflow_run_id"],
25272 "run-child"
25273 );
25274 }
25275
25276 #[test]
25277 fn workflow_can_handle_typed_child_failure() {
25278 let client = Client::new("http://127.0.0.1:8080").expect("client");
25279 let mut worker = Worker::new(client, "rust-parent-workers");
25280 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
25281 match ctx
25282 .start_child_workflow(
25283 "python.child",
25284 ChildWorkflowOptions::new("python-child-workers"),
25285 json!([]),
25286 )
25287 .await
25288 {
25289 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
25290 "reason": failure.reason,
25291 "failure_id": failure.failure_id,
25292 "exception_class": failure.exception_class,
25293 "child_run_id": failure.child_workflow_run_id,
25294 })),
25295 Err(error) => Err(error),
25296 Ok(_) => Err(Error::WorkerLoop(
25297 "child unexpectedly succeeded".to_string(),
25298 )),
25299 }
25300 });
25301 let mut task = child_parent_task(
25302 "ChildRunFailed",
25303 json!({
25304 "sequence": 1,
25305 "child_workflow_instance_id": "wf-child",
25306 "child_workflow_run_id": "run-child",
25307 "child_workflow_type": "python.child",
25308 "failure_id": "failure-child",
25309 "failure_category": "child_workflow",
25310 "message": "payment rejected",
25311 "exception": {
25312 "type": "PaymentRejected",
25313 "class": "payments.PaymentRejected",
25314 "message": "payment rejected"
25315 }
25316 }),
25317 );
25318 task.workflow_type = "rust.handled-parent".to_string();
25319 task.history_events[0].payload["parent_close_policy"] = json!("abandon");
25320
25321 let commands = worker.execute_workflow_task(task).expect("handled failure");
25322 assert_eq!(commands[0]["type"], "complete_workflow");
25323 let output =
25324 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
25325 assert_eq!(output["reason"], "child_workflow");
25326 assert_eq!(output["failure_id"], "failure-child");
25327 assert_eq!(output["exception_class"], "payments.PaymentRejected");
25328 assert_eq!(output["child_run_id"], "run-child");
25329 }
25330
25331 #[test]
25332 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
25333 let client = Client::new("http://127.0.0.1:8080").expect("client");
25334 let mut worker = Worker::new(client, "rust-workers");
25335
25336 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
25337 let signal = ctx.wait_signal("start").await?;
25338 let name = signal
25339 .first()
25340 .and_then(|value| value.as_str())
25341 .unwrap_or("world");
25342 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
25343 Ok(json!({
25344 "greeting": greeting,
25345 "language": "rust"
25346 }))
25347 });
25348
25349 let signal_arguments =
25350 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
25351 let task = WorkflowTask {
25352 task_id: "wft-rust-signal-1".to_string(),
25353 workflow_command_id: None,
25354 workflow_id: Some("wf-rust-hello".to_string()),
25355 run_id: Some("run-rust-hello".to_string()),
25356 workflow_type: "rust.hello_workflow".to_string(),
25357 cancel_requested: false,
25358 payload_codec: DEFAULT_CODEC.to_string(),
25359 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
25360 history_events: vec![HistoryEvent {
25361 event_type: "SignalReceived".to_string(),
25362 payload: json!({
25363 "signal_id": "sig-rust-1",
25364 "signal_name": "start"
25365 }),
25366 raw: HashMap::new(),
25367 }],
25368 total_history_events: Some(1),
25369 history_size_bytes: None,
25370 continue_as_new_recommended: None,
25371 history_budget_pressure: None,
25372 next_history_page_token: None,
25373 workflow_task_attempt: 1,
25374 workflow_signal_id: Some("sig-rust-1".to_string()),
25375 signal_name: Some("start".to_string()),
25376 signal_arguments: Some(signal_arguments),
25377 workflow_update_id: None,
25378 update_name: None,
25379 lease_owner: Some("rust-worker".to_string()),
25380 };
25381
25382 let commands = worker.execute_workflow_task(task).expect("workflow task");
25383
25384 assert_eq!(commands.len(), 1);
25385 assert_eq!(commands[0]["type"], "schedule_activity");
25386 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
25387 assert_eq!(
25388 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
25389 json!(["Rust"])
25390 );
25391 }
25392
25393 #[test]
25394 fn workflow_task_appends_paginated_history_events() {
25395 let mut task = WorkflowTask {
25396 task_id: "wft-rust-pages-1".to_string(),
25397 workflow_command_id: None,
25398 workflow_id: Some("wf-rust-pages".to_string()),
25399 run_id: Some("run-rust-pages".to_string()),
25400 workflow_type: "rust.hello_workflow".to_string(),
25401 cancel_requested: false,
25402 payload_codec: DEFAULT_CODEC.to_string(),
25403 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
25404 history_events: vec![HistoryEvent {
25405 event_type: "WorkflowStarted".to_string(),
25406 payload: json!({}),
25407 raw: HashMap::new(),
25408 }],
25409 total_history_events: Some(3),
25410 history_size_bytes: None,
25411 continue_as_new_recommended: None,
25412 history_budget_pressure: None,
25413 next_history_page_token: Some("MQ==".to_string()),
25414 workflow_task_attempt: 1,
25415 workflow_signal_id: None,
25416 signal_name: None,
25417 signal_arguments: None,
25418 workflow_update_id: None,
25419 update_name: None,
25420 lease_owner: Some("rust-worker".to_string()),
25421 };
25422
25423 task.append_history_page(WorkflowTaskHistoryPage {
25424 history_events: vec![
25425 HistoryEvent {
25426 event_type: "SignalReceived".to_string(),
25427 payload: json!({
25428 "signal_id": "sig-rust-1",
25429 "signal_name": "start",
25430 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
25431 .expect("signal arguments")
25432 }),
25433 raw: HashMap::new(),
25434 },
25435 HistoryEvent {
25436 event_type: "MarkerRecorded".to_string(),
25437 payload: json!({"sequence": 3}),
25438 raw: HashMap::new(),
25439 },
25440 ],
25441 total_history_events: Some(3),
25442 next_history_page_token: None,
25443 });
25444
25445 assert_eq!(task.history_events.len(), 3);
25446 assert_eq!(task.total_history_events, Some(3));
25447 assert_eq!(task.next_history_page_token, None);
25448
25449 let signal = task
25450 .history_events
25451 .iter()
25452 .find(|event| event.event_type == "SignalReceived")
25453 .expect("signal event");
25454 assert_eq!(
25455 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
25456 vec![AvroValue::String("Rust".to_string())]
25457 );
25458 }
25459
25460 #[tokio::test]
25461 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
25462 let client = Client::new("http://127.0.0.1:8080").expect("client");
25463 let mut worker = Worker::new(client, "rust-workers");
25464 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25465 worker.register_query("counter", "current", |ctx, _args| async move {
25466 let mut count = 0_i64;
25467 for signal in ctx.signal_events() {
25468 let value = signal
25469 .arguments
25470 .first()
25471 .and_then(Value::as_i64)
25472 .unwrap_or_default();
25473 match signal.name.as_str() {
25474 "increment" => count += value,
25475 "set" => count = value,
25476 _ => {}
25477 }
25478 }
25479 Ok(json!(count))
25480 });
25481
25482 let task = QueryTask {
25483 query_task_id: "query-rust-counter".to_string(),
25484 query_task_attempt: 1,
25485 lease_owner: Some("rust-worker".to_string()),
25486 workflow_id: Some("counter-1".to_string()),
25487 run_id: Some("run-counter-1".to_string()),
25488 workflow_type: "counter".to_string(),
25489 query_name: "current".to_string(),
25490 payload_codec: DEFAULT_CODEC.to_string(),
25491 workflow_arguments: Some(
25492 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
25493 ),
25494 query_arguments: Some(
25495 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
25496 ),
25497 history_events: vec![
25498 HistoryEvent {
25499 event_type: "SignalReceived".to_string(),
25500 payload: json!({
25501 "signal_id": "php-signal-1",
25502 "signal_name": "increment",
25503 "workflow_sequence": 1,
25504 "payload_codec": DEFAULT_CODEC,
25505 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
25506 }),
25507 raw: HashMap::new(),
25508 },
25509 HistoryEvent {
25510 event_type: "SignalReceived".to_string(),
25511 payload: json!({
25512 "signal_id": "python-signal-2",
25513 "signal_name": "increment",
25514 "workflow_sequence": 2,
25515 "payload_codec": DEFAULT_CODEC,
25516 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
25517 }),
25518 raw: HashMap::new(),
25519 },
25520 HistoryEvent {
25521 event_type: "SignalReceived".to_string(),
25522 payload: json!({
25523 "signal_id": "rust-signal-3",
25524 "signal_name": "set",
25525 "workflow_sequence": 3,
25526 "payload_codec": DEFAULT_CODEC,
25527 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
25528 }),
25529 raw: HashMap::new(),
25530 },
25531 ],
25532 history_export: None,
25533 run_status: Some("completed".to_string()),
25534 };
25535
25536 let result = worker.execute_query_task(task).await.expect("query result");
25537 assert_eq!(result.into_json().expect("query projection"), json!(0));
25538 }
25539
25540 #[tokio::test]
25541 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
25542 let worker = replay_counter_worker();
25543 let running_history = json!([
25544 {
25545 "type": "ActivityCompleted",
25546 "payload": {
25547 "sequence": 1,
25548 "activity_type": "load-counter",
25549 "payload_codec": DEFAULT_CODEC,
25550 "result": fixture_envelope(json!("loaded"))
25551 }
25552 },
25553 {
25554 "type": "SignalWaitOpened",
25555 "payload": {
25556 "sequence": 3,
25557 "signal_name": "increment"
25558 }
25559 },
25560 {
25561 "type": "SignalReceived",
25562 "payload": {
25563 "signal_id": "signal-3",
25564 "signal_name": "increment",
25565 "workflow_sequence": 2,
25566 "payload_codec": DEFAULT_CODEC,
25567 "arguments": fixture_envelope(json!([3]))
25568 }
25569 },
25570 {
25571 "type": "SignalApplied",
25572 "payload": {
25573 "sequence": 3,
25574 "signal_id": "signal-3",
25575 "signal_name": "increment",
25576 "payload_codec": DEFAULT_CODEC,
25577 "value": fixture_envelope(json!([3]))
25578 }
25579 }
25580 ]);
25581
25582 let running = worker
25583 .execute_query_task(replay_counter_query(
25584 "current",
25585 running_history.clone(),
25586 "running",
25587 ))
25588 .await
25589 .expect("running replay query");
25590 assert_eq!(
25591 running.clone().into_json().expect("query projection"),
25592 json!({"loaded": "loaded", "count": 3, "finished": false})
25593 );
25594
25595 let detached = worker
25596 .execute_query_task(replay_counter_query(
25597 "detached-mutation",
25598 running_history.clone(),
25599 "running",
25600 ))
25601 .await
25602 .expect("query mutates only its detached state clone");
25603 assert_eq!(detached.into_json().expect("query projection"), json!(999));
25604 let failed = worker
25605 .execute_query_task(replay_counter_query(
25606 "failed-mutation",
25607 running_history.clone(),
25608 "running",
25609 ))
25610 .await
25611 .expect_err("failed query");
25612 assert_eq!(failed.reason, "query_rejected");
25613 let unchanged = worker
25614 .execute_query_task(replay_counter_query("current", running_history, "running"))
25615 .await
25616 .expect("later query reconstructs unchanged state");
25617 assert_eq!(unchanged, running);
25618
25619 let restarted_worker = replay_counter_worker();
25620 let empty_arguments = fixture_envelope(json!([]));
25621 let loaded_result = fixture_envelope(json!("loaded"));
25622 let signal_three = fixture_blob(json!([3]));
25623 let signal_five = fixture_blob(json!([5]));
25624 let restarted_task: QueryTask = serde_json::from_value(json!({
25625 "query_task_id": "query-after-restart",
25626 "workflow_id": "counter-1",
25627 "run_id": "run-counter-1",
25628 "workflow_type": "replay-counter",
25629 "query_name": "current",
25630 "payload_codec": DEFAULT_CODEC,
25631 "workflow_arguments": empty_arguments.clone(),
25632 "query_arguments": empty_arguments,
25633 "history_events": [],
25634 "history_export": {
25635 "payloads": {"codec": DEFAULT_CODEC},
25636 "history_events": [
25637 {
25638 "type": "ActivityCompleted",
25639 "payload": {
25640 "sequence": 1,
25641 "activity_type": "load-counter",
25642 "payload_codec": DEFAULT_CODEC,
25643 "result": null
25644 }
25645 },
25646 {
25647 "type": "SignalWaitOpened",
25648 "payload": {
25649 "sequence": 3,
25650 "signal_name": "increment"
25651 }
25652 },
25653 {
25654 "type": "SignalReceived",
25655 "payload": {
25656 "signal_id": "signal-3",
25657 "signal_name": "increment",
25658 "workflow_sequence": 2
25659 }
25660 },
25661 {
25662 "type": "SignalApplied",
25663 "payload": {
25664 "sequence": 3,
25665 "signal_id": "signal-3",
25666 "signal_name": "increment"
25667 }
25668 },
25669 {
25670 "type": "SignalWaitOpened",
25671 "payload": {
25672 "sequence": 5,
25673 "signal_name": "increment"
25674 }
25675 },
25676 {
25677 "type": "SignalReceived",
25678 "payload": {
25679 "signal_id": "signal-5",
25680 "signal_name": "increment",
25681 "workflow_sequence": 4
25682 }
25683 },
25684 {
25685 "type": "SignalApplied",
25686 "payload": {
25687 "sequence": 5,
25688 "signal_id": "signal-5",
25689 "signal_name": "increment"
25690 }
25691 }
25692 ],
25693 "activities": [{
25694 "sequence": 1,
25695 "activity_type": "load-counter",
25696 "payload_codec": DEFAULT_CODEC,
25697 "result": loaded_result
25698 }],
25699 "signals": [
25700 {
25701 "id": "signal-3",
25702 "name": "increment",
25703 "workflow_sequence": 2,
25704 "payload_codec": DEFAULT_CODEC,
25705 "arguments": signal_three
25706 },
25707 {
25708 "id": "signal-5",
25709 "name": "increment",
25710 "workflow_sequence": 4,
25711 "payload_codec": DEFAULT_CODEC,
25712 "arguments": signal_five
25713 }
25714 ]
25715 },
25716 "run_status": "completed"
25717 }))
25718 .expect("cold replay query task");
25719 let completed = restarted_worker
25720 .execute_query_task(restarted_task)
25721 .await
25722 .expect("completed cold replay query");
25723 assert_eq!(
25724 completed.into_json().expect("query projection"),
25725 json!({"loaded": "loaded", "count": 8, "finished": true})
25726 );
25727 }
25728
25729 #[tokio::test]
25730 async fn replayed_query_replay_failures_are_machine_readable() {
25731 let worker = replay_counter_worker();
25732 let task = replay_counter_query(
25733 "current",
25734 json!([{
25735 "type": "ActivityCompleted",
25736 "payload": {
25737 "sequence": 1,
25738 "payload_codec": DEFAULT_CODEC,
25739 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
25740 }
25741 }]),
25742 "running",
25743 );
25744 let failure = worker
25745 .execute_query_task(task)
25746 .await
25747 .expect_err("invalid replay history payload");
25748 assert_eq!(failure.reason, "query_payload_decode_failed");
25749 assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
25750 assert!(failure.message.contains("invalid_payload_framing"));
25751 }
25752
25753 #[tokio::test]
25754 async fn query_task_restores_compact_history_from_export() {
25755 let client = Client::new("http://127.0.0.1:8080").expect("client");
25756 let mut worker = Worker::new(client, "rust-workers");
25757 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25758 worker.register_query("counter", "current", |ctx, _args| async move {
25759 Ok(json!(ctx.signals("increment")[0][0]))
25760 });
25761 let empty_arguments = fixture_envelope(json!([]));
25762 let exported_signal = fixture_blob(json!([9]));
25763 let task: QueryTask = serde_json::from_value(json!({
25764 "query_task_id": "query-export",
25765 "workflow_type": "counter",
25766 "query_name": "current",
25767 "payload_codec": DEFAULT_CODEC,
25768 "workflow_arguments": empty_arguments.clone(),
25769 "query_arguments": empty_arguments,
25770 "history_events": [],
25771 "history_export": {
25772 "payloads": {"codec": DEFAULT_CODEC},
25773 "history_events": [{
25774 "type": "SignalReceived",
25775 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
25776 }],
25777 "signals": [{
25778 "id": "signal-export",
25779 "name": "increment",
25780 "status": "applied",
25781 "workflow_sequence": 1,
25782 "payload_codec": DEFAULT_CODEC,
25783 "arguments": exported_signal
25784 }]
25785 }
25786 }))
25787 .expect("query task");
25788
25789 let result = worker.execute_query_task(task).await.expect("query result");
25790 assert_eq!(result.into_json().expect("query projection"), json!(9));
25791 }
25792
25793 #[tokio::test]
25794 async fn query_task_failures_have_stable_reasons() {
25795 let client = Client::new("http://127.0.0.1:8080").expect("client");
25796 let mut worker = Worker::new(client, "rust-workers");
25797 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25798 worker.register_query(
25799 "counter",
25800 "current",
25801 |_ctx, _args| async move { Ok(json!(0)) },
25802 );
25803
25804 let base_task = QueryTask {
25805 query_task_id: "query-errors".to_string(),
25806 query_task_attempt: 1,
25807 lease_owner: None,
25808 workflow_id: Some("counter-errors".to_string()),
25809 run_id: Some("run-errors".to_string()),
25810 workflow_type: "counter".to_string(),
25811 query_name: "missing".to_string(),
25812 payload_codec: DEFAULT_CODEC.to_string(),
25813 workflow_arguments: Some(fixture_envelope(json!([]))),
25814 query_arguments: Some(fixture_envelope(json!([]))),
25815 history_events: Vec::new(),
25816 history_export: None,
25817 run_status: Some("running".to_string()),
25818 };
25819
25820 let unknown = worker
25821 .execute_query_task(base_task.clone())
25822 .await
25823 .expect_err("unknown query");
25824 assert_eq!(unknown.reason, "rejected_unknown_query");
25825
25826 let mut malformed = base_task;
25827 malformed.query_name = "current".to_string();
25828 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
25829 let malformed = worker
25830 .execute_query_task(malformed)
25831 .await
25832 .expect_err("malformed payload");
25833 assert_eq!(malformed.reason, "query_payload_decode_failed");
25834
25835 let client = Client::new("http://127.0.0.1:8080").expect("client");
25836 let mut unavailable_worker = Worker::new(client, "rust-workers");
25837 unavailable_worker
25838 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25839 let empty_arguments = fixture_envelope(json!([]));
25840 let unavailable_task: QueryTask = serde_json::from_value(json!({
25841 "query_task_id": "query-unavailable",
25842 "workflow_type": "counter",
25843 "query_name": "current",
25844 "payload_codec": DEFAULT_CODEC,
25845 "workflow_arguments": empty_arguments.clone(),
25846 "query_arguments": empty_arguments
25847 }))
25848 .expect("query task");
25849 let unavailable = unavailable_worker
25850 .execute_query_task(unavailable_task)
25851 .await
25852 .expect_err("query handler unavailable");
25853 assert_eq!(unavailable.reason, "query_handler_unavailable");
25854 }
25855
25856 #[tokio::test]
25857 async fn client_query_decodes_result_and_typed_failure() {
25858 let server = MockWorkerServer::start();
25859 let client = Client::builder(server.base_url())
25860 .timeout(Duration::from_secs(2))
25861 .build()
25862 .expect("client");
25863
25864 let result = client
25865 .query_workflow("counter-1", "current", json!([]))
25866 .await
25867 .expect("query result");
25868 assert_eq!(result, json!({"count": 8}));
25869
25870 let error = client
25871 .query_workflow("counter-1", "missing", json!([]))
25872 .await
25873 .expect_err("unknown query");
25874 let Error::QueryFailed(failure) = error else {
25875 panic!("expected typed query failure");
25876 };
25877 assert_eq!(failure.status, 404);
25878 assert_eq!(failure.reason, "rejected_unknown_query");
25879 }
25880
25881 #[tokio::test]
25882 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
25883 let server = MockWorkerServer::start();
25884 let client = Client::builder(server.base_url())
25885 .timeout(Duration::from_secs(2))
25886 .build()
25887 .expect("client");
25888 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
25889
25890 client
25891 .start_workflow(
25892 "typed.echo",
25893 "rust-workers",
25894 "typed-start",
25895 arguments.clone(),
25896 )
25897 .await
25898 .expect("typed workflow start");
25899 assert_eq!(
25900 decode_wire_avro_value(
25901 &server.request_body("/api/workflows")["input"],
25902 DEFAULT_CODEC,
25903 )
25904 .expect("typed start input"),
25905 arguments
25906 );
25907
25908 client
25909 .signal_workflow("typed-1", "changed", arguments.clone())
25910 .await
25911 .expect("typed signal");
25912 assert_eq!(
25913 decode_wire_avro_value(
25914 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
25915 DEFAULT_CODEC,
25916 )
25917 .expect("typed signal input"),
25918 arguments
25919 );
25920
25921 assert_eq!(
25922 client
25923 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
25924 .await
25925 .expect("typed query"),
25926 typed_fidelity_probe()
25927 );
25928 assert_eq!(
25929 decode_wire_avro_value(
25930 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
25931 DEFAULT_CODEC,
25932 )
25933 .expect("typed query input"),
25934 arguments
25935 );
25936
25937 assert_eq!(
25938 client
25939 .update_workflow_avro_value(
25940 "typed-1",
25941 "replace",
25942 arguments.clone(),
25943 Some("typed-request"),
25944 )
25945 .await
25946 .expect("typed update"),
25947 typed_fidelity_probe()
25948 );
25949 let update = server.request_body("/api/workflows/typed-1/update/replace");
25950 assert_eq!(update["request_id"], "typed-request");
25951 assert_eq!(
25952 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
25953 arguments
25954 );
25955
25956 let handle = WorkflowHandle {
25957 client: client.clone(),
25958 workflow_id: "typed-1".to_string(),
25959 run_id: Some("run-typed-1".to_string()),
25960 workflow_type: "typed.echo".to_string(),
25961 };
25962 assert_eq!(
25963 handle
25964 .result_avro_value(WorkflowResultOptions::default())
25965 .await
25966 .expect("typed workflow result"),
25967 typed_fidelity_probe()
25968 );
25969
25970 client
25971 .complete_activity_task(
25972 "activity-typed",
25973 "attempt-typed",
25974 "rust-worker",
25975 typed_fidelity_probe(),
25976 DEFAULT_CODEC,
25977 )
25978 .await
25979 .expect("typed activity completion");
25980 assert_eq!(
25981 decode_wire_avro_value(
25982 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
25983 ["result"],
25984 DEFAULT_CODEC,
25985 )
25986 .expect("typed activity result"),
25987 typed_fidelity_probe()
25988 );
25989 client
25990 .fail_activity_task(
25991 "activity-typed",
25992 "attempt-typed",
25993 "rust-worker",
25994 "typed failure",
25995 true,
25996 )
25997 .await
25998 .expect("activity failure");
25999 }
26000
26001 #[tokio::test]
26002 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
26003 let server = MockWorkerServer::start();
26004 let client = Client::builder(server.base_url())
26005 .timeout(Duration::from_secs(2))
26006 .build()
26007 .expect("client");
26008
26009 let options = WorkflowCommandOptions::new()
26010 .reason("cleanup requested")
26011 .request_id("cancel-17");
26012 let cancelled = client
26013 .cancel_workflow("wf-lifecycle", options)
26014 .await
26015 .expect("instance cancellation");
26016 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
26017 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
26018 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
26019 assert_eq!(
26020 server.request_body("/api/workflows/wf-lifecycle/cancel"),
26021 json!({"reason":"cleanup requested","request_id":"cancel-17"})
26022 );
26023
26024 let terminated = client
26025 .terminate_workflow(
26026 "wf-lifecycle",
26027 WorkflowCommandOptions::new().reason("forced stop"),
26028 )
26029 .await
26030 .expect("instance termination");
26031 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
26032 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
26033
26034 client
26035 .cancel_workflow_run(
26036 "wf-lifecycle",
26037 "run-current",
26038 WorkflowCommandOptions::default(),
26039 )
26040 .await
26041 .expect("selected run cancellation");
26042 client
26043 .terminate_workflow_run(
26044 "wf-lifecycle",
26045 "run-current",
26046 WorkflowCommandOptions::default(),
26047 )
26048 .await
26049 .expect("selected run termination");
26050
26051 for (command, error) in [
26052 (
26053 WorkflowCommandKind::Cancel,
26054 client
26055 .cancel_workflow_run(
26056 "wf-lifecycle",
26057 "run-stale",
26058 WorkflowCommandOptions::default(),
26059 )
26060 .await
26061 .expect_err("stale cancellation must be rejected"),
26062 ),
26063 (
26064 WorkflowCommandKind::Terminate,
26065 client
26066 .terminate_workflow_run(
26067 "wf-lifecycle",
26068 "run-stale",
26069 WorkflowCommandOptions::default(),
26070 )
26071 .await
26072 .expect_err("stale termination must be rejected"),
26073 ),
26074 ] {
26075 let Error::WorkflowCommandRejected(rejection) = error else {
26076 panic!("expected typed command rejection");
26077 };
26078 assert_eq!(rejection.command, command);
26079 assert_eq!(rejection.status, 409);
26080 assert_eq!(rejection.reason, "historical_run_command_rejected");
26081 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
26082 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
26083 }
26084 }
26085
26086 #[tokio::test]
26087 async fn redrive_targets_a_failed_run_and_returns_successor_identity() {
26088 let server = MockWorkerServer::start();
26089 let client = Client::builder(server.base_url())
26090 .timeout(Duration::from_secs(2))
26091 .build()
26092 .expect("client");
26093
26094 let result = client
26095 .redrive_workflow_run("wf-lifecycle", "run-failed", Some("retry-1"))
26096 .await
26097 .expect("redrive accepted");
26098 assert_eq!(result.workflow_id, "wf-lifecycle");
26099 assert_eq!(result.source_run_id, "run-failed");
26100 assert_eq!(result.run_id, "run-successor");
26101 assert_eq!(result.resume_step_sequence, 2);
26102 assert_eq!(
26103 server.request_body("/api/workflows/wf-lifecycle/runs/run-failed/redrive"),
26104 json!({"request_id":"retry-1"})
26105 );
26106
26107 let repeated = client
26108 .redrive_workflow_run("wf-lifecycle", "run-failed-existing", Some("retry-1"))
26109 .await
26110 .expect("idempotent redrive response accepted");
26111 assert_eq!(repeated.run_id, "run-successor");
26112
26113 let error = client
26114 .redrive_workflow_run("wf-lifecycle", "run-completed", None)
26115 .await
26116 .expect_err("completed run must be rejected");
26117 let Error::Http { status, body } = error else {
26118 panic!("expected HTTP redrive rejection");
26119 };
26120 assert_eq!(status.as_u16(), 409);
26121 assert!(body.contains("run_not_failed"));
26122 assert_eq!(
26123 server.request_body("/api/workflows/wf-lifecycle/runs/run-completed/redrive"),
26124 json!({})
26125 );
26126
26127 let handle = WorkflowHandle {
26128 client: client.clone(),
26129 workflow_id: "wf-lifecycle".to_string(),
26130 run_id: Some("run-failed".to_string()),
26131 workflow_type: "test".to_string(),
26132 };
26133 assert_eq!(
26134 handle.redrive(None).await.expect("handle redrive").run_id,
26135 "run-successor"
26136 );
26137 let missing_run = WorkflowHandle {
26138 run_id: None,
26139 ..handle
26140 };
26141 assert!(matches!(
26142 missing_run.redrive(None).await,
26143 Err(Error::Codec(_))
26144 ));
26145 }
26146
26147 #[tokio::test]
26148 async fn workflow_start_options_send_server_enforced_deadlines() {
26149 let server = MockWorkerServer::start();
26150 let client = Client::builder(server.base_url())
26151 .timeout(Duration::from_secs(2))
26152 .build()
26153 .expect("client");
26154
26155 let handle = client
26156 .start_workflow_with_options(
26157 "rust.timeout",
26158 "rust-timeouts",
26159 "wf-start-options",
26160 WorkflowStartOptions::new()
26161 .execution_timeout_seconds(30)
26162 .run_timeout_seconds(1),
26163 json!([]),
26164 )
26165 .await
26166 .expect("workflow start");
26167
26168 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
26169 let body = server.request_body("/api/workflows");
26170 assert_eq!(body["execution_timeout_seconds"], 30);
26171 assert_eq!(body["run_timeout_seconds"], 1);
26172
26173 let invalid = client
26174 .start_workflow_with_options(
26175 "rust.timeout",
26176 "rust-timeouts",
26177 "wf-invalid-options",
26178 WorkflowStartOptions::new()
26179 .execution_timeout_seconds(1)
26180 .run_timeout_seconds(2),
26181 json!([]),
26182 )
26183 .await
26184 .expect_err("invalid deadline ordering");
26185 assert!(invalid
26186 .to_string()
26187 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
26188 }
26189
26190 #[tokio::test]
26191 async fn workflow_result_returns_each_typed_terminal_outcome() {
26192 let server = MockWorkerServer::start();
26193 let client = Client::builder(server.base_url())
26194 .timeout(Duration::from_secs(2))
26195 .build()
26196 .expect("client");
26197 let options = WorkflowResultOptions {
26198 poll_interval: Duration::ZERO,
26199 timeout: Duration::from_secs(1),
26200 };
26201
26202 let failed = WorkflowHandle {
26203 client: client.clone(),
26204 workflow_id: "wf-failed".to_string(),
26205 run_id: Some("run-failed".to_string()),
26206 workflow_type: "failure".to_string(),
26207 }
26208 .result(options)
26209 .await
26210 .expect_err("failed outcome");
26211 let Error::WorkflowFailed(failure) = failed else {
26212 panic!("expected WorkflowFailed");
26213 };
26214 assert_eq!(failure.workflow_id, "wf-failed");
26215 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
26216 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
26217 assert_eq!(failure.failure_category.as_deref(), Some("application"));
26218 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
26219 assert_eq!(
26220 failure.exception_class.as_deref(),
26221 Some("billing::PaymentError")
26222 );
26223 assert_eq!(failure.non_retryable, Some(true));
26224
26225 for (workflow_id, expected_kind, expected_reason) in [
26226 (
26227 "wf-cancelled",
26228 WorkflowTerminalKind::Cancelled,
26229 "cleanup requested",
26230 ),
26231 (
26232 "wf-terminated",
26233 WorkflowTerminalKind::Terminated,
26234 "forced stop",
26235 ),
26236 (
26237 "wf-timed-out",
26238 WorkflowTerminalKind::TimedOut,
26239 "run_timeout",
26240 ),
26241 ] {
26242 let error = WorkflowHandle {
26243 client: client.clone(),
26244 workflow_id: workflow_id.to_string(),
26245 run_id: None,
26246 workflow_type: "terminal".to_string(),
26247 }
26248 .result(options)
26249 .await
26250 .expect_err("typed terminal outcome");
26251 let outcome = match error {
26252 Error::WorkflowCancelled(outcome) => outcome,
26253 Error::WorkflowTerminated(outcome) => outcome,
26254 Error::WorkflowTimedOut(outcome) => outcome,
26255 other => panic!("unexpected terminal error: {other}"),
26256 };
26257 assert_eq!(outcome.kind, expected_kind);
26258 assert_eq!(outcome.workflow_id, workflow_id);
26259 assert_eq!(outcome.reason, expected_reason);
26260 }
26261
26262 let wait_timeout = WorkflowHandle {
26263 client,
26264 workflow_id: "wf-waiting".to_string(),
26265 run_id: Some("run-waiting".to_string()),
26266 workflow_type: "waiting".to_string(),
26267 }
26268 .result(WorkflowResultOptions {
26269 poll_interval: Duration::ZERO,
26270 timeout: Duration::ZERO,
26271 })
26272 .await
26273 .expect_err("client wait timeout");
26274 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
26275 panic!("expected typed client timeout");
26276 };
26277 assert_eq!(timeout.reason, "result_wait_timeout");
26278 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
26279 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
26280 }
26281
26282 #[tokio::test]
26283 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
26284 let server = MockWorkerServer::start();
26285 let client = Client::builder(server.base_url())
26286 .timeout(Duration::from_secs(2))
26287 .build()
26288 .expect("client");
26289
26290 let handle = WorkflowHandle {
26291 client,
26292 workflow_id: "wf-selected".to_string(),
26293 run_id: Some("run-selected".to_string()),
26294 workflow_type: "selected".to_string(),
26295 };
26296 let options = WorkflowResultOptions {
26297 poll_interval: Duration::ZERO,
26298 timeout: Duration::from_secs(1),
26299 };
26300
26301 let current = handle
26302 .result(options)
26303 .await
26304 .expect("instance result follows the current run");
26305 assert_eq!(current, json!("current run output"));
26306
26307 let error = handle
26308 .result_selected_run(options)
26309 .await
26310 .expect_err("the selected run is cancelled even though the current run completed");
26311
26312 let Error::WorkflowCancelled(outcome) = error else {
26313 panic!("expected selected run cancellation");
26314 };
26315 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
26316 assert_eq!(outcome.reason, "selected run cancelled");
26317 assert_eq!(
26318 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
26319 1
26320 );
26321 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
26322 }
26323
26324 #[tokio::test]
26325 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
26326 let server = MockWorkerServer::draining_polls();
26327 let client = Client::builder(server.base_url())
26328 .timeout(Duration::from_secs(2))
26329 .build()
26330 .expect("client");
26331
26332 let workflow = client
26333 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
26334 .await
26335 .expect("workflow drain response");
26336 let activity = client
26337 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
26338 .await
26339 .expect("activity drain response");
26340 let query = client
26341 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
26342 .await
26343 .expect("query drain response");
26344
26345 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
26346 assert_eq!(
26347 outcome,
26348 WorkerPollOutcome::Stop {
26349 poll_status: Some("draining".to_string()),
26350 reason: Some("worker_draining".to_string()),
26351 }
26352 );
26353 }
26354
26355 assert!(client
26356 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
26357 .await
26358 .expect("compatibility poll")
26359 .is_none());
26360 }
26361
26362 #[tokio::test]
26363 async fn managed_worker_honors_drain_stop_for_every_task_family() {
26364 let server = MockWorkerServer::draining_polls();
26365 let client = Client::builder(server.base_url())
26366 .timeout(Duration::from_secs(2))
26367 .build()
26368 .expect("client");
26369
26370 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
26371 .worker_id("draining-workflow-worker")
26372 .poll_timeout(Duration::ZERO);
26373 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
26374 workflow_worker
26375 .run()
26376 .await
26377 .expect("workflow drain is a clean stop");
26378
26379 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
26380 .worker_id("draining-activity-worker")
26381 .poll_timeout(Duration::ZERO);
26382 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
26383 activity_worker
26384 .run()
26385 .await
26386 .expect("activity drain is a clean stop");
26387
26388 let mut query_worker = Worker::new(client, "rust-workers")
26389 .worker_id("draining-query-worker")
26390 .poll_timeout(Duration::ZERO);
26391 query_worker.register_query("counter", "current", |_ctx, _args| async {
26392 Ok(Value::Null)
26393 });
26394 query_worker
26395 .run()
26396 .await
26397 .expect("query drain is a clean stop");
26398 }
26399
26400 #[tokio::test]
26401 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
26402 let server = MockWorkerServer::start();
26403 let client = Client::builder(server.base_url())
26404 .timeout(Duration::from_secs(2))
26405 .build()
26406 .expect("client");
26407
26408 let details: Value = serde_json::from_str(include_str!(
26409 "../tests/fixtures/activity-heartbeat-progress.json"
26410 ))
26411 .expect("heartbeat progress fixture");
26412 assert!(
26415 encode_typed_envelope(&AvroValue::from_serialize(&details).unwrap(), DEFAULT_CODEC)
26416 .unwrap()["blob"]
26417 .as_str()
26418 .unwrap()
26419 .len()
26420 > 191
26421 );
26422 let heartbeat = client
26423 .heartbeat_activity_task("activity-cancel", "attempt-cancel", "rust-worker", &details)
26424 .await
26425 .expect("cancellation heartbeat");
26426 assert!(heartbeat.cancel_requested);
26427 assert!(heartbeat.should_stop());
26428 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
26429 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
26430 let heartbeat_body =
26431 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
26432 assert_eq!(heartbeat_body["details"], details);
26433 assert_eq!(heartbeat_body["activity_attempt_id"], "attempt-cancel");
26434 assert_eq!(heartbeat_body["lease_owner"], "rust-worker");
26435
26436 let error = client
26437 .complete_activity_task(
26438 "activity-cancel",
26439 "attempt-cancel",
26440 "rust-worker",
26441 json!({"late":true}),
26442 DEFAULT_CODEC,
26443 )
26444 .await
26445 .expect_err("late completion must be refused");
26446 assert!(activity_task_rejection_is_final(&error));
26447 let Error::ActivityTaskRejected(rejection) = error else {
26448 panic!("expected typed activity rejection");
26449 };
26450 assert_eq!(rejection.status, 409);
26451 assert_eq!(rejection.reason, "run_cancelled");
26452 assert!(rejection.cancel_requested);
26453 assert_eq!(rejection.can_continue, Some(false));
26454 }
26455
26456 #[tokio::test]
26457 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
26458 let server = MockWorkerServer::cancelled_activity();
26459 let client = Client::builder(server.base_url())
26460 .timeout(Duration::from_secs(2))
26461 .build()
26462 .expect("client");
26463 let cancellation_observed = Arc::new(AtomicBool::new(false));
26464 let observed = Arc::clone(&cancellation_observed);
26465 let mut worker = Worker::new(client.clone(), "rust-workers")
26466 .worker_id("rust-cancel-worker")
26467 .poll_timeout(Duration::from_millis(10));
26468 worker.register_activity("cancel-aware", move |ctx, _args| {
26469 let observed = Arc::clone(&observed);
26470 async move {
26471 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
26472 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
26473 Ok(json!({"late":"completion"}))
26474 }
26475 });
26476
26477 assert_eq!(
26478 worker.run_once().await.expect("cancelled attempt handled"),
26479 1
26480 );
26481 assert!(cancellation_observed.load(Ordering::SeqCst));
26482 assert_eq!(
26483 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
26484 1
26485 );
26486
26487 let mut restarted = Worker::new(client, "rust-workers")
26488 .worker_id("rust-cancel-worker-restarted")
26489 .poll_timeout(Duration::from_millis(10));
26490 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
26491 assert_eq!(
26492 restarted
26493 .run_once()
26494 .await
26495 .expect("replacement worker continues polling"),
26496 0
26497 );
26498 }
26499
26500 #[tokio::test]
26501 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
26502 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
26503 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
26504 let client = Client::builder(server.base_url())
26505 .timeout(Duration::from_secs(2))
26506 .build()
26507 .expect("client");
26508
26509 let direct_error = client
26510 .complete_workflow_task(
26511 "workflow-timeout-task",
26512 "timeout-worker",
26513 3,
26514 vec![json!({
26515 "type": "complete_workflow",
26516 "result": fixture_envelope(Value::Null)
26517 })],
26518 )
26519 .await
26520 .expect_err("the low-level client preserves the completion rejection");
26521 let Error::Http { status, body } = direct_error else {
26522 panic!("expected the original HTTP completion rejection");
26523 };
26524 assert_eq!(status, reqwest::StatusCode::CONFLICT);
26525 assert_eq!(
26526 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
26527 "run_timed_out"
26528 );
26529
26530 let mut worker = Worker::new(client, "rust-workers")
26531 .worker_id("timeout-worker")
26532 .poll_timeout(Duration::from_millis(10));
26533 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
26534 Ok(json!({"late": "result"}))
26535 });
26536
26537 assert_eq!(
26538 worker
26539 .run_once()
26540 .await
26541 .expect("authoritative selected-run timeout settles the tick"),
26542 1
26543 );
26544 assert_eq!(
26545 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
26546 2,
26547 "both the direct client proof and managed worker must see the rejection"
26548 );
26549 }
26550
26551 #[tokio::test]
26552 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
26553 for (name, status, response) in [
26554 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
26555 (
26556 "command was recorded",
26557 "409 Conflict",
26558 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
26559 ),
26560 (
26561 "lease conflict",
26562 "409 Conflict",
26563 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
26564 ),
26565 (
26566 "nonterminal run",
26567 "409 Conflict",
26568 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
26569 ),
26570 (
26571 "different selected run",
26572 "409 Conflict",
26573 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
26574 ),
26575 (
26576 "different task attempt",
26577 "409 Conflict",
26578 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
26579 ),
26580 (
26581 "authentication failure",
26582 "401 Unauthorized",
26583 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
26584 ),
26585 (
26586 "authorization failure",
26587 "403 Forbidden",
26588 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
26589 ),
26590 (
26591 "protocol failure",
26592 "400 Bad Request",
26593 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
26594 ),
26595 (
26596 "malformed command",
26597 "422 Unprocessable Entity",
26598 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
26599 ),
26600 (
26601 "transient server failure",
26602 "503 Service Unavailable",
26603 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
26604 ),
26605 ] {
26606 let server = MockWorkerServer::workflow_completion(status, response);
26607 let client = Client::builder(server.base_url())
26608 .timeout(Duration::from_secs(2))
26609 .build()
26610 .expect("client");
26611 let mut worker = Worker::new(client, "rust-workers")
26612 .worker_id("timeout-worker")
26613 .poll_timeout(Duration::from_millis(10));
26614 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
26615 Ok(json!({"late": "result"}))
26616 });
26617
26618 let error = worker
26619 .run_once()
26620 .await
26621 .expect_err(&format!("{name} must remain an error"));
26622 assert!(
26623 matches!(error, Error::Http { .. } | Error::Protocol(_)),
26624 "{name} returned an unexpected error variant: {error}"
26625 );
26626 }
26627 }
26628
26629 #[tokio::test]
26630 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
26631 let server = MockWorkerServer::start();
26632 let client = Client::builder(server.base_url())
26633 .worker_token(Some("worker-secret".to_string()))
26634 .namespace("orders")
26635 .timeout(Duration::from_secs(2))
26636 .build()
26637 .expect("client");
26638 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
26639
26640 let result = client
26641 .deregister_worker_registration("worker/α space")
26642 .await
26643 .expect("deregister worker registration");
26644
26645 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
26646 assert_eq!(
26647 server.worker_protocol_for(path).as_deref(),
26648 Some(WORKER_PROTOCOL_VERSION)
26649 );
26650 assert_eq!(server.control_protocol_for(path), None);
26651 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
26652 assert_eq!(
26653 server.authorization_for(path).as_deref(),
26654 Some("Bearer worker-secret")
26655 );
26656 assert_eq!(
26657 result,
26658 WorkerDeregistrationEnvelope {
26659 worker_id: "deregistered-worker".to_string(),
26660 outcome: "deregistered".to_string(),
26661 recovered_workflow_task_count: 2,
26662 }
26663 );
26664 }
26665
26666 #[tokio::test]
26667 async fn low_level_registration_rejects_update_validators_before_transport() {
26668 let server = MockWorkerServer::start();
26669 let client = Client::builder(server.base_url())
26670 .timeout(Duration::from_secs(2))
26671 .build()
26672 .expect("client");
26673
26674 for update_validators in [json!(["approve"]), json!("approve")] {
26675 let error = client
26676 .register_worker_with_command_contracts(
26677 "validator-claiming-worker",
26678 "rust-workers",
26679 vec!["orders".to_string()],
26680 vec![],
26681 1,
26682 1,
26683 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
26684 json!({
26685 "orders": {
26686 "queries": ["current"],
26687 "updates": ["approve"],
26688 "update_validators": update_validators,
26689 },
26690 }),
26691 )
26692 .await
26693 .expect_err("unsupported validator claims must fail before registration");
26694
26695 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
26696 panic!("expected typed unsupported-validator failure");
26697 };
26698 assert_eq!(workflow_type, "orders");
26699 }
26700 assert_eq!(server.request_count("/api/worker/register"), 0);
26701 }
26702
26703 #[tokio::test]
26704 async fn worker_registers_complete_json_and_lossless_handler_argument_contracts() {
26705 let server = MockWorkerServer::start();
26706 let client = Client::builder(server.base_url())
26707 .timeout(Duration::from_secs(2))
26708 .build()
26709 .expect("client");
26710 let mut worker = Worker::new(client, "rust-workers");
26711 worker.register_workflow("orders", |_ctx, input| async move { Ok(input) });
26712 worker.register_query("orders", "z-json", |_ctx, args| async move { Ok(args) });
26713 worker.register_query_avro_value("orders", "a-avro", |_ctx, args| async move { Ok(args) });
26714 worker.register_update("orders", "z-json", |_ctx, args| async move { Ok(args) });
26715 worker.register_update_avro_value("orders", "a-avro", |_ctx, args| async move { Ok(args) });
26716 worker
26717 .declare_workflow_signals("orders", &["z-json", "a-avro", "z-json"])
26718 .expect("declare workflow signals");
26719
26720 worker.register().await.expect("register handler contracts");
26721
26722 let declaration =
26723 &server.request_body("/api/worker/register")["workflow_command_contracts"]["orders"];
26724 let expected_contracts = json!([
26725 {
26726 "name": "a-avro",
26727 "parameters": [{
26728 "name": "args", "position": 0, "required": false,
26729 "variadic": true, "type": null, "allows_null": true,
26730 "default_available": false, "default": null,
26731 }],
26732 },
26733 {
26734 "name": "z-json",
26735 "parameters": [{
26736 "name": "args", "position": 0, "required": false,
26737 "variadic": true, "type": null, "allows_null": true,
26738 "default_available": false, "default": null,
26739 }],
26740 },
26741 ]);
26742 assert_eq!(declaration["queries"], json!(["a-avro", "z-json"]));
26743 assert_eq!(declaration["query_contracts"], expected_contracts);
26744 assert_eq!(declaration["updates"], json!(["a-avro", "z-json"]));
26745 assert_eq!(declaration["update_contracts"], expected_contracts);
26746 assert_eq!(declaration["signals"], json!(["a-avro", "z-json"]));
26747 assert_eq!(declaration["signal_contracts"], expected_contracts);
26748 assert_eq!(declaration["update_validators"], json!([]));
26749 }
26750
26751 #[tokio::test]
26752 async fn workflow_signal_declarations_reject_invalid_configuration_without_mutation() {
26753 let server = MockWorkerServer::start();
26754 let client = Client::new(server.base_url()).expect("client");
26755 let mut worker = Worker::new(client, "rust-workers");
26756 worker.register_workflow("orders", |_ctx, args| async move { Ok(args) });
26757 assert!(worker
26758 .declare_workflow_signals("missing", &["finish"])
26759 .is_err());
26760 worker
26761 .declare_workflow_signals("orders", &["finish"])
26762 .unwrap();
26763 for names in [&[""][..], &[" finish"][..], &["finish "][..]] {
26764 assert!(worker.declare_workflow_signals("orders", names).is_err());
26765 }
26766 worker.register().await.unwrap();
26767 assert_eq!(
26768 server.request_body("/api/worker/register")["workflow_command_contracts"]["orders"]
26769 ["signals"],
26770 json!(["finish"])
26771 );
26772 worker.declare_workflow_signals("orders", &[]).unwrap();
26773 worker.register().await.unwrap();
26774 let registrations = server.request_bodies("/api/worker/register");
26775 assert_eq!(
26776 registrations[1]["workflow_command_contracts"]["orders"]["signals"],
26777 json!([])
26778 );
26779 assert_eq!(
26780 registrations[1]["workflow_command_contracts"]["orders"]["signal_contracts"],
26781 json!([])
26782 );
26783 }
26784
26785 #[tokio::test]
26786 async fn low_level_registration_preserves_query_and_update_contracts() {
26787 let server = MockWorkerServer::start();
26788 let client = Client::builder(server.base_url())
26789 .timeout(Duration::from_secs(2))
26790 .build()
26791 .expect("client");
26792 let contracts = json!({
26793 "orders": {
26794 "queries": ["current"],
26795 "updates": ["approve"],
26796 "update_validators": [],
26797 },
26798 "payments": {
26799 "queries": ["status"],
26800 "updates": ["capture"],
26801 },
26802 });
26803
26804 client
26805 .register_worker_with_command_contracts(
26806 "command-worker",
26807 "rust-workers",
26808 vec!["orders".to_string(), "payments".to_string()],
26809 vec![],
26810 1,
26811 1,
26812 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
26813 contracts.clone(),
26814 )
26815 .await
26816 .expect("query and update contracts must remain supported");
26817
26818 assert_eq!(
26819 server.request_body("/api/worker/register")["workflow_command_contracts"],
26820 contracts
26821 );
26822 assert!(server
26823 .request_body("/api/worker/register")
26824 .get("workflow_definition_fingerprints")
26825 .is_none());
26826 }
26827
26828 #[tokio::test]
26829 async fn rust_worker_advertises_source_identity_without_reusing_it_for_a_new_handler() {
26830 let server = MockWorkerServer::start();
26831 let client = Client::builder(server.base_url())
26832 .timeout(Duration::from_secs(2))
26833 .build()
26834 .expect("client");
26835 let mut worker = Worker::new(client.clone(), "rust-workers");
26836 worker.register_workflow("orders", |_ctx, _input| async { Ok(Value::Null) });
26837
26838 assert!(worker
26839 .set_workflow_definition_sources("missing", &[include_str!("lib.rs")])
26840 .is_err());
26841 assert!(worker
26842 .set_workflow_definition_sources("orders", &[])
26843 .is_err());
26844 worker
26845 .set_workflow_definition_sources("orders", &[include_str!("lib.rs")])
26846 .expect("embed workflow source");
26847 worker.register().await.expect("register worker");
26848 let first = server.request_bodies("/api/worker/register")[0]
26849 ["workflow_definition_fingerprints"]["orders"]
26850 .as_str()
26851 .expect("source-backed fingerprint")
26852 .to_string();
26853 assert!(first.starts_with("sha256:"));
26854
26855 worker
26856 .set_workflow_definition_sources("orders", &[include_str!("lib.rs"), "changed helper"])
26857 .expect("change embedded helper source");
26858 worker.register().await.expect("register changed source");
26859 let changed = server.request_bodies("/api/worker/register")[1]
26860 ["workflow_definition_fingerprints"]["orders"]
26861 .as_str()
26862 .expect("changed fingerprint")
26863 .to_string();
26864 assert_ne!(first, changed);
26865
26866 let mut replacement =
26867 Worker::new(client, "rust-workers").worker_id(worker.worker_id.clone());
26868 replacement.register_workflow("orders", |_ctx, _input| async { Ok(Value::Null) });
26869 replacement
26870 .register()
26871 .await
26872 .expect("register replacement handler");
26873 assert_eq!(
26874 server.request_bodies("/api/worker/register")[2]["workflow_definition_fingerprints"],
26875 json!({})
26876 );
26877 }
26878
26879 #[tokio::test]
26880 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
26881 let server = MockWorkerServer::start();
26882 let control_only = Client::builder(server.base_url())
26883 .control_token(Some("control-secret".to_string()))
26884 .build()
26885 .expect("control client");
26886
26887 let error = control_only
26888 .register_worker("worker", "queue", vec![], vec![], 1, 1)
26889 .await
26890 .expect_err("control token must not authorize a worker request");
26891 assert!(matches!(
26892 error,
26893 Error::MissingRoleCredentials { role: "worker", .. }
26894 ));
26895 assert_eq!(server.request_count("/api/worker/register"), 0);
26896
26897 let worker_only = Client::builder(server.base_url())
26898 .worker_token(Some("worker-secret".to_string()))
26899 .build()
26900 .expect("worker client");
26901 let error = worker_only
26902 .health()
26903 .await
26904 .expect_err("worker token must not authorize a control request");
26905 assert!(matches!(
26906 error,
26907 Error::MissingRoleCredentials {
26908 role: "control",
26909 ..
26910 }
26911 ));
26912 assert_eq!(server.request_count("/api/health"), 0);
26913 }
26914
26915 #[tokio::test]
26916 async fn shared_token_supports_worker_and_control_planes() {
26917 let server = MockWorkerServer::start();
26918 let client = Client::builder(server.base_url())
26919 .token(Some("shared-secret".to_string()))
26920 .build()
26921 .expect("client");
26922
26923 client.health().await.expect("control request");
26924 client
26925 .register_worker("worker", "queue", vec![], vec![], 1, 1)
26926 .await
26927 .expect("worker request");
26928
26929 assert_eq!(
26930 server.authorization_for("/api/health").as_deref(),
26931 Some("Bearer shared-secret")
26932 );
26933 assert_eq!(
26934 server.control_protocol_for("/api/health").as_deref(),
26935 Some(CONTROL_PLANE_VERSION)
26936 );
26937 assert_eq!(
26938 server.authorization_for("/api/worker/register").as_deref(),
26939 Some("Bearer shared-secret")
26940 );
26941 assert_eq!(
26942 server
26943 .worker_protocol_for("/api/worker/register")
26944 .as_deref(),
26945 Some(WORKER_PROTOCOL_VERSION)
26946 );
26947 }
26948
26949 #[tokio::test]
26950 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
26951 let server = MockWorkerServer::start();
26952 let client = Client::builder(server.base_url())
26953 .timeout(Duration::from_secs(2))
26954 .build()
26955 .expect("client");
26956
26957 client
26958 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
26959 .await
26960 .expect("register");
26961 client
26962 .heartbeat_worker("capture-worker", 1, 1)
26963 .await
26964 .expect("heartbeat");
26965 client
26966 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
26967 .await
26968 .expect("workflow poll");
26969 client
26970 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
26971 .await
26972 .expect("activity poll");
26973
26974 for path in [
26975 "/api/worker/register",
26976 "/api/worker/heartbeat",
26977 "/api/worker/workflow-tasks/poll",
26978 "/api/worker/activity-tasks/poll",
26979 ] {
26980 assert_eq!(
26981 server.worker_protocol_for(path).as_deref(),
26982 Some(WORKER_PROTOCOL_VERSION),
26983 "unexpected protocol for {path}"
26984 );
26985 }
26986
26987 assert_eq!(
26988 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
26989 1
26990 );
26991 assert_eq!(
26992 server.request_body("/api/worker/workflow-tasks/poll")["history_page_size"],
26993 500
26994 );
26995 assert_eq!(
26996 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
26997 1
26998 );
26999 assert!(
27000 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
27001 .as_str()
27002 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
27003 );
27004 assert!(
27005 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
27006 .as_str()
27007 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
27008 );
27009 }
27010
27011 #[tokio::test]
27012 async fn query_task_endpoints_send_the_query_feature_protocol() {
27013 let server = MockWorkerServer::start();
27014 let client = Client::builder(server.base_url())
27015 .timeout(Duration::from_secs(2))
27016 .build()
27017 .expect("client");
27018
27019 client
27020 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
27021 .await
27022 .expect("query poll");
27023 client
27024 .complete_query_task(
27025 "query-capture",
27026 "capture-worker",
27027 1,
27028 json!(8),
27029 DEFAULT_CODEC,
27030 )
27031 .await
27032 .expect("query complete");
27033 client
27034 .fail_query_task(
27035 "query-capture",
27036 "capture-worker",
27037 1,
27038 "failed",
27039 "query_rejected",
27040 "QueryFailed",
27041 )
27042 .await
27043 .expect("query fail");
27044
27045 for path in [
27046 "/api/worker/query-tasks/poll",
27047 "/api/worker/query-tasks/query-capture/complete",
27048 "/api/worker/query-tasks/query-capture/fail",
27049 ] {
27050 assert_eq!(
27051 server.worker_protocol_for(path).as_deref(),
27052 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
27053 "unexpected protocol for {path}"
27054 );
27055 }
27056
27057 assert_eq!(
27058 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
27059 1
27060 );
27061 assert!(
27062 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
27063 .as_str()
27064 .is_some_and(|id| id.starts_with("rust-query-poll-"))
27065 );
27066 }
27067
27068 #[tokio::test]
27069 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
27070 let server = MockWorkerServer::transient_worker_failures();
27071 let client = Client::builder(server.base_url())
27072 .timeout(Duration::from_secs(2))
27073 .build()
27074 .expect("client");
27075
27076 client
27077 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
27078 .await
27079 .expect("workflow poll retry");
27080 client
27081 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
27082 .await
27083 .expect("activity poll retry");
27084 client
27085 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
27086 .await
27087 .expect("query poll retry");
27088
27089 for path in [
27090 "/api/worker/workflow-tasks/poll",
27091 "/api/worker/activity-tasks/poll",
27092 "/api/worker/query-tasks/poll",
27093 ] {
27094 let bodies = server.request_bodies(path);
27095 assert_eq!(bodies.len(), 2, "{path} must be retried once");
27096 assert_eq!(
27097 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
27098 "{path} must preserve the request binding across retry"
27099 );
27100 }
27101 }
27102
27103 #[tokio::test]
27104 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
27105 let server = MockWorkerServer::consecutive_poll_failures(2);
27106 let client = Client::builder(server.base_url())
27107 .timeout(Duration::from_secs(2))
27108 .build()
27109 .expect("client");
27110 let mut worker = Worker::new(client, "capture")
27111 .worker_id("capture-worker")
27112 .poll_timeout(Duration::from_millis(10))
27113 .retry_policy(WorkerRetryPolicy {
27114 max_retries: 2,
27115 initial_backoff: Duration::from_millis(1),
27116 max_backoff: Duration::from_millis(1),
27117 });
27118 worker.register_workflow(
27119 "capture.workflow",
27120 |_ctx, _input| async move { Ok(Value::Null) },
27121 );
27122 worker.register_activity(
27123 "capture.activity",
27124 |_ctx, _input| async move { Ok(Value::Null) },
27125 );
27126 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
27127 Ok(Value::Null)
27128 });
27129
27130 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
27131
27132 for path in [
27133 "/api/worker/workflow-tasks/poll",
27134 "/api/worker/activity-tasks/poll",
27135 "/api/worker/query-tasks/poll",
27136 ] {
27137 let bodies = server.request_bodies(path);
27138 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
27139 assert!(
27140 bodies
27141 .iter()
27142 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
27143 "{path} must preserve one request binding across every retry"
27144 );
27145 }
27146 }
27147
27148 fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
27149 let reason = if unavailable {
27150 "storage_admission_unavailable"
27151 } else {
27152 "storage_pressure"
27153 };
27154 let mut body = json!({
27155 "reason": reason,
27156 "storage_state": if unavailable { "fenced" } else { "draining" },
27157 "retryable": true,
27158 "retry_after_seconds": 1,
27159 });
27160 if !mid_poll {
27161 body["request_admitted"] = json!(false);
27162 }
27163 if let Some(id) = poll_id {
27164 body["task"] = Value::Null;
27165 body["poll_status"] = json!(reason);
27166 body["poll_request_id"] = json!(id);
27167 body["retry_same_poll_request_id"] = json!(true);
27168 body["claim_admitted"] = json!(false);
27169 }
27170 body
27171 }
27172
27173 fn backend_refusal(path: &str, request: &str) -> Option<Value> {
27174 let operation = match path {
27175 "/api/worker/register" => "register_worker",
27176 "/api/worker/heartbeat" => "heartbeat_worker",
27177 "/api/worker/workflow-tasks/poll" => "poll_workflow_task",
27178 "/api/worker/activity-tasks/poll" => "poll_activity_task",
27179 "/api/worker/query-tasks/poll" => "poll_query_task",
27180 "/api/worker/update-validation-tasks/poll" => "poll_update_validation_task",
27181 _ => return None,
27182 };
27183 let request: Value = serde_json::from_str(request).ok()?;
27184 let mut response = json!({
27185 "reason": "backend_unavailable",
27186 "operation": operation,
27187 "outcome": "unknown",
27188 "retryable": true,
27189 "retry_after_seconds": 1,
27190 "worker_id": request["worker_id"],
27191 "task_queue": request.get("task_queue"),
27192 });
27193 if path.ends_with("/poll") {
27194 response["task"] = Value::Null;
27195 response["poll_status"] = json!("backend_unavailable");
27196 response["poll_request_id"] = request["poll_request_id"].clone();
27197 response["retry_same_poll_request_id"] = json!(true);
27198 }
27199 Some(response)
27200 }
27201
27202 fn backend_retry_override(
27203 path: &str,
27204 request: &str,
27205 number: usize,
27206 ) -> Option<(&'static str, String)> {
27207 if number > 8 {
27208 return None;
27209 }
27210 Some((
27211 "503 Service Unavailable",
27212 backend_refusal(path, request)?.to_string(),
27213 ))
27214 }
27215
27216 fn backend_unavailable_override(
27217 path: &str,
27218 request: &str,
27219 _number: usize,
27220 ) -> Option<(&'static str, String)> {
27221 Some((
27222 "503 Service Unavailable",
27223 backend_refusal(path, request)?.to_string(),
27224 ))
27225 }
27226
27227 fn storage_worker(server: &MockWorkerServer) -> Worker {
27228 Worker::new(Client::new(server.base_url()).expect("client"), "storage")
27229 .worker_id("storage-worker")
27230 .retry_policy(WorkerRetryPolicy {
27231 max_retries: 1,
27232 initial_backoff: Duration::from_millis(1),
27233 max_backoff: Duration::from_millis(1),
27234 })
27235 }
27236
27237 fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
27238 let requests = server.requests.lock().expect("requests");
27239 let bodies: Vec<_> = requests
27240 .iter()
27241 .filter(|request| request.path == path)
27242 .map(|request| &request.body)
27243 .collect();
27244 assert_eq!(bodies.len(), count, "{path}");
27245 assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
27246 }
27247
27248 #[test]
27249 fn backend_recovery_requires_the_explicit_worker_contract() {
27250 for (path, operation) in [
27251 ("/worker/register", "register_worker"),
27252 ("/worker/heartbeat", "heartbeat_worker"),
27253 ("/worker/workflow-tasks/poll", "poll_workflow_task"),
27254 ("/worker/activity-tasks/poll", "poll_activity_task"),
27255 ("/worker/query-tasks/poll", "poll_query_task"),
27256 (
27257 "/worker/update-validation-tasks/poll",
27258 "poll_update_validation_task",
27259 ),
27260 ] {
27261 let poll_id = path.ends_with("/poll").then_some("same-poll");
27262 let request = json!({"worker_id":"same-worker","task_queue":"same-queue","poll_request_id":poll_id});
27263 let body = backend_refusal(&format!("/api{path}"), &request.to_string())
27264 .expect("worker operation");
27265 assert_eq!(body["operation"], operation);
27266 let error = Error::Http {
27267 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27268 body: body.to_string(),
27269 };
27270 assert_eq!(
27271 worker_backend_unavailable_retry_after(&error, path, poll_id),
27272 Some(Duration::from_secs(1))
27273 );
27274 assert!(!worker_operation_is_retryable(&error));
27275 for (field, value) in [
27276 ("operation", json!("wrong_operation")),
27277 ("outcome", json!("completed")),
27278 ("retryable", json!(false)),
27279 ("retry_after_seconds", json!(0)),
27280 ("worker_id", json!("")),
27281 ] {
27282 let mut invalid = body.clone();
27283 invalid[field] = value;
27284 let error = Error::Http {
27285 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27286 body: invalid.to_string(),
27287 };
27288 assert!(
27289 worker_backend_unavailable_retry_after(&error, path, poll_id).is_none(),
27290 "{path}: {field}"
27291 );
27292 assert!(!worker_operation_is_retryable(&error));
27293 }
27294 if let Some(poll_id) = poll_id {
27295 for (field, value) in [
27296 ("poll_request_id", json!("wrong-poll")),
27297 ("poll_status", json!("empty")),
27298 ("task", json!({"task_id":"claimed"})),
27299 ("retry_same_poll_request_id", json!(false)),
27300 ] {
27301 let mut invalid = body.clone();
27302 invalid[field] = value;
27303 let error = Error::Http {
27304 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27305 body: invalid.to_string(),
27306 };
27307 assert!(
27308 worker_backend_unavailable_retry_after(&error, path, Some(poll_id))
27309 .is_none(),
27310 "{path}: {field}"
27311 );
27312 }
27313 }
27314 let error = Error::Http {
27315 status: reqwest::StatusCode::UNAUTHORIZED,
27316 body: body.to_string(),
27317 };
27318 assert!(worker_backend_unavailable_retry_after(&error, path, poll_id).is_none());
27319 assert!(!worker_operation_is_retryable(&error));
27320 }
27321 }
27322
27323 #[tokio::test]
27324 async fn backend_recovery_outlives_generic_retry_budget_and_preserves_polls() {
27325 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27326 request_override: Some(backend_retry_override),
27327 ..MockWorkerBehavior::default()
27328 });
27329 let mut worker =
27330 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
27331 worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
27332 worker.register_activity("backend.activity", |_, _| async { Ok(Value::Null) });
27333 worker.register_query("backend.workflow", "state", |_, _| async {
27334 Ok(Value::Null)
27335 });
27336 worker.register().await.expect("registration recovery");
27337 worker
27338 .client
27339 .heartbeat_worker("storage-worker", 1, 1)
27340 .await
27341 .expect("heartbeat recovery");
27342 assert_eq!(worker.run_once().await.expect("poll recovery"), 0);
27343 for path in [
27344 "/api/worker/register",
27345 "/api/worker/heartbeat",
27346 "/api/worker/workflow-tasks/poll",
27347 "/api/worker/activity-tasks/poll",
27348 "/api/worker/query-tasks/poll",
27349 ] {
27350 assert_identical_requests(&server, path, 9);
27351 }
27352 }
27353
27354 #[tokio::test]
27355 async fn backend_recovery_wait_stops_with_worker() {
27356 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27357 request_override: Some(backend_unavailable_override),
27358 ..MockWorkerBehavior::default()
27359 });
27360 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy {
27361 max_retries: 1,
27362 initial_backoff: Duration::from_secs(2),
27363 max_backoff: Duration::from_secs(2),
27364 });
27365 worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
27366 let started = tokio::time::Instant::now();
27367 let _ = tokio::time::timeout(
27368 Duration::from_secs(1),
27369 worker.run_until(tokio::time::sleep(Duration::from_millis(100))),
27370 )
27371 .await
27372 .expect("shutdown interrupts backend wait");
27373 assert!(started.elapsed() < Duration::from_secs(1));
27374 assert!(server.request_count("/api/worker/register") >= 1);
27375 }
27376
27377 #[test]
27378 fn storage_admission_requires_an_explicit_identity_preserving_contract() {
27379 for unavailable in [false, true] {
27380 for mid_poll in [false, true] {
27381 let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
27382 let error = Error::Http {
27383 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27384 body: body.to_string(),
27385 };
27386 assert_eq!(
27387 worker_storage_admission_retry_after(&error, Some("same-poll")),
27388 Some(Duration::from_secs(1))
27389 );
27390 assert!(
27391 !worker_operation_is_retryable(&error),
27392 "storage is not a bounded generic retry"
27393 );
27394 for (field, value) in [
27395 ("poll_request_id", json!("wrong-poll")),
27396 ("task", json!({"task_id":"claimed"})),
27397 ("retryable", json!(false)),
27398 ("retry_after_seconds", json!(0)),
27399 ("retry_after_seconds", json!(true)),
27400 ("retry_after_seconds", json!(1.0)),
27401 ("storage_state", json!("normal")),
27402 ("poll_status", json!("empty")),
27403 ("claim_admitted", json!(true)),
27404 ("retry_same_poll_request_id", json!(false)),
27405 ("request_admitted", json!(true)),
27406 ] {
27407 let mut invalid = body.clone();
27408 invalid[field] = value;
27409 let error = Error::Http {
27410 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27411 body: invalid.to_string(),
27412 };
27413 assert!(
27414 worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
27415 "{field}"
27416 );
27417 }
27418 }
27419 }
27420 let body = storage_refusal(None, false, false);
27421 let error = Error::Http {
27422 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27423 body: body.to_string(),
27424 };
27425 assert!(worker_storage_admission_retry_after(&error, None).is_some());
27426 assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
27427 let error = Error::Http {
27428 status: reqwest::StatusCode::FORBIDDEN,
27429 body: body.to_string(),
27430 };
27431 assert!(worker_storage_admission_retry_after(&error, None).is_none());
27432 let body = storage_refusal(None, false, true);
27433 let error = Error::Http {
27434 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27435 body: body.to_string(),
27436 };
27437 assert!(worker_storage_admission_retry_after(&error, None).is_none());
27438 }
27439
27440 #[tokio::test]
27441 async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
27442 for unavailable in [false, true] {
27443 for mid_poll in [false, true] {
27444 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27445 poll_failures_per_path: 1,
27446 storage_refusals: 7,
27447 storage_path: Some("/poll"),
27448 storage_unavailable: unavailable,
27449 storage_mid_poll: mid_poll,
27450 ..MockWorkerBehavior::default()
27451 });
27452 let mut worker = storage_worker(&server);
27453 worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
27454 assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
27455 for path in [
27456 "/api/worker/workflow-tasks/poll",
27457 "/api/worker/activity-tasks/poll",
27458 "/api/worker/query-tasks/poll",
27459 ] {
27460 assert_identical_requests(&server, path, 9);
27461 }
27462 }
27463 }
27464 }
27465
27466 #[tokio::test]
27467 async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
27468 struct CountedBody(Arc<AtomicUsize>);
27469 impl Serialize for CountedBody {
27470 fn serialize<S: Serializer>(
27471 &self,
27472 serializer: S,
27473 ) -> std::result::Result<S::Ok, S::Error> {
27474 let count = self.0.fetch_add(1, Ordering::SeqCst);
27475 json!({"serialization":count,"lease_owner":"worker","attempt":7})
27476 .serialize(serializer)
27477 }
27478 }
27479 for path in [
27480 "/api/worker/register",
27481 "/api/worker/heartbeat",
27482 "/api/worker/workflow-tasks/storage-task/complete",
27483 "/api/worker/workflow-tasks/storage-task/fail",
27484 "/api/worker/activity-tasks/storage-task/complete",
27485 "/api/worker/activity-tasks/storage-task/fail",
27486 "/api/worker/activity-tasks/storage-task/heartbeat",
27487 "/api/worker/query-tasks/storage-task/complete",
27488 "/api/worker/query-tasks/storage-task/fail",
27489 ] {
27490 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27491 storage_refusals: 7,
27492 storage_path: Some(path),
27493 ..MockWorkerBehavior::default()
27494 });
27495 let worker =
27496 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
27497 let calls = Arc::new(AtomicUsize::new(0));
27498 let _: Value = worker
27499 .client
27500 .request_json(
27501 reqwest::Method::POST,
27502 &path[4..],
27503 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
27504 Some(&CountedBody(Arc::clone(&calls))),
27505 )
27506 .await
27507 .expect("prepared request recovery");
27508 assert_eq!(calls.load(Ordering::SeqCst), 1);
27509 assert_identical_requests(&server, path, 8);
27510 }
27511 for worker_scope in [false, true] {
27512 let path = if worker_scope {
27513 "/api/health"
27514 } else {
27515 "/api/worker/register"
27516 };
27517 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27518 storage_refusals: usize::MAX,
27519 storage_path: Some(path),
27520 ..MockWorkerBehavior::default()
27521 });
27522 let worker = storage_worker(&server);
27523 let client = worker.client.clone();
27524 let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
27525 let error = if worker_scope {
27526 worker
27527 .client
27528 .health()
27529 .await
27530 .expect_err("control plane is not retried")
27531 } else {
27532 client
27533 .request_json::<Value, Value>(
27534 reqwest::Method::POST,
27535 "/worker/register",
27536 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
27537 Some(&json!({})),
27538 )
27539 .await
27540 .expect_err("direct client is not retried")
27541 };
27542 assert!(worker_storage_admission_body(&error).is_some());
27543 assert_eq!(server.request_count(path), 1);
27544 }
27545 }
27546
27547 #[tokio::test]
27548 async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
27549 for fail in [false, true] {
27550 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27551 storage_activity: true,
27552 storage_refusals: 7,
27553 storage_path: Some("/storage-activity/"),
27554 ..MockWorkerBehavior::default()
27555 });
27556 let mut worker = storage_worker(&server);
27557 let calls = Arc::new(AtomicUsize::new(0));
27558 let observed = Arc::clone(&calls);
27559 worker.register_activity("storage.activity", move |ctx, _| {
27560 let calls = Arc::clone(&observed);
27561 async move {
27562 calls.fetch_add(1, Ordering::SeqCst);
27563 ctx.heartbeat(json!({"step":2})).await?;
27564 if fail {
27565 Err(Error::WorkerLoop("intentional handler failure".to_string()))
27566 } else {
27567 Ok(json!({"receipt":true}))
27568 }
27569 }
27570 });
27571 assert_eq!(worker.run_once().await.expect("activity settled"), 1);
27572 assert_eq!(calls.load(Ordering::SeqCst), 1);
27573 assert_identical_requests(
27574 &server,
27575 "/api/worker/activity-tasks/storage-activity/heartbeat",
27576 8,
27577 );
27578 let suffix = if fail { "fail" } else { "complete" };
27579 assert_identical_requests(
27580 &server,
27581 &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
27582 8,
27583 );
27584 let other = if fail { "complete" } else { "fail" };
27585 assert_eq!(
27586 server.request_count(&format!(
27587 "/api/worker/activity-tasks/storage-activity/{other}"
27588 )),
27589 0
27590 );
27591 }
27592 }
27593
27594 #[tokio::test]
27595 async fn storage_waits_are_interruptible_without_false_activity_failure() {
27596 for path in [
27597 "/api/worker/register",
27598 "/api/worker/heartbeat",
27599 "/api/worker/activity-tasks/poll",
27600 "/api/worker/activity-tasks/storage-activity/heartbeat",
27601 "/api/worker/activity-tasks/storage-activity/complete",
27602 ] {
27603 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27604 storage_activity: true,
27605 storage_refusals: usize::MAX,
27606 storage_path: Some(path),
27607 ..MockWorkerBehavior::default()
27608 });
27609 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
27610 worker.register_activity("storage.activity", |ctx, _| async move {
27611 ctx.heartbeat(json!({"step":2})).await?;
27612 Ok(json!({"receipt":true}))
27613 });
27614 let shutdown = async {
27615 while server.request_count(path) == 0 {
27616 tokio::time::sleep(Duration::from_millis(1)).await;
27617 }
27618 };
27619 let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
27620 .await
27621 .expect("shutdown interrupts admission");
27622 assert!(
27623 result.is_err(),
27624 "a refused operation must not appear acknowledged: {path}, {result:?}"
27625 );
27626 assert_eq!(server.request_count(path), 1);
27627 assert_eq!(
27628 server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
27629 0
27630 );
27631 assert_eq!(
27632 server.request_count("/api/worker/registrations/mock-worker"),
27633 usize::from(!path.ends_with("/register"))
27634 );
27635 }
27636 }
27637
27638 #[tokio::test]
27639 async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
27640 for fail in [false, true] {
27641 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27642 storage_query: true,
27643 storage_refusals: 7,
27644 storage_path: Some("/storage-query/"),
27645 ..MockWorkerBehavior::default()
27646 });
27647 let mut worker = storage_worker(&server);
27648 worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
27649 let calls = Arc::new(AtomicUsize::new(0));
27650 let observed = Arc::clone(&calls);
27651 worker.register_query("storage.workflow", "state", move |_, _| {
27652 let calls = Arc::clone(&observed);
27653 async move {
27654 calls.fetch_add(1, Ordering::SeqCst);
27655 if fail {
27656 Err(Error::WorkerLoop("intentional query failure".to_string()))
27657 } else {
27658 Ok(json!({"state":"waiting"}))
27659 }
27660 }
27661 });
27662 assert_eq!(worker.run_once().await.expect("query settled"), 1);
27663 assert_eq!(calls.load(Ordering::SeqCst), 1);
27664 let suffix = if fail { "fail" } else { "complete" };
27665 assert_identical_requests(
27666 &server,
27667 &format!("/api/worker/query-tasks/storage-query/{suffix}"),
27668 8,
27669 );
27670 let other = if fail { "complete" } else { "fail" };
27671 assert_eq!(
27672 server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
27673 0
27674 );
27675 }
27676 }
27677
27678 #[tokio::test]
27679 async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
27680 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27681 storage_refusals: 7,
27682 storage_path: Some("/poll"),
27683 unauthorized_polls: true,
27684 ..MockWorkerBehavior::default()
27685 });
27686 let error = storage_worker(&server)
27687 .run_once()
27688 .await
27689 .expect_err("auth remains terminal");
27690 assert!(matches!(
27691 error,
27692 Error::Http {
27693 status: reqwest::StatusCode::UNAUTHORIZED,
27694 ..
27695 }
27696 ));
27697 assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
27698
27699 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27700 storage_refusals: 7,
27701 storage_path: Some("/activity-cancel/complete"),
27702 ..MockWorkerBehavior::default()
27703 });
27704 let worker =
27705 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
27706 let error = worker
27707 .client
27708 .complete_activity_task(
27709 "activity-cancel",
27710 "attempt-cancel",
27711 "worker",
27712 json!({}),
27713 DEFAULT_CODEC,
27714 )
27715 .await
27716 .expect_err("cancellation remains terminal");
27717 assert!(activity_task_rejection_is_final(&error));
27718 assert_identical_requests(
27719 &server,
27720 "/api/worker/activity-tasks/activity-cancel/complete",
27721 8,
27722 );
27723
27724 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27725 storage_refusals: usize::MAX,
27726 storage_path: Some("/poll"),
27727 storage_wrong_poll_id: true,
27728 ..MockWorkerBehavior::default()
27729 });
27730 assert!(storage_worker(&server).run_once().await.is_err());
27731 assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
27732 }
27733
27734 #[tokio::test]
27735 async fn storage_pollers_stop_when_the_run_future_is_aborted() {
27736 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
27737 storage_refusals: usize::MAX,
27738 storage_path: Some("/poll"),
27739 ..MockWorkerBehavior::default()
27740 });
27741 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
27742 worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
27743 let run = tokio::spawn(async move { worker.run().await });
27744 tokio::time::timeout(Duration::from_secs(2), async {
27745 while server.request_count("/api/worker/activity-tasks/poll") == 0 {
27746 tokio::time::sleep(Duration::from_millis(1)).await;
27747 }
27748 })
27749 .await
27750 .expect("poll started");
27751 run.abort();
27752 assert!(run.await.expect_err("cancelled run").is_cancelled());
27753 tokio::time::sleep(Duration::from_millis(250)).await;
27754 assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
27755 }
27756
27757 #[tokio::test]
27758 async fn query_protocol_rejection_from_older_server_is_typed() {
27759 let server = MockWorkerServer::reject_query_protocol();
27760 let client = Client::builder(server.base_url())
27761 .timeout(Duration::from_secs(2))
27762 .build()
27763 .expect("client");
27764
27765 let error = client
27766 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
27767 .await
27768 .expect_err("server below query protocol floor must reject");
27769 let Error::Protocol(failure) = error else {
27770 panic!("expected typed protocol failure");
27771 };
27772
27773 assert_eq!(failure.status, 400);
27774 assert_eq!(failure.reason, "unsupported_protocol_version");
27775 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
27776 assert_eq!(
27777 failure.requested_version.as_deref(),
27778 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
27779 );
27780 assert_eq!(
27781 server
27782 .worker_protocol_for("/api/worker/query-tasks/poll")
27783 .as_deref(),
27784 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
27785 );
27786 }
27787
27788 #[tokio::test]
27789 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
27790 let server = MockWorkerServer::reject_query_protocol();
27791 let client = Client::builder(server.base_url())
27792 .timeout(Duration::from_secs(2))
27793 .build()
27794 .expect("client");
27795 let mut worker = Worker::new(client, "rust-workers")
27796 .worker_id("baseline-worker")
27797 .poll_timeout(Duration::from_millis(10));
27798
27799 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
27800 Ok(Value::Null)
27801 });
27802
27803 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
27804 assert_eq!(
27805 server
27806 .worker_protocol_for("/api/worker/workflow-tasks/poll")
27807 .as_deref(),
27808 Some(WORKER_PROTOCOL_VERSION)
27809 );
27810 assert_eq!(
27811 server.worker_protocol_for("/api/worker/query-tasks/poll"),
27812 None,
27813 "a worker without query handlers must not use the query-task endpoint"
27814 );
27815 }
27816
27817 #[tokio::test]
27818 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
27819 let server = MockWorkerServer::reject_query_completion();
27820 let client = Client::builder(server.base_url())
27821 .timeout(Duration::from_secs(2))
27822 .build()
27823 .expect("client");
27824
27825 let error = client
27826 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
27827 .await
27828 .expect_err("expired completion must be rejected");
27829 let Error::QueryFailed(failure) = error else {
27830 panic!("expected typed query failure");
27831 };
27832 assert_eq!(failure.status, 409);
27833 assert_eq!(failure.reason, "query_task_timed_out");
27834
27835 let mut worker = Worker::new(client, "rust-workers")
27836 .worker_id("late-worker")
27837 .poll_timeout(Duration::from_millis(10));
27838 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
27839 worker.register_query(
27840 "counter",
27841 "current",
27842 |_ctx, _args| async move { Ok(json!(8)) },
27843 );
27844
27845 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
27846 assert_eq!(
27847 worker
27848 .run_once()
27849 .await
27850 .expect("worker continues after late completion"),
27851 0
27852 );
27853 assert_eq!(
27854 server.request_count("/api/worker/query-tasks/query-late/complete"),
27855 2
27856 );
27857 assert_eq!(
27858 server.request_count("/api/worker/query-tasks/query-late/fail"),
27859 0,
27860 "a server completion rejection must not be reported as an encoding failure"
27861 );
27862 }
27863
27864 #[tokio::test]
27865 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
27866 let server = MockWorkerServer::start();
27867 let client = Client::builder(server.base_url())
27868 .timeout(Duration::from_secs(2))
27869 .build()
27870 .expect("client");
27871 let mut worker = Worker::new(client, "rust-workers")
27872 .worker_id("joined-worker")
27873 .poll_timeout(Duration::from_millis(10));
27874 worker.register_workflow(
27875 "joined.workflow",
27876 |_ctx, _input| async move { Ok(Value::Null) },
27877 );
27878 worker.register_activity(
27879 "joined.activity",
27880 |_ctx, _input| async move { Ok(Value::Null) },
27881 );
27882 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
27883 Ok(Value::Null)
27884 });
27885
27886 worker
27887 .run_until(tokio::time::sleep(Duration::from_millis(20)))
27888 .await
27889 .expect("normal shutdown");
27890
27891 let deregistration_path = "/api/worker/registrations/mock-worker";
27892 assert_eq!(server.request_count(deregistration_path), 1);
27893 for poll_path in [
27894 "/api/worker/workflow-tasks/poll",
27895 "/api/worker/activity-tasks/poll",
27896 "/api/worker/query-tasks/poll",
27897 ] {
27898 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
27899 }
27900 assert_eq!(
27901 server.captured_paths().last().map(String::as_str),
27902 Some(deregistration_path),
27903 "deregistration must start only after every poller has joined"
27904 );
27905 }
27906
27907 #[tokio::test]
27908 async fn registration_failure_does_not_deregister() {
27909 let server = MockWorkerServer::rejected_registration();
27910 let client = Client::builder(server.base_url())
27911 .timeout(Duration::from_secs(2))
27912 .build()
27913 .expect("client");
27914 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
27915
27916 let error = worker
27917 .run_until(async {})
27918 .await
27919 .expect_err("registration must fail");
27920 assert!(matches!(
27921 error,
27922 Error::Http {
27923 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
27924 ..
27925 }
27926 ));
27927 assert!(server
27928 .captured_paths()
27929 .iter()
27930 .all(|path| !path.starts_with("/api/worker/registrations/")));
27931 }
27932
27933 #[tokio::test]
27934 async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
27935 let server = MockWorkerServer::rejected_registration_protocol();
27936 let client = Client::builder(server.base_url())
27937 .timeout(Duration::from_secs(2))
27938 .build()
27939 .expect("client");
27940 let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
27941
27942 let error = worker
27943 .run_until(async {})
27944 .await
27945 .expect_err("a protocol 1.16 server must reject this worker");
27946 let Error::Protocol(failure) = error else {
27947 panic!("expected typed protocol rejection");
27948 };
27949 assert_eq!(failure.reason, "unsupported_protocol_version");
27950 assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
27951 assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
27952 assert_eq!(
27953 server
27954 .worker_protocol_for("/api/worker/register")
27955 .as_deref(),
27956 Some(WORKER_PROTOCOL_VERSION)
27957 );
27958 }
27959
27960 #[tokio::test]
27961 async fn declined_registration_does_not_deregister() {
27962 let server = MockWorkerServer::declined_registration();
27963 let client = Client::builder(server.base_url())
27964 .timeout(Duration::from_secs(2))
27965 .build()
27966 .expect("client");
27967 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
27968
27969 let error = worker
27970 .run_until(async {})
27971 .await
27972 .expect_err("declined registration must fail");
27973 assert!(matches!(error, Error::WorkerLoop(_)));
27974 assert!(error.to_string().contains("was not accepted"));
27975 assert!(server
27976 .captured_paths()
27977 .iter()
27978 .all(|path| !path.starts_with("/api/worker/registrations/")));
27979 }
27980
27981 #[tokio::test]
27982 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
27983 let server = MockWorkerServer::rejected_deregistration();
27984 let client = Client::builder(server.base_url())
27985 .timeout(Duration::from_secs(2))
27986 .build()
27987 .expect("client");
27988 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
27989
27990 let error = worker
27991 .run_until(async {})
27992 .await
27993 .expect_err("deregistration must fail");
27994 assert!(matches!(
27995 error,
27996 Error::Http {
27997 status: reqwest::StatusCode::FORBIDDEN,
27998 ..
27999 }
28000 ));
28001 assert_eq!(
28002 server.request_count("/api/worker/registrations/mock-worker"),
28003 1
28004 );
28005 }
28006
28007 #[tokio::test]
28008 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
28009 let server = MockWorkerServer::rejected_deregistration_protocol();
28010 let client = Client::builder(server.base_url())
28011 .timeout(Duration::from_secs(2))
28012 .build()
28013 .expect("client");
28014 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
28015
28016 let error = worker
28017 .run_until(async {})
28018 .await
28019 .expect_err("protocol rejection must fail shutdown");
28020 let Error::Protocol(failure) = error else {
28021 panic!("expected typed protocol failure");
28022 };
28023 assert_eq!(failure.reason, "unsupported_protocol_version");
28024 assert_eq!(
28025 failure.requested_version.as_deref(),
28026 Some(WORKER_PROTOCOL_VERSION)
28027 );
28028 assert_eq!(
28029 server.request_count("/api/worker/registrations/mock-worker"),
28030 1
28031 );
28032 }
28033
28034 #[tokio::test]
28035 async fn primary_poller_error_retains_deregistration_failure_context() {
28036 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
28037 let client = Client::builder(server.base_url())
28038 .timeout(Duration::from_secs(2))
28039 .build()
28040 .expect("client");
28041 let mut worker = Worker::new(client, "rust-workers")
28042 .worker_id("combined-failure")
28043 .poll_timeout(Duration::from_millis(10));
28044 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
28045 Ok(Value::Null)
28046 });
28047
28048 let error = worker
28049 .run()
28050 .await
28051 .expect_err("worker and cleanup must fail");
28052 let summary = error.to_string();
28053 assert!(summary.contains("authentication_failed"));
28054 assert!(summary.contains("worker cannot deregister"));
28055 let Error::WorkerShutdown {
28056 primary,
28057 deregistration,
28058 } = error
28059 else {
28060 panic!("expected combined worker shutdown error");
28061 };
28062 assert!(matches!(
28063 *primary,
28064 Error::Http {
28065 status: reqwest::StatusCode::UNAUTHORIZED,
28066 ..
28067 }
28068 ));
28069 assert!(matches!(
28070 *deregistration,
28071 Error::Http {
28072 status: reqwest::StatusCode::FORBIDDEN,
28073 ..
28074 }
28075 ));
28076 assert_eq!(
28077 server.request_count("/api/worker/registrations/mock-worker"),
28078 1
28079 );
28080 }
28081
28082 #[tokio::test]
28083 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
28084 let server = MockWorkerServer::start();
28085 let client = Client::builder(server.base_url())
28086 .timeout(Duration::from_secs(2))
28087 .build()
28088 .expect("client");
28089 let mut worker = Worker::new(client, "rust-workers")
28090 .worker_id("activity-only-worker")
28091 .poll_timeout(Duration::from_millis(10));
28092
28093 worker.register_activity(
28094 "activity.only",
28095 |_ctx, _args| async move { Ok(Value::Null) },
28096 );
28097
28098 worker.run_until(async {}).await.expect("run worker");
28099 }
28100
28101 #[tokio::test]
28102 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
28103 let server = MockWorkerServer::start();
28104 let client = Client::builder(server.base_url())
28105 .timeout(Duration::from_secs(2))
28106 .build()
28107 .expect("client");
28108 let mut worker = Worker::new(client, "rust-workers")
28109 .worker_id("workflow-only-worker")
28110 .poll_timeout(Duration::from_millis(10));
28111
28112 worker.register_workflow(
28113 "workflow.only",
28114 |_ctx, _input| async move { Ok(Value::Null) },
28115 );
28116
28117 worker.run_until(async {}).await.expect("run worker");
28118 }
28119
28120 #[tokio::test]
28121 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
28122 let server = MockWorkerServer::start();
28123 let client = Client::builder(server.base_url())
28124 .timeout(Duration::from_secs(2))
28125 .build()
28126 .expect("client");
28127 let observations = Arc::new(Mutex::new(Vec::new()));
28128 let observed = Arc::clone(&observations);
28129 let mut worker = Worker::new(client, "rust-workers")
28130 .worker_id("observed-heartbeat-worker")
28131 .poll_timeout(Duration::from_millis(10))
28132 .on_worker_heartbeat(move |observation| {
28133 observed
28134 .lock()
28135 .expect("heartbeat observations")
28136 .push(observation.clone());
28137 });
28138
28139 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
28140 Ok(Value::Null)
28141 });
28142 let acknowledged = Arc::clone(&observations);
28143 worker
28144 .run_until(async move {
28145 tokio::time::timeout(Duration::from_secs(2), async move {
28146 loop {
28147 if !acknowledged
28148 .lock()
28149 .expect("heartbeat observations")
28150 .is_empty()
28151 {
28152 break;
28153 }
28154 tokio::time::sleep(Duration::from_millis(1)).await;
28155 }
28156 })
28157 .await
28158 .expect("heartbeat acknowledgement within timeout");
28159 })
28160 .await
28161 .expect("run worker");
28162
28163 let observations = observations.lock().expect("heartbeat observations");
28164 let first = observations.first().expect("heartbeat acknowledgement");
28165 assert_eq!(first.worker_id, "observed-heartbeat-worker");
28166 assert_eq!(first.task_queue, "rust-workers");
28167 assert!(first.acknowledged_at_unix_millis > 0);
28168 assert_eq!(first.acknowledgement, json!({}));
28169 }
28170
28171 #[tokio::test]
28172 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
28173 let server = MockWorkerServer::delayed_heartbeat_worker();
28174 let client = Client::builder(server.base_url())
28175 .timeout(Duration::from_secs(3))
28176 .build()
28177 .expect("client");
28178 let observations = Arc::new(Mutex::new(Vec::new()));
28179 let observed = Arc::clone(&observations);
28180 let mut worker = Worker::new(client, "rust-snapshot-workers")
28181 .worker_id("rust-snapshot-worker")
28182 .poll_timeout(Duration::from_millis(10))
28183 .on_worker_heartbeat(move |observation| {
28184 observed
28185 .lock()
28186 .expect("heartbeat observations")
28187 .push(observation.clone());
28188 });
28189
28190 worker.register_workflow("snapshot", |ctx, _input| async move {
28191 ctx.wait_signal("finish").await?;
28192 Ok(json!({"status": "finished"}))
28193 });
28194 worker.register_query("snapshot", "current", |ctx, _args| async move {
28195 Ok(json!(ctx
28196 .signals("increment")
28197 .iter()
28198 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
28199 .sum::<i64>()))
28200 });
28201 worker.register_activity("cancel-aware", |_ctx, _args| async move {
28202 Ok(json!({"late": "completion"}))
28203 });
28204
28205 worker
28206 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
28207 .await
28208 .expect("delayed heartbeat must allow a clean worker shutdown");
28209
28210 let observations = observations.lock().expect("heartbeat observations");
28211 assert!(
28212 observations.len() >= 3,
28213 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
28214 );
28215 assert!(
28216 observations.windows(2).all(|pair| {
28217 pair[1].acknowledged_at_unix_millis
28218 .saturating_sub(pair[0].acknowledged_at_unix_millis)
28219 >= 850
28220 }),
28221 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
28222 );
28223 drop(observations);
28224
28225 let heartbeat_times = server.request_times("/api/worker/heartbeat");
28226 let delayed_request_at = *heartbeat_times
28227 .get(1)
28228 .expect("intentionally delayed heartbeat request");
28229 let delay_window_start = delayed_request_at + Duration::from_millis(100);
28230 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
28231 for path in [
28232 "/api/worker/workflow-tasks/poll",
28233 "/api/worker/activity-tasks/poll",
28234 "/api/worker/query-tasks/poll",
28235 ] {
28236 assert!(
28237 server
28238 .request_times(path)
28239 .iter()
28240 .any(|received_at| *received_at >= delay_window_start
28241 && *received_at <= delay_window_end),
28242 "{path} must keep polling while a heartbeat acknowledgement is delayed"
28243 );
28244 }
28245 assert!(
28246 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
28247 "workflow work must be settled"
28248 );
28249 assert!(
28250 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
28251 "activity work must be settled"
28252 );
28253 assert!(
28254 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
28255 "query work must be settled"
28256 );
28257 }
28258
28259 #[tokio::test]
28260 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
28261 let server = MockWorkerServer::heartbeat_retry_worker();
28262 let client = Client::builder(server.base_url())
28263 .timeout(Duration::from_secs(2))
28264 .build()
28265 .expect("client");
28266 let observations = Arc::new(Mutex::new(Vec::new()));
28267 let observed = Arc::clone(&observations);
28268 let worker = Worker::new(client, "rust-workers")
28269 .worker_id("heartbeat-retry-worker")
28270 .retry_policy(WorkerRetryPolicy {
28271 max_retries: 1,
28272 initial_backoff: Duration::from_millis(300),
28273 max_backoff: Duration::from_millis(300),
28274 })
28275 .on_worker_heartbeat(move |observation| {
28276 observed
28277 .lock()
28278 .expect("heartbeat observations")
28279 .push(observation.clone());
28280 });
28281
28282 worker
28283 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
28284 .await
28285 .expect("retryable heartbeat failure must remain bounded and recover");
28286
28287 let observations = observations.lock().expect("heartbeat observations");
28288 assert!(observations.len() >= 3, "heartbeat retry must recover");
28289 assert!(
28290 observations.windows(2).all(|pair| {
28291 pair[1]
28292 .acknowledged_at_unix_millis
28293 .saturating_sub(pair[0].acknowledged_at_unix_millis)
28294 >= 850
28295 }),
28296 "a successful retry must start a fresh advertised cadence: {observations:?}"
28297 );
28298 assert_eq!(
28299 server.request_count("/api/worker/heartbeat"),
28300 observations.len() + 1,
28301 "one retryable failure must add exactly one bounded request"
28302 );
28303 }
28304
28305 #[tokio::test]
28306 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
28307 let server = MockWorkerServer::waiting_query_worker();
28308 let client = Client::builder(server.base_url())
28309 .timeout(Duration::from_secs(2))
28310 .build()
28311 .expect("client");
28312 let observations = Arc::new(Mutex::new(Vec::new()));
28313 let observed = Arc::clone(&observations);
28314 let mut worker = Worker::new(client, "rust-snapshot-workers")
28315 .worker_id("rust-snapshot-worker")
28316 .poll_timeout(Duration::from_millis(10))
28317 .on_worker_heartbeat(move |observation| {
28318 observed
28319 .lock()
28320 .expect("heartbeat observations")
28321 .push(observation.clone());
28322 });
28323
28324 worker.register_workflow("snapshot", |ctx, _input| async move {
28325 ctx.wait_signal("finish").await?;
28326 Ok(json!({"status": "finished"}))
28327 });
28328 worker.register_query("snapshot", "current", |ctx, _args| async move {
28329 let current = ctx
28330 .signals("increment")
28331 .iter()
28332 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
28333 .sum::<i64>();
28334 Ok(json!(current))
28335 });
28336 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
28337 worker
28338 .declare_workflow_signals("snapshot", &["finish"])
28339 .expect("declare finish signal");
28340
28341 worker
28342 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
28343 .await
28344 .expect("pending workflow and query poller must remain live until shutdown");
28345
28346 assert!(
28347 observations.lock().expect("heartbeat observations").len() >= 4,
28348 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
28349 );
28350 assert!(
28351 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
28352 "workflow polling must continue after empty replay acknowledgements"
28353 );
28354 assert!(
28355 server.request_count("/api/worker/query-tasks/poll") >= 2,
28356 "query polling must continue after serving the current query"
28357 );
28358 assert_eq!(
28359 server.request_body("/api/worker/register")["capabilities"],
28360 json!([
28361 CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
28362 DURABLE_SELECTION_CAPABILITY,
28363 MEMO_UPSERTS_CAPABILITY,
28364 TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
28365 QUERY_TASKS_CAPABILITY,
28366 WORKFLOW_UPDATES_CAPABILITY,
28367 MESSAGE_STREAMS_CAPABILITY
28368 ])
28369 );
28370 assert_eq!(
28371 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
28372 json!({
28373 "queries": ["current"],
28374 "query_contracts": [{
28375 "name": "current",
28376 "parameters": [{
28377 "name": "args", "position": 0, "required": false,
28378 "variadic": true, "type": null, "allows_null": true,
28379 "default_available": false, "default": null,
28380 }],
28381 }],
28382 "signals": ["finish"],
28383 "signal_contracts": [{
28384 "name": "finish",
28385 "parameters": [{
28386 "name": "args", "position": 0, "required": false,
28387 "variadic": true, "type": null, "allows_null": true,
28388 "default_available": false, "default": null,
28389 }],
28390 }],
28391 "updates": ["replace"],
28392 "update_contracts": [{
28393 "name": "replace",
28394 "parameters": [{
28395 "name": "args", "position": 0, "required": false,
28396 "variadic": true, "type": null, "allows_null": true,
28397 "default_available": false, "default": null,
28398 }],
28399 }],
28400 "update_validators": [],
28401 })
28402 );
28403
28404 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
28405 assert_eq!(
28406 opened["commands"],
28407 json!([{
28408 "type": "open_signal_wait",
28409 "signal_name": "finish",
28410 }])
28411 );
28412
28413 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
28414 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
28415 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
28416 let failure = server.request_body(&fail_path);
28417 assert_eq!(
28418 failure["failure"]["type"],
28419 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
28420 );
28421 assert_eq!(server.request_count(&completion_path), 0);
28422 }
28423
28424 let query_completion =
28425 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
28426 assert_eq!(query_completion["result"], json!(8));
28427
28428 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
28429 assert_eq!(
28430 server.request_count(terminal_path),
28431 1,
28432 "the matching signal must settle the workflow exactly once"
28433 );
28434 let terminal = server.request_body(terminal_path);
28435 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
28436 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
28437 assert_eq!(
28438 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
28439 .expect("terminal workflow result"),
28440 json!({"status": "finished"})
28441 );
28442 }
28443
28444 #[tokio::test]
28445 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
28446 let server = MockWorkerServer::transient_worker_failures();
28447 let client = Client::builder(server.base_url())
28448 .timeout(Duration::from_secs(2))
28449 .build()
28450 .expect("client");
28451 let mut worker = Worker::new(client, "rust-workers")
28452 .worker_id("retry-worker")
28453 .poll_timeout(Duration::from_millis(10))
28454 .retry_policy(WorkerRetryPolicy {
28455 max_retries: 2,
28456 initial_backoff: Duration::from_millis(1),
28457 max_backoff: Duration::from_millis(1),
28458 });
28459 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28460 worker.register_activity(
28461 "counter.activity",
28462 |_ctx, _input| async move { Ok(Value::Null) },
28463 );
28464 worker.register_query(
28465 "counter",
28466 "current",
28467 |_ctx, _args| async move { Ok(json!(8)) },
28468 );
28469
28470 worker
28471 .run_until(tokio::time::sleep(Duration::from_millis(75)))
28472 .await
28473 .expect("transient failures must not stop the worker");
28474
28475 for path in [
28476 "/api/worker/heartbeat",
28477 "/api/worker/workflow-tasks/poll",
28478 "/api/worker/activity-tasks/poll",
28479 "/api/worker/query-tasks/poll",
28480 ] {
28481 assert!(
28482 server.request_count(path) >= 2,
28483 "{path} must continue after its transient failure"
28484 );
28485 }
28486 }
28487
28488 #[tokio::test]
28489 async fn worker_continues_after_long_poll_capacity_backpressure() {
28490 let server = MockWorkerServer::capacity_limited_activity_poll();
28491 let client = Client::builder(server.base_url())
28492 .timeout(Duration::from_secs(2))
28493 .build()
28494 .expect("client");
28495 let mut worker = Worker::new(client, "rust-workers")
28496 .worker_id("capacity-worker")
28497 .poll_timeout(Duration::from_millis(10))
28498 .retry_policy(WorkerRetryPolicy {
28499 max_retries: 0,
28500 initial_backoff: Duration::from_millis(1),
28501 max_backoff: Duration::from_millis(1),
28502 });
28503 worker.register_activity("capacity.activity", |_ctx, _input| async move {
28504 Ok(json!({"handled": true}))
28505 });
28506
28507 worker
28508 .run_until(tokio::time::sleep(Duration::from_millis(50)))
28509 .await
28510 .expect("capacity backpressure must not stop the worker");
28511
28512 assert!(
28513 server.request_count("/api/worker/activity-tasks/poll") >= 2,
28514 "the activity poller must continue after capacity backpressure"
28515 );
28516 assert_eq!(
28517 server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
28518 1,
28519 "the worker must complete work returned after capacity recovers"
28520 );
28521 }
28522
28523 #[test]
28524 fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
28525 let capacity = Error::Http {
28526 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
28527 body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
28528 };
28529 assert_eq!(
28530 worker_poll_capacity_retry_after(&capacity),
28531 Some(Duration::from_secs(3))
28532 );
28533
28534 let rejected_capacity = Error::Http {
28535 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
28536 body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
28537 };
28538 assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
28539 assert!(!worker_operation_is_retryable(&rejected_capacity));
28540
28541 let ordinary_rate_limit = Error::Http {
28542 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
28543 body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
28544 .to_string(),
28545 };
28546 assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
28547 assert!(worker_operation_is_retryable(&ordinary_rate_limit));
28548 }
28549
28550 #[tokio::test]
28551 async fn worker_recovers_all_poll_families_and_heartbeat_beyond_retry_budget() {
28552 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
28553 poll_failures_per_path: 8,
28554 heartbeat_failures: 8,
28555 ..MockWorkerBehavior::default()
28556 });
28557 let client = Client::builder(server.base_url())
28558 .timeout(Duration::from_secs(2))
28559 .build()
28560 .expect("client");
28561 let acknowledgements = Arc::new(AtomicUsize::new(0));
28562 let observed = Arc::clone(&acknowledgements);
28563 let mut worker = Worker::new(client, "rust-workers")
28564 .recover_transient_outages(true)
28565 .poll_timeout(Duration::from_millis(10))
28566 .retry_policy(WorkerRetryPolicy {
28567 max_retries: 2,
28568 initial_backoff: Duration::from_millis(2),
28569 max_backoff: Duration::from_millis(5),
28570 })
28571 .on_worker_heartbeat(move |_| {
28572 observed.fetch_add(1, Ordering::SeqCst);
28573 });
28574 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28575 worker.register_activity(
28576 "counter.activity",
28577 |_ctx, _input| async move { Ok(Value::Null) },
28578 );
28579 worker.register_query("counter", "current", |_ctx, _args| async move {
28580 Ok(Value::Null)
28581 });
28582
28583 worker
28584 .run_until(tokio::time::sleep(Duration::from_millis(500)))
28585 .await
28586 .expect("a recoverable outage must not stop the service worker");
28587
28588 assert_eq!(acknowledgements.load(Ordering::SeqCst), 1);
28589 assert_eq!(server.request_count("/api/worker/heartbeat"), 9);
28590 for path in [
28591 "/api/worker/workflow-tasks/poll",
28592 "/api/worker/activity-tasks/poll",
28593 "/api/worker/query-tasks/poll",
28594 ] {
28595 let bodies = server.request_bodies(path);
28596 assert!(bodies.len() >= 10, "{path} must resume after recovery");
28597 let first = &bodies[0]["poll_request_id"];
28598 assert!(first.as_str().is_some_and(|id| !id.is_empty()));
28599 for body in &bodies[..9] {
28600 assert_eq!(&body["poll_request_id"], first, "{path} retry identity");
28601 }
28602 assert_ne!(&bodies[9]["poll_request_id"], first);
28603 let times = server.request_times(path);
28604 for pair in times[..9].windows(2) {
28605 assert!(pair[1].duration_since(pair[0]) >= Duration::from_millis(2));
28606 }
28607 }
28608 }
28609
28610 #[tokio::test]
28611 async fn worker_outage_retry_waits_are_interrupted_by_shutdown() {
28612 for heartbeat_outage in [false, true] {
28613 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
28614 poll_failures_per_path: if heartbeat_outage { 0 } else { usize::MAX },
28615 heartbeat_failures: if heartbeat_outage { usize::MAX } else { 0 },
28616 ..MockWorkerBehavior::default()
28617 });
28618 let client = Client::builder(server.base_url()).build().expect("client");
28619 let mut worker = Worker::new(client, "rust-workers")
28620 .recover_transient_outages(true)
28621 .retry_policy(WorkerRetryPolicy {
28622 max_retries: 1,
28623 initial_backoff: Duration::from_secs(5),
28624 max_backoff: Duration::from_secs(5),
28625 });
28626 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28627 let started = Instant::now();
28628 worker
28629 .run_until(tokio::time::sleep(Duration::from_millis(200)))
28630 .await
28631 .expect("shutdown during an outage is a normal stop");
28632 assert!(started.elapsed() < Duration::from_secs(1));
28633 let path = if heartbeat_outage {
28634 "/api/worker/heartbeat"
28635 } else {
28636 "/api/worker/workflow-tasks/poll"
28637 };
28638 assert_eq!(server.request_count(path), 1);
28639 assert_eq!(
28640 server.request_count("/api/worker/registrations/mock-worker"),
28641 1
28642 );
28643 }
28644 }
28645
28646 #[tokio::test]
28647 async fn worker_outage_recovery_keeps_run_once_bounded() {
28648 let server = MockWorkerServer::unavailable_polls();
28649 let client = Client::builder(server.base_url()).build().expect("client");
28650 let mut worker = Worker::new(client, "rust-workers")
28651 .recover_transient_outages(true)
28652 .retry_policy(WorkerRetryPolicy {
28653 max_retries: 2,
28654 initial_backoff: Duration::from_millis(1),
28655 max_backoff: Duration::from_millis(1),
28656 });
28657 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28658 assert!(matches!(worker.run_once().await, Err(Error::Transport(_))));
28659 assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 3);
28660 }
28661
28662 #[tokio::test]
28663 async fn worker_outage_recovery_respects_disabled_retries() {
28664 let server = MockWorkerServer::unavailable_polls();
28665 let client = Client::builder(server.base_url()).build().expect("client");
28666 let mut worker = Worker::new(client, "rust-workers")
28667 .recover_transient_outages(true)
28668 .retry_policy(WorkerRetryPolicy {
28669 max_retries: 0,
28670 ..WorkerRetryPolicy::default()
28671 });
28672 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28673 assert!(matches!(worker.run().await, Err(Error::Transport(_))));
28674 assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
28675 }
28676
28677 #[tokio::test]
28678 async fn worker_fatal_poll_error_interrupts_outage_heartbeat_retries() {
28679 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
28680 heartbeat_failures: usize::MAX,
28681 unauthorized_polls: true,
28682 ..MockWorkerBehavior::default()
28683 });
28684 let client = Client::builder(server.base_url()).build().expect("client");
28685 let mut worker = Worker::new(client, "rust-workers")
28686 .recover_transient_outages(true)
28687 .retry_policy(WorkerRetryPolicy {
28688 max_retries: 1,
28689 initial_backoff: Duration::from_secs(5),
28690 max_backoff: Duration::from_secs(5),
28691 });
28692 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28693 let error = tokio::time::timeout(Duration::from_secs(1), worker.run())
28694 .await
28695 .expect("fatal poll errors must interrupt heartbeat recovery")
28696 .expect_err("authentication failure is terminal");
28697 assert!(
28698 matches!(error, Error::Http { status, .. } if status == reqwest::StatusCode::UNAUTHORIZED)
28699 );
28700 assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
28701 }
28702
28703 #[tokio::test]
28704 async fn worker_bounds_transport_retries() {
28705 let server = MockWorkerServer::unavailable_polls();
28706 let client = Client::builder(server.base_url())
28707 .timeout(Duration::from_secs(2))
28708 .build()
28709 .expect("client");
28710 let mut worker = Worker::new(client, "rust-workers")
28711 .worker_id("bounded-retry-worker")
28712 .poll_timeout(Duration::from_millis(10))
28713 .retry_policy(WorkerRetryPolicy {
28714 max_retries: 2,
28715 initial_backoff: Duration::from_millis(1),
28716 max_backoff: Duration::from_millis(1),
28717 });
28718 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28719
28720 let error = worker.run().await.expect_err("retry bound must terminate");
28721 assert!(matches!(error, Error::Transport(_)));
28722 assert_eq!(
28723 server.request_count("/api/worker/workflow-tasks/poll"),
28724 3,
28725 "one initial request plus exactly two retries"
28726 );
28727 }
28728
28729 #[tokio::test]
28730 async fn worker_retry_policy_can_disable_poll_retries() {
28731 let server = MockWorkerServer::unavailable_polls();
28732 let client = Client::builder(server.base_url())
28733 .timeout(Duration::from_secs(2))
28734 .build()
28735 .expect("client");
28736 let mut worker = Worker::new(client, "rust-workers")
28737 .worker_id("no-retry-worker")
28738 .poll_timeout(Duration::from_millis(10))
28739 .retry_policy(WorkerRetryPolicy {
28740 max_retries: 0,
28741 initial_backoff: Duration::from_millis(1),
28742 max_backoff: Duration::from_millis(1),
28743 });
28744 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28745
28746 let error = worker
28747 .run_once()
28748 .await
28749 .expect_err("disabled retries must return the first transport failure");
28750 assert!(matches!(error, Error::Transport(_)));
28751 assert_eq!(
28752 server.request_count("/api/worker/workflow-tasks/poll"),
28753 1,
28754 "max_retries=0 must send only the initial request"
28755 );
28756 }
28757
28758 #[tokio::test]
28759 async fn worker_does_not_retry_authentication_failures() {
28760 let server = MockWorkerServer::unauthorized_polls();
28761 let client = Client::builder(server.base_url())
28762 .timeout(Duration::from_secs(2))
28763 .build()
28764 .expect("client");
28765 let mut worker = Worker::new(client, "rust-workers")
28766 .worker_id("unauthorized-worker")
28767 .recover_transient_outages(true)
28768 .poll_timeout(Duration::from_millis(10));
28769 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
28770
28771 let error = worker
28772 .run()
28773 .await
28774 .expect_err("authentication must terminate");
28775 let Error::Http { status, body } = error else {
28776 panic!("expected stable HTTP authentication error");
28777 };
28778 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
28779 assert!(body.contains("authentication_failed"));
28780 assert_eq!(
28781 server.request_count("/api/worker/workflow-tasks/poll"),
28782 1,
28783 "authentication failures must not be retried"
28784 );
28785 }
28786
28787 #[test]
28788 fn duplicate_registration_keeps_the_original_workflow() {
28789 let client = Client::new("http://127.0.0.1:9").expect("client");
28790 let mut worker = Worker::new(client, "registration-tests");
28791 worker.register_workflow("same", |_ctx, _input| async { Ok(json!("first")) });
28792 let original = Arc::clone(&worker.workflows["same"].execute);
28793 worker
28794 .set_workflow_definition_sources("same", &["first workflow source"])
28795 .expect("source identity");
28796 let fingerprint = worker.workflows["same"].definition_fingerprint.clone();
28797 worker.register_typed_workflow("same", |_ctx, _input: Value| async { Ok(json!("second")) });
28798 assert!(Arc::ptr_eq(&original, &worker.workflows["same"].execute));
28799 assert_eq!(worker.workflows["same"].definition_fingerprint, fingerprint);
28800 }
28801
28802 #[test]
28803 fn duplicate_registration_keeps_the_original_activity() {
28804 let client = Client::new("http://127.0.0.1:9").expect("client");
28805 let mut worker = Worker::new(client, "registration-tests");
28806 worker.register_activity("same", |_ctx, _input| async { Ok(json!("first")) });
28807 let original = Arc::clone(&worker.activities["same"]);
28808 worker.register_activity_avro_value("same", |_ctx, _input| async {
28809 Ok(AvroValue::String("second".into()))
28810 });
28811 assert!(Arc::ptr_eq(&original, &worker.activities["same"]));
28812 }
28813
28814 #[test]
28815 fn duplicate_registration_keeps_original_query_and_update_handlers() {
28816 let client = Client::new("http://127.0.0.1:9").expect("client");
28817 let mut worker = Worker::new(client, "registration-tests");
28818 worker.register_query("orders", "same", |_ctx, _input| async {
28819 Ok(json!("first"))
28820 });
28821 let RegisteredQuery::Snapshot(original) = worker.queries["orders"]["same"].clone() else {
28822 panic!("snapshot query");
28823 };
28824 worker.register_replayed_query("orders", "same", |_ctx, _state: Arc<()>, _input| async {
28825 Ok(json!("second"))
28826 });
28827 let RegisteredQuery::Snapshot(retained) = &worker.queries["orders"]["same"] else {
28828 panic!("duplicate replaced the query adapter");
28829 };
28830 assert!(Arc::ptr_eq(&original, retained));
28831 worker.register_update("orders", "same", |_ctx, _input| async {
28832 Ok(json!("first"))
28833 });
28834 let original = Arc::clone(&worker.updates["orders"]["same"]);
28835 worker.register_update_avro_value("orders", "same", |_ctx, _input| async {
28836 Ok(AvroValue::Null)
28837 });
28838 assert!(Arc::ptr_eq(&original, &worker.updates["orders"]["same"]));
28839 }
28840
28841 #[tokio::test]
28842 async fn duplicate_registration_fails_before_server_contact() {
28843 let server = MockWorkerServer::start();
28844 let client = Client::builder(server.base_url()).build().expect("client");
28845 let mut worker =
28846 Worker::new(client, "registration-tests").poll_timeout(Duration::from_millis(10));
28847 worker.register_workflow("same", |_ctx, _input| async { Ok(json!("first")) });
28848 worker.register_workflow("same", |_ctx, _input| async { Ok(json!("second")) });
28849 let error = worker
28850 .run_once()
28851 .await
28852 .expect_err("ambiguous worker must not poll");
28853 assert!(
28854 error.to_string().contains("duplicate_registration"),
28855 "{error}"
28856 );
28857 assert!(server.requests.lock().expect("requests").is_empty());
28858 }
28859
28860 #[derive(Clone, Debug)]
28861 struct CapturedRequest {
28862 headers: String,
28863 method: String,
28864 path: String,
28865 authorization: Option<String>,
28866 namespace: Option<String>,
28867 worker_protocol: Option<String>,
28868 control_protocol: Option<String>,
28869 body: String,
28870 received_at: Instant,
28871 }
28872
28873 struct MockWorkerServer {
28874 addr: SocketAddr,
28875 stop: Arc<AtomicBool>,
28876 requests: Arc<Mutex<Vec<CapturedRequest>>>,
28877 thread: Option<thread::JoinHandle<()>>,
28878 }
28879
28880 type RequestOverride = fn(&str, &str, usize) -> Option<(&'static str, String)>;
28881
28882 #[derive(Clone, Copy, Default)]
28883 struct MockWorkerBehavior {
28884 response_override: Option<fn(&str) -> Option<(&'static str, String)>>,
28885 request_override: Option<RequestOverride>,
28886 storage_refusals: usize,
28887 storage_path: Option<&'static str>,
28888 storage_unavailable: bool,
28889 storage_mid_poll: bool,
28890 storage_activity: bool,
28891 storage_query: bool,
28892 storage_wrong_poll_id: bool,
28893 reject_query_protocol: bool,
28894 reject_query_completion: bool,
28895 waiting_query_worker: bool,
28896 decline_registration: bool,
28897 complete_named_signal: bool,
28898 poll_failures_per_path: usize,
28899 long_poll_capacity_responses_per_path: usize,
28900 heartbeat_failures: usize,
28901 heartbeat_failure_request: Option<usize>,
28902 delayed_heartbeat_request: Option<usize>,
28903 heartbeat_response_delay: Duration,
28904 concurrent_requests: bool,
28905 unauthorized_polls: bool,
28906 reject_registration: bool,
28907 reject_registration_protocol: bool,
28908 reject_deregistration: bool,
28909 reject_deregistration_protocol: bool,
28910 cancelled_activity: bool,
28911 draining_polls: bool,
28912 invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
28913 workflow_completion_status: Option<&'static str>,
28914 workflow_completion_body: Option<&'static str>,
28915 }
28916
28917 impl MockWorkerServer {
28918 fn start() -> Self {
28919 Self::start_with_behavior(MockWorkerBehavior::default())
28920 }
28921
28922 fn reject_query_protocol() -> Self {
28923 Self::start_with_behavior(MockWorkerBehavior {
28924 reject_query_protocol: true,
28925 ..MockWorkerBehavior::default()
28926 })
28927 }
28928
28929 fn reject_query_completion() -> Self {
28930 Self::start_with_behavior(MockWorkerBehavior {
28931 reject_query_completion: true,
28932 ..MockWorkerBehavior::default()
28933 })
28934 }
28935
28936 fn waiting_query_worker() -> Self {
28937 Self::start_with_behavior(MockWorkerBehavior {
28938 waiting_query_worker: true,
28939 complete_named_signal: true,
28940 ..MockWorkerBehavior::default()
28941 })
28942 }
28943
28944 fn transient_worker_failures() -> Self {
28945 Self::start_with_behavior(MockWorkerBehavior {
28946 poll_failures_per_path: 1,
28947 heartbeat_failures: 1,
28948 ..MockWorkerBehavior::default()
28949 })
28950 }
28951
28952 fn consecutive_poll_failures(count: usize) -> Self {
28953 Self::start_with_behavior(MockWorkerBehavior {
28954 poll_failures_per_path: count,
28955 ..MockWorkerBehavior::default()
28956 })
28957 }
28958
28959 fn capacity_limited_activity_poll() -> Self {
28960 Self::start_with_behavior(MockWorkerBehavior {
28961 long_poll_capacity_responses_per_path: 1,
28962 ..MockWorkerBehavior::default()
28963 })
28964 }
28965
28966 fn delayed_heartbeat_worker() -> Self {
28967 Self::start_with_behavior(MockWorkerBehavior {
28968 waiting_query_worker: true,
28969 delayed_heartbeat_request: Some(2),
28970 heartbeat_response_delay: Duration::from_millis(1_500),
28971 concurrent_requests: true,
28972 cancelled_activity: true,
28973 ..MockWorkerBehavior::default()
28974 })
28975 }
28976
28977 fn heartbeat_retry_worker() -> Self {
28978 Self::start_with_behavior(MockWorkerBehavior {
28979 waiting_query_worker: true,
28980 heartbeat_failure_request: Some(2),
28981 concurrent_requests: true,
28982 ..MockWorkerBehavior::default()
28983 })
28984 }
28985
28986 fn unavailable_polls() -> Self {
28987 Self::start_with_behavior(MockWorkerBehavior {
28988 poll_failures_per_path: usize::MAX,
28989 ..MockWorkerBehavior::default()
28990 })
28991 }
28992
28993 fn unauthorized_polls() -> Self {
28994 Self::start_with_behavior(MockWorkerBehavior {
28995 unauthorized_polls: true,
28996 ..MockWorkerBehavior::default()
28997 })
28998 }
28999
29000 fn rejected_registration() -> Self {
29001 Self::start_with_behavior(MockWorkerBehavior {
29002 reject_registration: true,
29003 ..MockWorkerBehavior::default()
29004 })
29005 }
29006
29007 fn rejected_registration_protocol() -> Self {
29008 Self::start_with_behavior(MockWorkerBehavior {
29009 reject_registration_protocol: true,
29010 ..MockWorkerBehavior::default()
29011 })
29012 }
29013
29014 fn declined_registration() -> Self {
29015 Self::start_with_behavior(MockWorkerBehavior {
29016 decline_registration: true,
29017 ..MockWorkerBehavior::default()
29018 })
29019 }
29020
29021 fn rejected_deregistration() -> Self {
29022 Self::start_with_behavior(MockWorkerBehavior {
29023 reject_deregistration: true,
29024 ..MockWorkerBehavior::default()
29025 })
29026 }
29027
29028 fn rejected_deregistration_protocol() -> Self {
29029 Self::start_with_behavior(MockWorkerBehavior {
29030 reject_deregistration_protocol: true,
29031 ..MockWorkerBehavior::default()
29032 })
29033 }
29034
29035 fn unauthorized_polls_and_rejected_deregistration() -> Self {
29036 Self::start_with_behavior(MockWorkerBehavior {
29037 unauthorized_polls: true,
29038 reject_deregistration: true,
29039 ..MockWorkerBehavior::default()
29040 })
29041 }
29042
29043 fn cancelled_activity() -> Self {
29044 Self::start_with_behavior(MockWorkerBehavior {
29045 cancelled_activity: true,
29046 ..MockWorkerBehavior::default()
29047 })
29048 }
29049
29050 fn draining_polls() -> Self {
29051 Self::start_with_behavior(MockWorkerBehavior {
29052 draining_polls: true,
29053 ..MockWorkerBehavior::default()
29054 })
29055 }
29056
29057 fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
29058 Self::start_with_behavior(MockWorkerBehavior {
29059 invalid_task_payload_codec: Some(codec),
29060 ..MockWorkerBehavior::default()
29061 })
29062 }
29063
29064 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
29065 Self::start_with_behavior(MockWorkerBehavior {
29066 workflow_completion_status: Some(status),
29067 workflow_completion_body: Some(body),
29068 ..MockWorkerBehavior::default()
29069 })
29070 }
29071
29072 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
29073 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
29074 listener
29075 .set_nonblocking(true)
29076 .expect("configure mock listener");
29077 let addr = listener.local_addr().expect("mock server address");
29078 let stop = Arc::new(AtomicBool::new(false));
29079 let server_stop = Arc::clone(&stop);
29080 let requests = Arc::new(Mutex::new(Vec::new()));
29081 let server_requests = Arc::clone(&requests);
29082 let thread = thread::spawn(move || {
29083 let mut request_threads = Vec::new();
29084 while !server_stop.load(Ordering::SeqCst) {
29085 match listener.accept() {
29086 Ok((mut stream, _)) => {
29087 if behavior.concurrent_requests {
29088 let requests = Arc::clone(&server_requests);
29089 request_threads.push(thread::spawn(move || {
29090 handle_mock_worker_request(&mut stream, &requests, behavior)
29091 }));
29092 } else {
29093 handle_mock_worker_request(&mut stream, &server_requests, behavior);
29094 }
29095 }
29096 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
29097 let mut index = 0;
29098 while index < request_threads.len() {
29099 if request_threads[index].is_finished() {
29100 request_threads
29101 .swap_remove(index)
29102 .join()
29103 .expect("join mock request");
29104 } else {
29105 index += 1;
29106 }
29107 }
29108 thread::sleep(Duration::from_millis(5));
29109 }
29110 Err(_) => break,
29111 }
29112 }
29113 for request_thread in request_threads {
29114 request_thread.join().expect("join mock request");
29115 }
29116 });
29117
29118 Self {
29119 addr,
29120 stop,
29121 requests,
29122 thread: Some(thread),
29123 }
29124 }
29125
29126 fn base_url(&self) -> String {
29127 format!("http://{}", self.addr)
29128 }
29129
29130 fn worker_protocol_for(&self, path: &str) -> Option<String> {
29131 self.requests
29132 .lock()
29133 .expect("captured requests")
29134 .iter()
29135 .find(|request| request.path == path)
29136 .and_then(|request| request.worker_protocol.clone())
29137 }
29138
29139 fn control_protocol_for(&self, path: &str) -> Option<String> {
29140 self.requests
29141 .lock()
29142 .expect("captured requests")
29143 .iter()
29144 .find(|request| request.path == path)
29145 .and_then(|request| request.control_protocol.clone())
29146 }
29147
29148 fn method_for(&self, path: &str) -> Option<String> {
29149 self.requests
29150 .lock()
29151 .expect("captured requests")
29152 .iter()
29153 .find(|request| request.path == path)
29154 .map(|request| request.method.clone())
29155 }
29156
29157 fn authorization_for(&self, path: &str) -> Option<String> {
29158 self.requests
29159 .lock()
29160 .expect("captured requests")
29161 .iter()
29162 .find(|request| request.path == path)
29163 .and_then(|request| request.authorization.clone())
29164 }
29165
29166 fn namespace_for(&self, path: &str) -> Option<String> {
29167 self.requests
29168 .lock()
29169 .expect("captured requests")
29170 .iter()
29171 .find(|request| request.path == path)
29172 .and_then(|request| request.namespace.clone())
29173 }
29174
29175 fn request_count(&self, path: &str) -> usize {
29176 self.requests
29177 .lock()
29178 .expect("captured requests")
29179 .iter()
29180 .filter(|request| request.path == path)
29181 .count()
29182 }
29183
29184 fn captured_paths(&self) -> Vec<String> {
29185 self.requests
29186 .lock()
29187 .expect("captured requests")
29188 .iter()
29189 .map(|request| request.path.clone())
29190 .collect()
29191 }
29192
29193 fn request_times(&self, path: &str) -> Vec<Instant> {
29194 self.requests
29195 .lock()
29196 .expect("captured requests")
29197 .iter()
29198 .filter(|request| request.path == path)
29199 .map(|request| request.received_at)
29200 .collect()
29201 }
29202
29203 fn request_body(&self, path: &str) -> Value {
29204 let requests = self.requests.lock().expect("captured requests");
29205 let body = &requests
29206 .iter()
29207 .find(|request| request.path == path)
29208 .unwrap_or_else(|| panic!("missing request for {path}"))
29209 .body;
29210 serde_json::from_str(body).unwrap_or_else(|error| {
29211 panic!("invalid JSON request body for {path}: {error}: {body:?}")
29212 })
29213 }
29214
29215 fn request_bodies(&self, path: &str) -> Vec<Value> {
29216 self.requests
29217 .lock()
29218 .expect("captured requests")
29219 .iter()
29220 .filter(|request| request.path == path)
29221 .map(|request| {
29222 serde_json::from_str(&request.body).unwrap_or_else(|error| {
29223 panic!(
29224 "invalid JSON request body for {path}: {error}: {:?}",
29225 request.body
29226 )
29227 })
29228 })
29229 .collect()
29230 }
29231 }
29232
29233 impl Drop for MockWorkerServer {
29234 fn drop(&mut self) {
29235 self.stop.store(true, Ordering::SeqCst);
29236 let _ = TcpStream::connect(self.addr);
29237
29238 if let Some(thread) = self.thread.take() {
29239 thread.join().expect("join mock server");
29240 }
29241 }
29242 }
29243
29244 fn handle_mock_worker_request(
29245 stream: &mut TcpStream,
29246 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
29247 behavior: MockWorkerBehavior,
29248 ) {
29249 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
29250 let mut buffer = [0_u8; 8192];
29251 let mut request = Vec::new();
29252
29253 loop {
29254 match stream.read(&mut buffer) {
29255 Ok(0) => break,
29256 Ok(read) => {
29257 request.extend_from_slice(&buffer[..read]);
29258 if mock_request_is_complete(&request) {
29259 break;
29260 }
29261 }
29262 Err(error)
29263 if matches!(
29264 error.kind(),
29265 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
29266 ) =>
29267 {
29268 break;
29269 }
29270 Err(_) => return,
29271 }
29272 }
29273
29274 let request = String::from_utf8_lossy(&request);
29275 let body = request
29276 .split_once("\r\n\r\n")
29277 .map(|(_, body)| body)
29278 .unwrap_or_default();
29279 let path = request
29280 .lines()
29281 .next()
29282 .and_then(|line| line.split_whitespace().nth(1))
29283 .unwrap_or_default();
29284 let method = request
29285 .lines()
29286 .next()
29287 .and_then(|line| line.split_whitespace().next())
29288 .unwrap_or_default();
29289 let authorization = request.lines().find_map(|line| {
29290 let (name, value) = line.split_once(':')?;
29291 name.eq_ignore_ascii_case("Authorization")
29292 .then(|| value.trim().to_string())
29293 });
29294 let namespace = request.lines().find_map(|line| {
29295 let (name, value) = line.split_once(':')?;
29296 name.eq_ignore_ascii_case("X-Namespace")
29297 .then(|| value.trim().to_string())
29298 });
29299 let worker_protocol = request.lines().find_map(|line| {
29300 let (name, value) = line.split_once(':')?;
29301 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
29302 .then(|| value.trim().to_string())
29303 });
29304 let control_protocol = request.lines().find_map(|line| {
29305 let (name, value) = line.split_once(':')?;
29306 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
29307 .then(|| value.trim().to_string())
29308 });
29309 let request_number = {
29310 let mut requests = requests.lock().expect("captured requests");
29311 requests.push(CapturedRequest {
29312 headers: request
29313 .split_once("\r\n\r\n")
29314 .map_or("", |(headers, _)| headers)
29315 .to_owned(),
29316 method: method.to_string(),
29317 path: path.to_string(),
29318 authorization,
29319 namespace,
29320 worker_protocol: worker_protocol.clone(),
29321 control_protocol,
29322 body: body.to_string(),
29323 received_at: Instant::now(),
29324 });
29325 requests
29326 .iter()
29327 .filter(|request| request.path == path)
29328 .count()
29329 };
29330
29331 if let Some(response) = behavior
29332 .request_override
29333 .and_then(|handler| handler(path, body, request_number))
29334 {
29335 write_mock_response(stream, response.0, &response.1);
29336 return;
29337 }
29338 if let Some(response) = behavior.response_override.and_then(|handler| handler(path)) {
29339 write_mock_response(stream, response.0, &response.1);
29340 return;
29341 }
29342 if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
29343 return;
29344 }
29345 let pressure_path = behavior
29346 .storage_path
29347 .is_some_and(|part| path.contains(part));
29348 let prior_failures = if path.ends_with("/poll") {
29349 behavior.poll_failures_per_path
29350 } else {
29351 0
29352 };
29353 if pressure_path
29354 && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
29355 {
29356 let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
29357 let poll_id = path
29358 .ends_with("/poll")
29359 .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
29360 let mut refusal = storage_refusal(
29361 poll_id,
29362 behavior.storage_unavailable,
29363 behavior.storage_mid_poll,
29364 );
29365 if behavior.storage_wrong_poll_id {
29366 refusal["poll_request_id"] = json!("wrong-poll");
29367 }
29368 write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
29369 return;
29370 }
29371 if path.contains("/storage-task/")
29372 || path.contains("/storage-activity/")
29373 || path.contains("/storage-query/")
29374 {
29375 write_mock_response(stream, "200 OK", "{}");
29376 return;
29377 }
29378 if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
29379 write_mock_response(stream, "200 OK", &json!({"task":{
29380 "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
29381 "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
29382 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
29383 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
29384 "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
29385 }}).to_string());
29386 return;
29387 }
29388 if behavior.storage_activity
29389 && path == "/api/worker/activity-tasks/poll"
29390 && request_number == 1
29391 {
29392 write_mock_response(stream, "200 OK", &json!({"task":{
29393 "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
29394 "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
29395 "attempt_number":7, "lease_owner":"storage-worker"
29396 }}).to_string());
29397 return;
29398 }
29399
29400 if path == "/api/worker/register" {
29401 if behavior.reject_registration_protocol {
29402 write_mock_response(
29403 stream,
29404 "400 Bad Request",
29405 r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
29406 );
29407 return;
29408 }
29409 if behavior.reject_registration {
29410 write_mock_response(
29411 stream,
29412 "503 Service Unavailable",
29413 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
29414 );
29415 return;
29416 }
29417 }
29418
29419 if path.starts_with("/api/worker/registrations/") {
29420 if behavior.reject_deregistration_protocol {
29421 write_mock_response(
29422 stream,
29423 "400 Bad Request",
29424 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
29425 );
29426 } else if behavior.reject_deregistration {
29427 write_mock_response(
29428 stream,
29429 "403 Forbidden",
29430 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
29431 );
29432 } else {
29433 write_mock_response(
29434 stream,
29435 "200 OK",
29436 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
29437 );
29438 }
29439 return;
29440 }
29441
29442 let is_poll = matches!(
29443 path,
29444 "/api/worker/workflow-tasks/poll"
29445 | "/api/worker/activity-tasks/poll"
29446 | "/api/worker/query-tasks/poll"
29447 );
29448 if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
29449 write_mock_response(
29450 stream,
29451 "429 Too Many Requests",
29452 r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
29453 );
29454 return;
29455 }
29456 if is_poll && request_number <= behavior.poll_failures_per_path {
29457 return;
29458 }
29459 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
29460 return;
29461 }
29462 if path == "/api/worker/heartbeat"
29463 && behavior.heartbeat_failure_request == Some(request_number)
29464 {
29465 return;
29466 }
29467 if path == "/api/worker/heartbeat"
29468 && behavior.delayed_heartbeat_request == Some(request_number)
29469 {
29470 thread::sleep(behavior.heartbeat_response_delay);
29471 }
29472 if behavior.unauthorized_polls && is_poll {
29473 write_mock_response(
29474 stream,
29475 "401 Unauthorized",
29476 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
29477 );
29478 return;
29479 }
29480 if behavior.draining_polls && is_poll {
29481 write_mock_response(
29482 stream,
29483 "409 Conflict",
29484 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
29485 );
29486 return;
29487 }
29488
29489 if let Some(codec_case) = behavior.invalid_task_payload_codec {
29490 if is_poll && request_number == 1 {
29491 let mut task = match path {
29492 "/api/worker/workflow-tasks/poll" => json!({
29493 "task_id": "codec-workflow",
29494 "workflow_type": "codec.workflow",
29495 "payload_codec": DEFAULT_CODEC,
29496 "workflow_task_attempt": 1,
29497 "lease_owner": "codec-worker"
29498 }),
29499 "/api/worker/activity-tasks/poll" => json!({
29500 "task_id": "codec-activity",
29501 "activity_attempt_id": "codec-activity-attempt",
29502 "activity_type": "codec.activity",
29503 "payload_codec": DEFAULT_CODEC,
29504 "attempt_number": 1,
29505 "lease_owner": "codec-worker"
29506 }),
29507 "/api/worker/query-tasks/poll" => json!({
29508 "query_task_id": "codec-query",
29509 "query_task_attempt": 1,
29510 "workflow_type": "codec.workflow",
29511 "query_name": "known",
29512 "payload_codec": DEFAULT_CODEC,
29513 "lease_owner": "codec-worker"
29514 }),
29515 _ => unreachable!("is_poll limits task codec probe paths"),
29516 };
29517 codec_case.apply(&mut task);
29518 write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
29519 return;
29520 }
29521
29522 if matches!(
29523 path,
29524 "/api/worker/workflow-tasks/codec-workflow/fail"
29525 | "/api/worker/activity-tasks/codec-activity/fail"
29526 | "/api/worker/query-tasks/codec-query/fail"
29527 ) {
29528 write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
29529 return;
29530 }
29531 }
29532
29533 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
29534 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
29535 let body = format!(
29536 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
29537 );
29538 write_mock_response(stream, "400 Bad Request", &body);
29539 return;
29540 }
29541
29542 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
29543 {
29544 write_mock_response(
29545 stream,
29546 "409 Conflict",
29547 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
29548 );
29549 return;
29550 }
29551
29552 if behavior.workflow_completion_status.is_some()
29553 && path == "/api/worker/workflow-tasks/poll"
29554 && request_number == 1
29555 {
29556 write_mock_response(
29557 stream,
29558 "200 OK",
29559 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
29560 );
29561 return;
29562 }
29563
29564 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
29565 if let (Some(status), Some(body)) = (
29566 behavior.workflow_completion_status,
29567 behavior.workflow_completion_body,
29568 ) {
29569 write_mock_response(stream, status, body);
29570 return;
29571 }
29572 }
29573
29574 if behavior.waiting_query_worker {
29575 if behavior.complete_named_signal
29576 && path == "/api/worker/workflow-tasks/poll"
29577 && request_number == 1
29578 {
29579 let body = json!({
29580 "task": {
29581 "task_id": "snapshot-open",
29582 "workflow_id": "snapshot-1",
29583 "run_id": "snapshot-run-1",
29584 "workflow_type": "snapshot",
29585 "payload_codec": DEFAULT_CODEC,
29586 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
29587 .expect("Avro workflow arguments"),
29588 "history_events": [],
29589 "workflow_task_attempt": 1,
29590 "lease_owner": "rust-snapshot-worker"
29591 }
29592 })
29593 .to_string();
29594 write_mock_response(stream, "200 OK", &body);
29595 return;
29596 }
29597
29598 let signal_request = request_number - usize::from(behavior.complete_named_signal);
29599 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
29600 if path == "/api/worker/workflow-tasks/poll"
29601 && signal_request >= 1
29602 && signal_request <= signal_request_limit
29603 {
29604 let finish = behavior.complete_named_signal && signal_request == 3;
29605 let amounts = if signal_request == 1 {
29606 vec![3]
29607 } else {
29608 vec![3, 5]
29609 };
29610 let task_id = if signal_request == 1 {
29611 "snapshot-wait-3"
29612 } else if finish {
29613 "snapshot-finish"
29614 } else {
29615 "snapshot-wait-5"
29616 };
29617 let mut history_events = std::iter::once(json!({
29618 "event_type": "SignalWaitOpened",
29619 "payload": {"sequence": 1, "signal_name": "finish"}
29620 }))
29621 .chain(amounts.iter().enumerate().map(|(index, amount)| {
29622 json!({
29623 "event_type": "SignalReceived",
29624 "payload": {
29625 "signal_id": format!("increment-{amount}"),
29626 "signal_name": "increment",
29627 "workflow_sequence": index + 2,
29628 "payload_codec": DEFAULT_CODEC,
29629 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
29630 .expect("Avro signal envelope")
29631 }
29632 })
29633 }))
29634 .collect::<Vec<_>>();
29635 let (resume_id, resume_name, resume_arguments) = if finish {
29636 history_events.push(json!({
29637 "event_type": "SignalReceived",
29638 "payload": {
29639 "signal_id": "finish",
29640 "signal_name": "finish",
29641 "workflow_sequence": 4,
29642 "payload_codec": DEFAULT_CODEC,
29643 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
29644 .expect("Avro finish signal envelope")
29645 }
29646 }));
29647 (
29648 "finish".to_string(),
29649 "finish".to_string(),
29650 encode_value_envelope(&json!([]), DEFAULT_CODEC)
29651 .expect("Avro finish resume signal"),
29652 )
29653 } else {
29654 let amount = amounts.last().expect("amount");
29655 (
29656 format!("increment-{amount}"),
29657 "increment".to_string(),
29658 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
29659 .expect("Avro increment resume signal"),
29660 )
29661 };
29662 let body = json!({
29663 "task": {
29664 "task_id": task_id,
29665 "workflow_id": "snapshot-1",
29666 "run_id": "snapshot-run-1",
29667 "workflow_type": "snapshot",
29668 "payload_codec": DEFAULT_CODEC,
29669 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
29670 .expect("Avro workflow arguments"),
29671 "history_events": history_events,
29672 "workflow_task_attempt": 1,
29673 "workflow_signal_id": resume_id,
29674 "signal_name": resume_name,
29675 "signal_arguments": resume_arguments,
29676 "lease_owner": "rust-snapshot-worker"
29677 }
29678 })
29679 .to_string();
29680 write_mock_response(stream, "200 OK", &body);
29681 return;
29682 }
29683
29684 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
29685 let history_events = [3, 5]
29686 .into_iter()
29687 .enumerate()
29688 .map(|(index, amount)| {
29689 json!({
29690 "event_type": "SignalReceived",
29691 "payload": {
29692 "signal_id": format!("increment-{amount}"),
29693 "signal_name": "increment",
29694 "workflow_sequence": index + 2,
29695 "payload_codec": DEFAULT_CODEC,
29696 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
29697 .expect("Avro query signal envelope")
29698 }
29699 })
29700 })
29701 .collect::<Vec<_>>();
29702 let body = json!({
29703 "task": {
29704 "query_task_id": "snapshot-current",
29705 "query_task_attempt": 1,
29706 "lease_owner": "rust-snapshot-worker",
29707 "workflow_id": "snapshot-1",
29708 "run_id": "snapshot-run-1",
29709 "workflow_type": "snapshot",
29710 "query_name": "current",
29711 "payload_codec": DEFAULT_CODEC,
29712 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
29713 .expect("Avro workflow arguments"),
29714 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
29715 .expect("Avro query arguments"),
29716 "history_events": history_events,
29717 "run_status": "waiting"
29718 }
29719 })
29720 .to_string();
29721 write_mock_response(stream, "200 OK", &body);
29722 return;
29723 }
29724
29725 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
29726 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
29727 {
29728 write_mock_response(
29729 stream,
29730 "200 OK",
29731 r#"{"outcome":"waiting_for_history","recorded":true}"#,
29732 );
29733 return;
29734 }
29735
29736 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
29737 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
29738 return;
29739 }
29740
29741 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
29742 write_mock_response(
29743 stream,
29744 "200 OK",
29745 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
29746 );
29747 return;
29748 }
29749
29750 if path == "/api/worker/query-tasks/snapshot-current/complete" {
29751 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
29752 return;
29753 }
29754 }
29755
29756 if matches!(
29757 path,
29758 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
29759 ) {
29760 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
29761 .expect("typed mock result");
29762 let body = json!({
29763 "result": typed_fidelity_probe().into_json().expect("result projection"),
29764 "result_envelope": result,
29765 })
29766 .to_string();
29767 write_mock_response(stream, "200 OK", &body);
29768 return;
29769 }
29770
29771 if path == "/api/workflows/typed-1" {
29772 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
29773 .expect("typed mock result");
29774 let body = json!({
29775 "workflow_id": "typed-1",
29776 "run_id": "run-typed-1",
29777 "workflow_type": "typed.echo",
29778 "status": "completed",
29779 "output": typed_fidelity_probe().into_json().expect("output projection"),
29780 "output_envelope": result,
29781 })
29782 .to_string();
29783 write_mock_response(stream, "200 OK", &body);
29784 return;
29785 }
29786
29787 let (status, body) = match path {
29788 "/api/cluster/info" => ("200 OK", r#"{"limits":{"max_payload_bytes":2097152}}"#),
29789 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
29790 "/api/workflows" => (
29791 "201 Created",
29792 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
29793 ),
29794 "/api/worker/register" if behavior.decline_registration => (
29795 "200 OK",
29796 r#"{"worker_id":"declined-worker","registered":false}"#,
29797 ),
29798 "/api/worker/register" if behavior.waiting_query_worker => (
29799 "200 OK",
29800 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
29801 ),
29802 "/api/worker/register" => (
29803 "200 OK",
29804 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
29805 ),
29806 "/api/worker/heartbeat" => ("200 OK", "{}"),
29807 "/api/worker/activity-tasks/poll"
29808 if behavior.cancelled_activity && request_number == 1 =>
29809 {
29810 (
29811 "200 OK",
29812 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
29813 )
29814 }
29815 "/api/worker/activity-tasks/poll"
29816 if behavior.long_poll_capacity_responses_per_path > 0
29817 && request_number
29818 == behavior
29819 .long_poll_capacity_responses_per_path
29820 .saturating_add(1) =>
29821 {
29822 (
29823 "200 OK",
29824 r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
29825 )
29826 }
29827 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
29828 ("200 OK", r#"{"task":null}"#)
29829 }
29830 "/api/worker/query-tasks/poll"
29831 if behavior.reject_query_completion && request_number == 1 =>
29832 {
29833 (
29834 "200 OK",
29835 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
29836 )
29837 }
29838 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
29839 "/api/worker/query-tasks/query-capture/complete"
29840 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
29841 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
29842 "200 OK",
29843 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
29844 ),
29845 "/api/worker/activity-tasks/activity-cancel/complete" => (
29846 "409 Conflict",
29847 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
29848 ),
29849 "/api/worker/activity-tasks/activity-typed/complete"
29850 | "/api/worker/activity-tasks/activity-typed/fail"
29851 | "/api/worker/activity-tasks/capacity-activity/complete"
29852 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
29853 "/api/workflows/counter-1/query/current" => (
29854 "200 OK",
29855 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
29856 ),
29857 "/api/workflows/counter-1/query/missing" => (
29858 "404 Not Found",
29859 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
29860 ),
29861 "/api/workflows/wf-lifecycle/cancel" => (
29862 "200 OK",
29863 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
29864 ),
29865 "/api/workflows/wf-lifecycle/terminate" => (
29866 "200 OK",
29867 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
29868 ),
29869 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
29870 "200 OK",
29871 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
29872 ),
29873 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
29874 "200 OK",
29875 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
29876 ),
29877 "/api/workflows/wf-lifecycle/runs/run-failed/redrive" => (
29878 "202 Accepted",
29879 r#"{"workflow_id":"wf-lifecycle","continued_from_run_id":"run-failed","run_id":"run-successor","outcome":"redriven","command_status":"accepted","resume_step_sequence":2}"#,
29880 ),
29881 "/api/workflows/wf-lifecycle/runs/run-failed-existing/redrive" => (
29882 "200 OK",
29883 r#"{"workflow_id":"wf-lifecycle","continued_from_run_id":"run-failed-existing","run_id":"run-successor","outcome":"redriven","command_status":"accepted","resume_step_sequence":2}"#,
29884 ),
29885 "/api/workflows/wf-lifecycle/runs/run-completed/redrive" => (
29886 "409 Conflict",
29887 r#"{"workflow_id":"wf-lifecycle","run_id":"run-completed","reason":"run_not_failed","message":"Run is not failed."}"#,
29888 ),
29889 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
29890 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
29891 "409 Conflict",
29892 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
29893 ),
29894 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
29895 "200 OK",
29896 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
29897 ),
29898 "/api/workflows/wf-cancelled" => (
29899 "200 OK",
29900 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
29901 ),
29902 "/api/workflows/wf-terminated" => (
29903 "200 OK",
29904 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
29905 ),
29906 "/api/workflows/wf-timed-out" => (
29907 "200 OK",
29908 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
29909 ),
29910 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
29911 "200 OK",
29912 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
29913 ),
29914 "/api/workflows/wf-selected" => (
29915 "200 OK",
29916 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
29917 ),
29918 "/api/workflows/wf-selected/runs/run-selected" => (
29919 "200 OK",
29920 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
29921 ),
29922 _ => ("404 Not Found", r#"{"message":"not found"}"#),
29923 };
29924 write_mock_response(stream, status, body);
29925 }
29926
29927 fn mock_request_is_complete(request: &[u8]) -> bool {
29928 let Some(header_end) = request
29929 .windows(4)
29930 .position(|window| window == b"\r\n\r\n")
29931 .map(|position| position + 4)
29932 else {
29933 return false;
29934 };
29935 let headers = String::from_utf8_lossy(&request[..header_end]);
29936 let content_length = headers.lines().find_map(|line| {
29937 let (name, value) = line.split_once(':')?;
29938 name.eq_ignore_ascii_case("content-length")
29939 .then(|| value.trim().parse::<usize>().ok())
29940 .flatten()
29941 });
29942
29943 request.len() >= header_end + content_length.unwrap_or(0)
29944 }
29945
29946 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
29947 let response = format!(
29948 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
29949 body.len()
29950 );
29951
29952 let _ = stream.write_all(response.as_bytes());
29953 let _ = stream.flush();
29954 }
29955
29956 fn activity_completion_backend_recovery(
29957 path: &str,
29958 _body: &str,
29959 request_number: usize,
29960 ) -> Option<(&'static str, String)> {
29961 match path {
29962 "/api/worker/activity-tasks/poll" if request_number == 1 => Some((
29963 "200 OK",
29964 r#"{"task":{"task_id":"recover-activity","activity_attempt_id":"recover-attempt","activity_type":"recover.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"recover-worker"}}"#.to_string(),
29965 )),
29966 "/api/worker/activity-tasks/recover-activity/complete" if request_number == 1 => Some((
29967 "503 Service Unavailable",
29968 r#"{"reason":"backend_unavailable","operation":"complete_activity_task","outcome":"unknown","worker_id":"recover-worker","task_queue":null,"retryable":true,"retry_after_seconds":1,"task_id":"recover-activity","lease_owner":"recover-worker","activity_attempt_id":"recover-attempt"}"#.to_string(),
29969 )),
29970 "/api/worker/activity-tasks/recover-activity/complete" => Some((
29971 "200 OK",
29972 r#"{"outcome":"completed","recorded":true}"#.to_string(),
29973 )),
29974 _ => None,
29975 }
29976 }
29977
29978 fn workflow_completion_backend_recovery(
29979 path: &str,
29980 _body: &str,
29981 request_number: usize,
29982 ) -> Option<(&'static str, String)> {
29983 match path {
29984 "/api/worker/workflow-tasks/poll" if request_number == 1 => Some((
29985 "200 OK",
29986 include_str!("../tests/fixtures/worker-completion-recovery.json").to_string(),
29987 )),
29988 "/api/worker/workflow-tasks/recover-workflow/complete" if request_number == 1 => Some((
29989 "503 Service Unavailable",
29990 r#"{"reason":"backend_unavailable","operation":"complete_workflow_task","outcome":"unknown","worker_id":"recover-worker","task_queue":null,"retryable":true,"retry_after_seconds":1,"task_id":"recover-workflow","lease_owner":"recover-worker","workflow_task_attempt":3}"#.to_string(),
29991 )),
29992 "/api/worker/workflow-tasks/recover-workflow/complete" => Some((
29993 "200 OK",
29994 r#"{"outcome":"completed","recorded":true}"#.to_string(),
29995 )),
29996 _ => None,
29997 }
29998 }
29999
30000 fn activity_completion_stale_after_backend_recovery(
30001 path: &str,
30002 body: &str,
30003 request_number: usize,
30004 ) -> Option<(&'static str, String)> {
30005 if path == "/api/worker/activity-tasks/recover-activity/complete" && request_number > 1 {
30006 return Some((
30007 "409 Conflict",
30008 r#"{"task_id":"recover-activity","activity_attempt_id":"recover-attempt","lease_owner":"recover-worker","reason":"stale_attempt","outcome":"completed","recorded":false,"activity_status":"completed","attempt_status":"completed","task_status":"completed"}"#.to_string(),
30009 ));
30010 }
30011 activity_completion_backend_recovery(path, body, request_number)
30012 }
30013
30014 fn activity_completion_remains_unavailable(
30015 path: &str,
30016 body: &str,
30017 request_number: usize,
30018 ) -> Option<(&'static str, String)> {
30019 if path == "/api/worker/activity-tasks/recover-activity/complete" {
30020 return activity_completion_backend_recovery(path, body, 1);
30021 }
30022 activity_completion_backend_recovery(path, body, request_number)
30023 }
30024
30025 #[tokio::test]
30026 async fn managed_activity_retries_same_fenced_completion_after_backend_loss() {
30027 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
30028 request_override: Some(activity_completion_backend_recovery),
30029 ..MockWorkerBehavior::default()
30030 });
30031 let client = Client::builder(server.base_url())
30032 .timeout(Duration::from_secs(2))
30033 .build()
30034 .expect("client");
30035 let handler_calls = Arc::new(AtomicUsize::new(0));
30036 let mut worker = Worker::new(client, "recover-queue")
30037 .worker_id("recover-worker")
30038 .poll_timeout(Duration::from_millis(10));
30039 worker.register_activity("recover.activity", {
30040 let handler_calls = Arc::clone(&handler_calls);
30041 move |_ctx, _args| {
30042 let handler_calls = Arc::clone(&handler_calls);
30043 async move {
30044 handler_calls.fetch_add(1, Ordering::SeqCst);
30045 Ok(json!("done"))
30046 }
30047 }
30048 });
30049
30050 assert_eq!(worker.run_once().await.expect("activity settled"), 1);
30051 assert_eq!(handler_calls.load(Ordering::SeqCst), 1);
30052 let bodies = server.request_bodies("/api/worker/activity-tasks/recover-activity/complete");
30053 assert_eq!(bodies.len(), 2);
30054 assert_eq!(bodies[0], bodies[1]);
30055 }
30056
30057 #[tokio::test]
30058 async fn managed_workflow_retries_same_fenced_completion_after_backend_loss() {
30059 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
30060 request_override: Some(workflow_completion_backend_recovery),
30061 ..MockWorkerBehavior::default()
30062 });
30063 let client = Client::builder(server.base_url())
30064 .timeout(Duration::from_secs(2))
30065 .build()
30066 .expect("client");
30067 let handler_calls = Arc::new(AtomicUsize::new(0));
30068 let mut worker = Worker::new(client, "recover-queue")
30069 .worker_id("recover-worker")
30070 .poll_timeout(Duration::from_millis(10));
30071 worker.register_workflow("recover.workflow", {
30072 let handler_calls = Arc::clone(&handler_calls);
30073 move |_ctx, _args| {
30074 let handler_calls = Arc::clone(&handler_calls);
30075 async move {
30076 handler_calls.fetch_add(1, Ordering::SeqCst);
30077 Ok(json!("done"))
30078 }
30079 }
30080 });
30081
30082 assert_eq!(worker.run_once().await.expect("workflow settled"), 1);
30083 assert_eq!(handler_calls.load(Ordering::SeqCst), 1);
30084 let bodies = server.request_bodies("/api/worker/workflow-tasks/recover-workflow/complete");
30085 assert_eq!(bodies.len(), 2);
30086 assert_eq!(bodies[0], bodies[1]);
30087 }
30088
30089 #[tokio::test]
30090 async fn managed_activity_accepts_authoritative_stale_outcome_after_unknown_completion() {
30091 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
30092 request_override: Some(activity_completion_stale_after_backend_recovery),
30093 ..MockWorkerBehavior::default()
30094 });
30095 let client = Client::builder(server.base_url())
30096 .timeout(Duration::from_secs(2))
30097 .build()
30098 .expect("client");
30099 let handler_calls = Arc::new(AtomicUsize::new(0));
30100 let mut worker = Worker::new(client, "recover-queue")
30101 .worker_id("recover-worker")
30102 .poll_timeout(Duration::from_millis(10));
30103 worker.register_activity("recover.activity", {
30104 let handler_calls = Arc::clone(&handler_calls);
30105 move |_ctx, _args| {
30106 let handler_calls = Arc::clone(&handler_calls);
30107 async move {
30108 handler_calls.fetch_add(1, Ordering::SeqCst);
30109 Ok(json!("done"))
30110 }
30111 }
30112 });
30113
30114 assert_eq!(
30115 worker
30116 .run_once()
30117 .await
30118 .expect("authoritative stale outcome"),
30119 1
30120 );
30121 assert_eq!(handler_calls.load(Ordering::SeqCst), 1);
30122 let bodies = server.request_bodies("/api/worker/activity-tasks/recover-activity/complete");
30123 assert_eq!(bodies.len(), 2);
30124 assert_eq!(bodies[0], bodies[1]);
30125 }
30126
30127 #[test]
30128 fn fenced_completion_retry_rejects_mismatched_or_incomplete_identity() {
30129 let request = json!({"lease_owner":"recover-worker","activity_attempt_id":"recover-attempt","result":{"codec":"avro","blob":"value"}});
30130 let response = json!({
30131 "reason":"backend_unavailable", "operation":"complete_activity_task",
30132 "outcome":"unknown", "worker_id":"recover-worker", "task_queue":null,
30133 "retryable":true, "retry_after_seconds":1, "task_id":"recover-activity",
30134 "lease_owner":"recover-worker", "activity_attempt_id":"recover-attempt"
30135 });
30136 let path = "/worker/activity-tasks/recover-activity/complete";
30137 let error = |body: Value| Error::Http {
30138 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
30139 body: body.to_string(),
30140 };
30141 assert_eq!(
30142 worker_fenced_completion_retry_after(&error(response.clone()), path, Some(&request)),
30143 Some(Duration::from_secs(1))
30144 );
30145
30146 for (field, invalid) in [
30147 ("operation", json!("poll_activity_task")),
30148 ("outcome", json!("rejected")),
30149 ("worker_id", json!("another-worker")),
30150 ("task_queue", json!("recover-queue")),
30151 ("retryable", json!(false)),
30152 ("retry_after_seconds", json!(0)),
30153 ("task_id", json!("another-task")),
30154 ("lease_owner", json!("another-worker")),
30155 ("activity_attempt_id", json!("another-attempt")),
30156 ] {
30157 let mut invalid_response = response.clone();
30158 invalid_response[field] = invalid;
30159 assert_eq!(
30160 worker_fenced_completion_retry_after(
30161 &error(invalid_response),
30162 path,
30163 Some(&request)
30164 ),
30165 None,
30166 "must not retry mismatched {field}"
30167 );
30168 }
30169 assert_eq!(
30170 worker_fenced_completion_retry_after(&error(response), path, None),
30171 None
30172 );
30173 }
30174
30175 #[tokio::test]
30176 async fn shutdown_interrupts_fenced_completion_backend_wait() {
30177 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
30178 request_override: Some(activity_completion_remains_unavailable),
30179 ..MockWorkerBehavior::default()
30180 });
30181 let client = Client::builder(server.base_url())
30182 .timeout(Duration::from_secs(2))
30183 .build()
30184 .expect("client");
30185 let stop = Arc::new(AtomicBool::new(false));
30186 let worker = Worker::new(client, "recover-queue")
30187 .worker_id("recover-worker")
30188 .with_storage_admission(Arc::clone(&stop));
30189 let completion = tokio::spawn(async move {
30190 worker
30191 .client
30192 .complete_activity_task(
30193 "recover-activity",
30194 "recover-attempt",
30195 "recover-worker",
30196 json!("done"),
30197 DEFAULT_CODEC,
30198 )
30199 .await
30200 });
30201 let path = "/api/worker/activity-tasks/recover-activity/complete";
30202 tokio::time::timeout(Duration::from_secs(2), async {
30203 while server.request_count(path) == 0 {
30204 tokio::time::sleep(Duration::from_millis(10)).await;
30205 }
30206 })
30207 .await
30208 .expect("first completion request");
30209 stop.store(true, Ordering::SeqCst);
30210 let error = tokio::time::timeout(Duration::from_secs(1), completion)
30211 .await
30212 .expect("shutdown must interrupt retry wait")
30213 .expect("completion task")
30214 .expect_err("backend is still unavailable");
30215 assert!(matches!(error, Error::ActivityTaskRejected(rejection) if rejection.status == 503));
30216 assert_eq!(server.request_count(path), 1);
30217 }
30218}