1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{type_name, Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use chrono::DateTime;
20use futures_util::{future::OptionFuture, task::noop_waker_ref};
21use serde::{
22 de::DeserializeOwned,
23 ser::{SerializeMap, SerializeSeq},
24 Deserialize, Deserializer, Serialize, Serializer,
25};
26pub use serde_json::{json, Value};
27use sha2::{Digest, Sha256};
28use thiserror::Error;
29pub use uuid::Uuid;
30
31pub const WORKER_PROTOCOL_VERSION: &str = "1.17";
32pub const CONTROL_PLANE_VERSION: &str = "2";
33pub const DEFAULT_CODEC: &str = "avro";
34pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
35pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
37 "condition_wait_occurrence_identity";
38pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
40pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
42pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
44pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
46pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
48pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
49pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
50pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
51pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
52pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
53pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
55pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
57pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
59pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
61pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
63pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
65
66pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
67 let Some((major, minor)) = version.split_once('.') else {
68 return false;
69 };
70 major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
71}
72
73fn validate_user_signal_name(signal_name: &str) -> Result<()> {
74 if signal_name == MESSAGE_STREAM_SIGNAL {
75 return Err(Error::Codec(format!(
76 "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
77 )));
78 }
79 Ok(())
80}
81
82const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
83const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
84 "Workflow task waiting for scheduled history.";
85const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
86const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
87const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
88const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
89const MAX_MEMO_ENTRIES: usize = 100;
90const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
91const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
92
93const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
94 "lease_expired",
95 "query_task_not_found",
96 "query_task_not_leased",
97 "query_task_timed_out",
98];
99
100pub const AVRO_VALUE_SCHEMA_JSON: &str =
102 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
103pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
104pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
105const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
106
107static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
108static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
109 OnceLock::new();
110
111#[derive(Clone, Copy)]
112enum RequestProtocol {
113 ControlPlane,
114 Worker(&'static str),
115}
116
117pub type Result<T> = std::result::Result<T, Error>;
118
119#[derive(Debug, Error)]
120pub enum Error {
121 #[error("transport error: {0}")]
122 Transport(#[from] reqwest::Error),
123 #[error(
124 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
125 )]
126 InvalidBaseUrl,
127 #[error("json error: {0}")]
128 Json(#[from] serde_json::Error),
129 #[error("http {status}: {body}")]
130 Http {
131 status: reqwest::StatusCode,
132 body: String,
133 },
134 #[error("codec error: {0}")]
135 Codec(String),
136 #[error(transparent)]
137 QueryFailed(QueryFailure),
138 #[error(transparent)]
139 Protocol(ProtocolFailure),
140 #[error(transparent)]
141 NonDeterministicReplay(ReplayFailure),
142 #[error(transparent)]
143 ChildWorkflowFailed(ChildWorkflowFailure),
144 #[error(transparent)]
145 ActivityFailed(ActivityFailure),
146 #[error(transparent)]
147 ParallelFailed(ParallelFailure),
148 #[error(transparent)]
149 SagaCompensationFailed(SagaCompensationFailure),
150 #[error(transparent)]
151 InvalidParallelGroup(ParallelGroupError),
152 #[error(transparent)]
153 WorkflowCancellationRequested(WorkflowCancellationRequested),
154 #[error(transparent)]
155 WorkflowCommandRejected(WorkflowCommandRejection),
156 #[error(transparent)]
157 WorkflowFailed(WorkflowTerminalOutcome),
158 #[error(transparent)]
159 WorkflowCancelled(WorkflowTerminalOutcome),
160 #[error(transparent)]
161 WorkflowTerminated(WorkflowTerminalOutcome),
162 #[error(transparent)]
163 WorkflowTimedOut(WorkflowTerminalOutcome),
164 #[error(transparent)]
165 ActivityTaskRejected(ActivityTaskRejection),
166 #[error("workflow handler {0:?} is not registered")]
167 WorkflowNotRegistered(String),
168 #[error("activity handler {0:?} is not registered")]
169 ActivityNotRegistered(String),
170 #[error(
171 "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
172 )]
173 HandlerType {
174 handler_kind: HandlerKind,
175 handler_name: String,
176 value_kind: HandlerValueKind,
177 rust_type: &'static str,
178 message: String,
179 },
180 #[error("workflow future yielded without emitting a durable command")]
181 WorkflowYieldedWithoutCommand,
182 #[error(
183 "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
184 )]
185 MissingWorkflowCommandIdentity,
186 #[error("workflow state lock is poisoned")]
187 WorkflowStatePoisoned,
188 #[error("timer duration is too large for the worker protocol")]
189 TimerDurationOverflow,
190 #[error(transparent)]
191 InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
192 #[error(transparent)]
193 InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
194 #[error("operation timed out")]
195 Timeout,
196 #[error(
197 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
198 )]
199 MissingRoleCredentials {
200 role: &'static str,
201 opposite_role: &'static str,
202 },
203 #[error("worker loop error: {0}")]
204 WorkerLoop(String),
205 #[error(
206 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
207 )]
208 UnsupportedUpdateValidators { workflow_type: String },
209 #[error("{primary}; worker deregistration also failed: {deregistration}")]
210 WorkerShutdown {
211 primary: Box<Error>,
212 deregistration: Box<Error>,
213 },
214 #[error("invalid child workflow options: {0}")]
215 InvalidChildWorkflowOptions(String),
216 #[error("invalid workflow memo update: {0}")]
217 InvalidMemoUpdate(String),
218 #[error(
219 "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
220 )]
221 WorkflowMemoUpdatesUnavailable,
222 #[error(transparent)]
223 InvalidActivityOptions(ActivityOptionsError),
224 #[error(transparent)]
225 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
226 #[doc(hidden)]
227 #[error("workflow requested continue as new")]
228 ContinueAsNew(ContinueAsNewRequest),
229}
230
231#[derive(Clone, Debug, Error, PartialEq, Eq)]
233pub enum ConditionWaitOptionsError {
234 #[error("condition_key must be non-empty")]
235 EmptyKey,
236 #[error("condition_definition_fingerprint must be non-empty")]
237 EmptyPredicateIdentity,
238 #[error("condition timeout is too large for the worker protocol")]
239 TimeoutOverflow,
240}
241
242#[derive(Clone, Debug, PartialEq, Eq)]
249pub struct ConditionWaitOptions {
250 condition_key: String,
251 predicate_identity: String,
252 timeout: Option<Duration>,
253}
254
255impl ConditionWaitOptions {
256 pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
257 Self {
258 condition_key: condition_key.into(),
259 predicate_identity: predicate_identity.into(),
260 timeout: None,
261 }
262 }
263
264 pub fn timeout(mut self, timeout: Duration) -> Self {
265 self.timeout = Some(timeout);
266 self
267 }
268
269 fn validate(
270 &self,
271 ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
272 let condition_key = self.condition_key.trim();
273 if condition_key.is_empty() {
274 return Err(ConditionWaitOptionsError::EmptyKey);
275 }
276 let predicate_identity = self.predicate_identity.trim();
277 if predicate_identity.is_empty() {
278 return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
279 }
280 let timeout_seconds = self
281 .timeout
282 .map(|timeout| {
283 timeout
284 .as_secs()
285 .checked_add(u64::from(timeout.subsec_nanos() > 0))
286 .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
287 })
288 .transpose()?;
289
290 Ok(ValidatedConditionWaitOptions {
291 condition_key: condition_key.to_string(),
292 predicate_identity: predicate_identity.to_string(),
293 timeout_seconds,
294 })
295 }
296}
297
298#[derive(Clone, Debug, PartialEq, Eq)]
299struct ValidatedConditionWaitOptions {
300 condition_key: String,
301 predicate_identity: String,
302 timeout_seconds: Option<u64>,
303}
304
305const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
306
307#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
309#[serde(rename_all = "snake_case")]
310pub enum ConditionWaitResult {
311 Satisfied,
312 TimedOut,
313}
314
315impl ConditionWaitResult {
316 pub fn is_satisfied(self) -> bool {
317 self == Self::Satisfied
318 }
319
320 pub fn is_timed_out(self) -> bool {
321 self == Self::TimedOut
322 }
323}
324
325#[doc(hidden)]
327pub fn __condition_definition_fingerprint(source: &str) -> String {
328 let mut digest = Sha256::new();
329 digest.update(b"durable-workflow-rust.wait-condition.v1\0");
330 digest.update(source.as_bytes());
331 format!("sha256:{:x}", digest.finalize())
332}
333
334#[macro_export]
340macro_rules! wait_condition {
341 ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
342 $ctx.wait_condition(
343 $crate::ConditionWaitOptions::new(
344 $key,
345 $crate::__condition_definition_fingerprint(concat!(
346 module_path!(),
347 "\0",
348 stringify!($predicate)
349 )),
350 )
351 .timeout($timeout),
352 $predicate,
353 )
354 }};
355 ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
356 $ctx.wait_condition(
357 $crate::ConditionWaitOptions::new(
358 $key,
359 $crate::__condition_definition_fingerprint(concat!(
360 module_path!(),
361 "\0",
362 stringify!($predicate)
363 )),
364 ),
365 $predicate,
366 )
367 }};
368}
369
370const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
371const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
372const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 255;
373const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
374const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
375
376#[derive(Clone, Debug, Error, PartialEq, Eq)]
378pub enum SearchAttributeUpdateError {
379 #[error("search-attribute update requires at least one attribute")]
380 Empty,
381 #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
382 InvalidKey(String),
383 #[error("search-attribute update exceeds the limit of 100 attributes")]
384 TooManyAttributes,
385 #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
386 ValueTooLong {
387 key: String,
388 kind: &'static str,
389 limit: usize,
390 },
391 #[error(
392 "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
393 )]
394 EmptyString(String),
395 #[error("search attribute {0:?} has a non-finite float value")]
396 NonFiniteFloat(String),
397 #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
398 InvalidDateTime(String),
399 #[error("search-attribute update exceeds the 65536-byte protocol limit")]
400 PayloadTooLarge,
401}
402
403#[derive(Clone, Debug, PartialEq)]
405pub enum SearchAttributeValue {
406 String(String),
407 Keyword(String),
408 KeywordList(Vec<String>),
409 Int(i64),
410 Float(f64),
411 Bool(bool),
412 DateTime(String),
413 Delete,
414}
415
416impl SearchAttributeValue {
417 fn type_name(&self) -> Option<&'static str> {
418 match self {
419 Self::String(_) => Some("string"),
420 Self::Keyword(_) => Some("keyword"),
421 Self::KeywordList(_) => Some("keyword_list"),
422 Self::Int(_) => Some("int"),
423 Self::Float(_) => Some("float"),
424 Self::Bool(_) => Some("bool"),
425 Self::DateTime(_) => Some("datetime"),
426 Self::Delete => None,
427 }
428 }
429
430 fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
431 let normalize_string = |value: String, kind: &'static str, limit: usize| {
432 let value = value.trim().to_string();
433 if value.is_empty() {
434 return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
435 }
436 if value.len() > limit {
437 return Err(SearchAttributeUpdateError::ValueTooLong {
438 key: key.to_string(),
439 kind,
440 limit,
441 });
442 }
443 Ok(value)
444 };
445
446 match self {
447 Self::String(value) => Ok(Self::String(normalize_string(
448 value,
449 "string",
450 MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
451 )?)),
452 Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
453 value,
454 "keyword",
455 MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
456 )?)),
457 Self::KeywordList(values) => {
458 let values = values
459 .into_iter()
460 .map(|value| {
461 let value = value.trim().to_string();
462 if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
463 return Err(SearchAttributeUpdateError::ValueTooLong {
464 key: key.to_string(),
465 kind: "keyword-list entry",
466 limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
467 });
468 }
469 Ok(value)
470 })
471 .collect::<std::result::Result<Vec<_>, _>>()?;
472 Ok(Self::KeywordList(values))
473 }
474 Self::Float(value) if !value.is_finite() => {
475 Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
476 }
477 Self::DateTime(value) => {
478 let value =
479 normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
480 if DateTime::parse_from_rfc3339(&value).is_err() {
481 return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
482 }
483 Ok(Self::DateTime(value))
484 }
485 value => Ok(value),
486 }
487 }
488
489 fn into_json(self) -> Value {
490 match self {
491 Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
492 Value::String(value)
493 }
494 Self::KeywordList(values) => {
495 Value::Array(values.into_iter().map(Value::String).collect())
496 }
497 Self::Int(value) => json!(value),
498 Self::Float(value) => json!(value),
499 Self::Bool(value) => json!(value),
500 Self::Delete => Value::Null,
501 }
502 }
503}
504
505#[derive(Clone, Debug, Default, PartialEq)]
507pub struct SearchAttributeUpdate {
508 attributes: BTreeMap<String, SearchAttributeValue>,
509}
510
511impl SearchAttributeUpdate {
512 pub fn new() -> Self {
513 Self::default()
514 }
515
516 pub fn set(
517 mut self,
518 key: impl Into<String>,
519 value: SearchAttributeValue,
520 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
521 let key = key.into();
522 validate_search_attribute_key(&key)?;
523 if !self.attributes.contains_key(&key)
524 && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
525 {
526 return Err(SearchAttributeUpdateError::TooManyAttributes);
527 }
528 self.attributes.insert(key.clone(), value.normalized(&key)?);
529 self.validate_size()?;
530 Ok(self)
531 }
532
533 pub fn string(
534 self,
535 key: impl Into<String>,
536 value: impl Into<String>,
537 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
538 self.set(key, SearchAttributeValue::String(value.into()))
539 }
540
541 pub fn keyword(
542 self,
543 key: impl Into<String>,
544 value: impl Into<String>,
545 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
546 self.set(key, SearchAttributeValue::Keyword(value.into()))
547 }
548
549 pub fn keyword_list<I, V>(
550 self,
551 key: impl Into<String>,
552 values: I,
553 ) -> std::result::Result<Self, SearchAttributeUpdateError>
554 where
555 I: IntoIterator<Item = V>,
556 V: Into<String>,
557 {
558 self.set(
559 key,
560 SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
561 )
562 }
563
564 pub fn int(
565 self,
566 key: impl Into<String>,
567 value: i64,
568 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
569 self.set(key, SearchAttributeValue::Int(value))
570 }
571
572 pub fn float(
573 self,
574 key: impl Into<String>,
575 value: f64,
576 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
577 self.set(key, SearchAttributeValue::Float(value))
578 }
579
580 pub fn bool(
581 self,
582 key: impl Into<String>,
583 value: bool,
584 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
585 self.set(key, SearchAttributeValue::Bool(value))
586 }
587
588 pub fn datetime(
589 self,
590 key: impl Into<String>,
591 value: impl Into<String>,
592 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
593 self.set(key, SearchAttributeValue::DateTime(value.into()))
594 }
595
596 pub fn delete(
597 self,
598 key: impl Into<String>,
599 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
600 self.set(key, SearchAttributeValue::Delete)
601 }
602
603 fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
604 let (attributes, _) = self.clone().into_wire_parts();
605 if serde_json::to_vec(&attributes)
606 .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
607 .unwrap_or(true)
608 {
609 return Err(SearchAttributeUpdateError::PayloadTooLarge);
610 }
611 Ok(())
612 }
613
614 fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
615 let mut attributes = serde_json::Map::new();
616 let mut attribute_types = BTreeMap::new();
617 for (key, value) in self.attributes {
618 if let Some(type_name) = value.type_name() {
619 attribute_types.insert(key.clone(), type_name.to_string());
620 }
621 attributes.insert(key, value.into_json());
622 }
623 (Value::Object(attributes), attribute_types)
624 }
625
626 fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
627 if self.attributes.is_empty() {
628 return Err(SearchAttributeUpdateError::Empty);
629 }
630 self.validate_size()
631 }
632}
633
634fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
635 let valid = !key.is_empty()
636 && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
637 && key
638 .bytes()
639 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
640 if valid {
641 Ok(())
642 } else {
643 Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
644 }
645}
646
647#[derive(Clone, Copy, Debug, PartialEq, Eq)]
649pub enum HandlerKind {
650 Workflow,
651 Activity,
652}
653
654impl std::fmt::Display for HandlerKind {
655 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
656 formatter.write_str(match self {
657 Self::Workflow => "workflow",
658 Self::Activity => "activity",
659 })
660 }
661}
662
663#[derive(Clone, Copy, Debug, PartialEq, Eq)]
665pub enum HandlerValueKind {
666 Input,
667 Result,
668}
669
670impl std::fmt::Display for HandlerValueKind {
671 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
672 formatter.write_str(match self {
673 Self::Input => "input",
674 Self::Result => "result",
675 })
676 }
677}
678
679#[derive(Clone, Copy, Debug, PartialEq, Eq)]
681pub enum WorkflowCommandKind {
682 Cancel,
683 Terminate,
684}
685
686impl WorkflowCommandKind {
687 fn as_str(self) -> &'static str {
688 match self {
689 Self::Cancel => "cancel",
690 Self::Terminate => "terminate",
691 }
692 }
693}
694
695#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
697pub struct WorkflowCommandOptions {
698 #[serde(skip_serializing_if = "Option::is_none")]
699 pub reason: Option<String>,
700 #[serde(skip_serializing_if = "Option::is_none")]
701 pub request_id: Option<String>,
702}
703
704#[derive(Clone, Debug, PartialEq, Eq)]
711pub struct WorkflowStartOptions {
712 pub execution_timeout_seconds: u64,
713 pub run_timeout_seconds: u64,
714}
715
716impl Default for WorkflowStartOptions {
717 fn default() -> Self {
718 Self {
719 execution_timeout_seconds: 3600,
720 run_timeout_seconds: 600,
721 }
722 }
723}
724
725impl WorkflowStartOptions {
726 pub fn new() -> Self {
727 Self::default()
728 }
729
730 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
731 self.execution_timeout_seconds = seconds;
732 self
733 }
734
735 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
736 self.run_timeout_seconds = seconds;
737 self
738 }
739
740 fn validate(&self) -> Result<()> {
741 if self.execution_timeout_seconds == 0 {
742 return Err(Error::Codec(
743 "execution_timeout_seconds must be at least 1".to_string(),
744 ));
745 }
746 if self.run_timeout_seconds == 0 {
747 return Err(Error::Codec(
748 "run_timeout_seconds must be at least 1".to_string(),
749 ));
750 }
751 if self.run_timeout_seconds > self.execution_timeout_seconds {
752 return Err(Error::Codec(
753 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
754 ));
755 }
756
757 Ok(())
758 }
759}
760
761#[derive(Clone, Debug, Default, PartialEq, Eq)]
766pub struct ContinueAsNewOptions {
767 pub workflow_type: Option<String>,
768 pub task_queue: Option<String>,
769}
770
771impl ContinueAsNewOptions {
772 pub fn new() -> Self {
773 Self::default()
774 }
775
776 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
777 self.workflow_type = Some(workflow_type.into());
778 self
779 }
780
781 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
782 self.task_queue = Some(task_queue.into());
783 self
784 }
785
786 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
787 for (field, value) in [
788 ("workflow_type", self.workflow_type.as_deref()),
789 ("task_queue", self.task_queue.as_deref()),
790 ] {
791 if value.is_some_and(|value| value.trim().is_empty()) {
792 return Err(ContinueAsNewOptionsError {
793 field,
794 message: format!("{field} must not be empty"),
795 });
796 }
797 }
798 Ok(())
799 }
800}
801
802#[derive(Clone, Debug, Error, PartialEq, Eq)]
804#[error("invalid continue-as-new option {field}: {message}")]
805pub struct ContinueAsNewOptionsError {
806 pub field: &'static str,
807 pub message: String,
808}
809
810#[derive(Clone, Debug, Default, PartialEq, Eq)]
812pub struct WorkflowHistoryBudget {
813 pub event_count: u64,
814 pub size_bytes: Option<u64>,
815 pub continue_as_new_recommended: bool,
816 pub pressure: Option<String>,
817}
818
819#[doc(hidden)]
820#[derive(Clone, Debug)]
821pub struct ContinueAsNewRequest {
822 arguments: AvroValue,
823 options: ContinueAsNewOptions,
824}
825
826impl WorkflowCommandOptions {
827 pub fn new() -> Self {
828 Self::default()
829 }
830
831 pub fn reason(mut self, reason: impl Into<String>) -> Self {
832 self.reason = Some(reason.into());
833 self
834 }
835
836 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
837 self.request_id = Some(request_id.into());
838 self
839 }
840}
841
842#[derive(Clone, Debug, PartialEq)]
844pub struct WorkflowCommandResult {
845 pub command: WorkflowCommandKind,
846 pub workflow_id: String,
847 pub run_id: Option<String>,
848 pub outcome: Option<String>,
849 pub reason: Option<String>,
850 pub command_status: Option<String>,
851 pub raw: Value,
852}
853
854#[derive(Clone, Debug, Error)]
856#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
857pub struct WorkflowCommandRejection {
858 pub command: WorkflowCommandKind,
859 pub status: u16,
860 pub reason: String,
861 pub message: String,
862 pub workflow_id: String,
863 pub run_id: Option<String>,
864 pub target_scope: Option<String>,
865 pub body: Value,
866}
867
868#[derive(Clone, Copy, Debug, PartialEq, Eq)]
870pub enum WorkflowTerminalKind {
871 Failed,
872 Cancelled,
873 Terminated,
874 TimedOut,
875}
876
877#[derive(Clone, Debug, Error)]
883#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
884pub struct WorkflowTerminalOutcome {
885 pub kind: WorkflowTerminalKind,
886 pub workflow_id: String,
887 pub run_id: Option<String>,
888 pub reason: String,
889 pub failure_category: Option<String>,
890 pub failure_id: Option<String>,
891 pub exception_type: Option<String>,
892 pub exception_class: Option<String>,
893 pub non_retryable: Option<bool>,
894 pub message: Option<String>,
895 pub exception: Option<Value>,
896 pub raw: Value,
897}
898
899#[derive(Clone, Debug, Error)]
901#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
902pub struct ActivityTaskRejection {
903 pub operation: String,
904 pub status: u16,
905 pub reason: String,
906 pub task_id: String,
907 pub activity_attempt_id: String,
908 pub cancel_requested: bool,
909 pub can_continue: Option<bool>,
910 pub run_closed_reason: Option<String>,
911 pub body: Value,
912}
913
914#[derive(Clone, Copy, Debug, PartialEq, Eq)]
916pub enum ActivityOptionsErrorKind {
917 EmptyTaskQueue,
918 EmptyRetryPolicy,
919 InvalidMaxAttempts,
920 BackoffWithoutRetryBudget,
921 TooManyBackoffIntervals,
922 InvalidBackoffCoefficient,
923 BackoffGenerationTooLarge,
924 BackoffOverflow,
925 EmptyNonRetryableErrorType,
926 TimeoutNotPositive,
927 TimeoutOverflow,
928 TimeoutOrder,
929}
930
931#[derive(Clone, Debug, Error, PartialEq, Eq)]
933#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
934pub struct ActivityOptionsError {
935 pub kind: ActivityOptionsErrorKind,
936 pub field: Option<&'static str>,
937 pub message: String,
938}
939
940impl ActivityOptionsError {
941 fn new(
942 kind: ActivityOptionsErrorKind,
943 field: Option<&'static str>,
944 message: impl Into<String>,
945 ) -> Self {
946 Self {
947 kind,
948 field,
949 message: message.into(),
950 }
951 }
952}
953
954#[derive(Clone, Copy, Debug, PartialEq, Eq)]
956pub enum ActivityFailureKind {
957 Failed,
958 Cancelled,
959 TimedOut,
960}
961
962#[derive(Clone, Debug, Error)]
967#[error("activity failed ({reason}): {message}")]
968pub struct ActivityFailure {
969 pub kind: ActivityFailureKind,
970 pub reason: String,
971 pub message: String,
972 pub activity_execution_id: Option<String>,
973 pub activity_attempt_id: Option<String>,
974 pub activity_type: Option<String>,
975 pub activity_class: Option<String>,
976 pub attempt_number: Option<u64>,
977 pub failure_id: Option<String>,
978 pub failure_category: Option<String>,
979 pub timeout_kind: Option<String>,
980 pub non_retryable: bool,
981 pub exception_type: Option<String>,
982 pub exception_class: Option<String>,
983 pub code: Option<Value>,
984 pub exception: Option<Value>,
985}
986
987#[derive(Clone, Copy, Debug, PartialEq, Eq)]
989pub enum ChildWorkflowFailureKind {
990 Failed,
991 Cancelled,
992 Terminated,
993}
994
995#[derive(Clone, Debug, Error)]
1001#[error("child workflow failed ({reason}): {message}")]
1002pub struct ChildWorkflowFailure {
1003 pub kind: ChildWorkflowFailureKind,
1004 pub reason: String,
1005 pub message: String,
1006 pub parent_workflow_id: Option<String>,
1007 pub parent_workflow_run_id: Option<String>,
1008 pub child_workflow_id: Option<String>,
1009 pub child_workflow_run_id: Option<String>,
1010 pub child_workflow_type: Option<String>,
1011 pub failure_id: Option<String>,
1012 pub failure_category: Option<String>,
1013 pub exception_type: Option<String>,
1014 pub exception_class: Option<String>,
1015 pub non_retryable: bool,
1016 pub code: Option<Value>,
1017 pub exception: Option<Value>,
1018}
1019
1020#[derive(Clone, Debug, PartialEq, Eq)]
1022pub struct WorkflowIdentity {
1023 pub workflow_id: Option<String>,
1024 pub run_id: Option<String>,
1025}
1026
1027#[derive(Clone, Debug, PartialEq)]
1029pub struct ChildWorkflowResult {
1030 pub parent: WorkflowIdentity,
1031 pub child: WorkflowIdentity,
1032 pub child_workflow_type: Option<String>,
1033 pub result: Value,
1034}
1035
1036#[derive(Clone, Debug, PartialEq)]
1038pub struct ChildWorkflowAvroResult {
1039 pub parent: WorkflowIdentity,
1040 pub child: WorkflowIdentity,
1041 pub child_workflow_type: Option<String>,
1042 pub result: AvroValue,
1043}
1044
1045#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1051pub struct ParallelGroupMetadata {
1052 pub parallel_group_id: String,
1053 pub parallel_group_kind: String,
1054 pub parallel_group_base_sequence: u64,
1055 pub parallel_group_size: usize,
1056 pub parallel_group_index: usize,
1057}
1058
1059#[derive(Clone, Debug, PartialEq)]
1061pub enum ParallelResult {
1062 Activity(Value),
1063 ChildWorkflow(ChildWorkflowResult),
1064 Timer,
1065 Group(Vec<ParallelResult>),
1066}
1067
1068#[derive(Clone, Debug, PartialEq)]
1070pub enum ParallelAvroResult {
1071 Activity(AvroValue),
1072 ChildWorkflow(ChildWorkflowAvroResult),
1073 Timer,
1074 Group(Vec<ParallelAvroResult>),
1075}
1076
1077impl ParallelAvroResult {
1078 fn into_json_result(self) -> Result<ParallelResult> {
1079 match self {
1080 Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1081 Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1082 parent: result.parent,
1083 child: result.child,
1084 child_workflow_type: result.child_workflow_type,
1085 result: result.result.into_json()?,
1086 })),
1087 Self::Timer => Ok(ParallelResult::Timer),
1088 Self::Group(results) => Ok(ParallelResult::Group(
1089 results
1090 .into_iter()
1091 .map(Self::into_json_result)
1092 .collect::<Result<Vec<_>>>()?,
1093 )),
1094 }
1095 }
1096}
1097
1098#[derive(Clone, Debug, PartialEq)]
1100pub struct ParallelCompletion {
1101 pub member_path: Vec<usize>,
1102 pub result: ParallelResult,
1103}
1104
1105#[derive(Debug, Error)]
1113#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1114pub struct ParallelFailure {
1115 pub group_id: String,
1116 pub member_path: Vec<usize>,
1117 pub group_path: Vec<ParallelGroupMetadata>,
1118 pub completed: Vec<ParallelCompletion>,
1119 #[source]
1120 pub cause: Box<Error>,
1121}
1122
1123#[derive(Clone, Debug, Error, PartialEq, Eq)]
1125#[error("invalid deterministic parallel group ({reason}): {message}")]
1126pub struct ParallelGroupError {
1127 pub reason: &'static str,
1128 pub member_path: Vec<usize>,
1129 pub message: String,
1130}
1131
1132#[derive(Clone, Debug, Error, PartialEq, Eq)]
1134#[error("workflow cancellation was requested")]
1135pub struct WorkflowCancellationRequested;
1136
1137#[derive(Debug, Error)]
1139#[error(
1140 "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1141)]
1142pub struct SagaCompensationFailure {
1143 pub initiating_failure: Box<Error>,
1144 pub compensation_failure: Box<Error>,
1145 pub compensation_activity_type: String,
1146 pub compensation_registration_order: usize,
1147}
1148
1149#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1151pub enum ParentClosePolicy {
1152 #[default]
1153 Abandon,
1154 RequestCancel,
1155 Terminate,
1156}
1157
1158impl ParentClosePolicy {
1159 fn as_str(self) -> &'static str {
1160 match self {
1161 Self::Abandon => "abandon",
1162 Self::RequestCancel => "request_cancel",
1163 Self::Terminate => "terminate",
1164 }
1165 }
1166}
1167
1168#[derive(Clone, Debug, Default, PartialEq, Eq)]
1170pub struct ChildWorkflowRetryPolicy {
1171 pub max_attempts: Option<u32>,
1172 pub backoff_seconds: Vec<u64>,
1173 pub non_retryable_error_types: Vec<String>,
1174}
1175
1176#[derive(Clone, Debug, PartialEq, Eq)]
1180pub struct ChildWorkflowOptions {
1181 pub task_queue: String,
1182 pub parent_close_policy: ParentClosePolicy,
1183 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1184 pub execution_timeout_seconds: Option<u64>,
1185 pub run_timeout_seconds: Option<u64>,
1186}
1187
1188impl ChildWorkflowOptions {
1189 pub fn new(task_queue: impl Into<String>) -> Self {
1190 Self {
1191 task_queue: task_queue.into(),
1192 parent_close_policy: ParentClosePolicy::Abandon,
1193 retry_policy: None,
1194 execution_timeout_seconds: None,
1195 run_timeout_seconds: None,
1196 }
1197 }
1198
1199 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1200 self.parent_close_policy = policy;
1201 self
1202 }
1203
1204 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1205 self.retry_policy = Some(policy);
1206 self
1207 }
1208
1209 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1210 self.execution_timeout_seconds = Some(seconds);
1211 self
1212 }
1213
1214 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1215 self.run_timeout_seconds = Some(seconds);
1216 self
1217 }
1218}
1219
1220#[derive(Clone, Debug, PartialEq, Eq)]
1222pub enum ActivityBackoff {
1223 Explicit(Vec<Duration>),
1226 Exponential {
1228 initial_interval: Duration,
1229 coefficient: u32,
1230 maximum_interval: Option<Duration>,
1231 },
1232}
1233
1234#[derive(Clone, Debug, Default, PartialEq, Eq)]
1236pub struct ActivityRetryPolicy {
1237 pub max_attempts: Option<u32>,
1238 pub backoff: Option<ActivityBackoff>,
1239 pub non_retryable_error_types: Vec<String>,
1240}
1241
1242impl ActivityRetryPolicy {
1243 pub fn new(max_attempts: u32) -> Self {
1245 Self {
1246 max_attempts: Some(max_attempts),
1247 ..Self::default()
1248 }
1249 }
1250
1251 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1252 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1253 self
1254 }
1255
1256 pub fn exponential_backoff(
1257 mut self,
1258 initial_interval: Duration,
1259 coefficient: u32,
1260 maximum_interval: Option<Duration>,
1261 ) -> Self {
1262 self.backoff = Some(ActivityBackoff::Exponential {
1263 initial_interval,
1264 coefficient,
1265 maximum_interval,
1266 });
1267 self
1268 }
1269
1270 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1271 self.non_retryable_error_types.push(error_type.into());
1272 self
1273 }
1274
1275 pub fn non_retryable_error_types(
1276 mut self,
1277 error_types: impl IntoIterator<Item = impl Into<String>>,
1278 ) -> Self {
1279 self.non_retryable_error_types
1280 .extend(error_types.into_iter().map(Into::into));
1281 self
1282 }
1283}
1284
1285#[derive(Clone, Debug, Default, PartialEq, Eq)]
1290pub struct ActivityOptions {
1291 pub task_queue: Option<String>,
1292 pub retry_policy: Option<ActivityRetryPolicy>,
1293 pub start_to_close_timeout: Option<Duration>,
1294 pub schedule_to_start_timeout: Option<Duration>,
1295 pub schedule_to_close_timeout: Option<Duration>,
1296 pub heartbeat_timeout: Option<Duration>,
1297}
1298
1299impl ActivityOptions {
1300 pub fn new() -> Self {
1301 Self::default()
1302 }
1303
1304 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1305 self.task_queue = Some(task_queue.into());
1306 self
1307 }
1308
1309 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1310 self.retry_policy = Some(policy);
1311 self
1312 }
1313
1314 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1315 self.start_to_close_timeout = Some(timeout);
1316 self
1317 }
1318
1319 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1320 self.schedule_to_start_timeout = Some(timeout);
1321 self
1322 }
1323
1324 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1325 self.schedule_to_close_timeout = Some(timeout);
1326 self
1327 }
1328
1329 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1330 self.heartbeat_timeout = Some(timeout);
1331 self
1332 }
1333
1334 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1335 if self
1336 .task_queue
1337 .as_deref()
1338 .is_some_and(|queue| queue.trim().is_empty())
1339 {
1340 return Err(ActivityOptionsError::new(
1341 ActivityOptionsErrorKind::EmptyTaskQueue,
1342 Some("task_queue"),
1343 "task_queue must not be empty",
1344 ));
1345 }
1346
1347 for (field, value) in [
1348 ("start_to_close_timeout", self.start_to_close_timeout),
1349 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1350 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1351 ("heartbeat_timeout", self.heartbeat_timeout),
1352 ] {
1353 if value.is_some_and(|value| value.is_zero()) {
1354 return Err(ActivityOptionsError::new(
1355 ActivityOptionsErrorKind::TimeoutNotPositive,
1356 Some(field),
1357 format!("{field} must be positive"),
1358 ));
1359 }
1360 }
1361
1362 validate_timeout_order(
1363 "heartbeat_timeout",
1364 self.heartbeat_timeout,
1365 "start_to_close_timeout",
1366 self.start_to_close_timeout,
1367 )?;
1368 validate_timeout_order(
1369 "start_to_close_timeout",
1370 self.start_to_close_timeout,
1371 "schedule_to_close_timeout",
1372 self.schedule_to_close_timeout,
1373 )?;
1374 validate_timeout_order(
1375 "schedule_to_start_timeout",
1376 self.schedule_to_start_timeout,
1377 "schedule_to_close_timeout",
1378 self.schedule_to_close_timeout,
1379 )?;
1380
1381 Ok(ValidatedActivityOptions {
1382 task_queue: self.task_queue.clone(),
1383 retry_policy: self
1384 .retry_policy
1385 .as_ref()
1386 .map(validate_activity_retry_policy)
1387 .transpose()?,
1388 start_to_close_timeout: timeout_seconds(
1389 "start_to_close_timeout",
1390 self.start_to_close_timeout,
1391 )?,
1392 schedule_to_start_timeout: timeout_seconds(
1393 "schedule_to_start_timeout",
1394 self.schedule_to_start_timeout,
1395 )?,
1396 schedule_to_close_timeout: timeout_seconds(
1397 "schedule_to_close_timeout",
1398 self.schedule_to_close_timeout,
1399 )?,
1400 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1401 })
1402 }
1403}
1404
1405pub enum ParallelOperation {
1411 Activity {
1412 activity_type: String,
1413 options: ActivityOptions,
1414 arguments: Result<AvroValue>,
1415 },
1416 ChildWorkflow {
1417 workflow_type: String,
1418 options: ChildWorkflowOptions,
1419 arguments: Result<AvroValue>,
1420 },
1421 Timer(Duration),
1422 Group(Vec<ParallelOperation>),
1423}
1424
1425impl ParallelOperation {
1426 pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1427 Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1428 }
1429
1430 pub fn activity_with_options<T: Serialize>(
1431 activity_type: impl Into<String>,
1432 options: ActivityOptions,
1433 args: T,
1434 ) -> Self {
1435 Self::Activity {
1436 activity_type: activity_type.into(),
1437 options,
1438 arguments: AvroValue::from_serialize(&args),
1439 }
1440 }
1441
1442 pub fn child_workflow<T: Serialize>(
1443 workflow_type: impl Into<String>,
1444 options: ChildWorkflowOptions,
1445 args: T,
1446 ) -> Self {
1447 Self::ChildWorkflow {
1448 workflow_type: workflow_type.into(),
1449 options,
1450 arguments: AvroValue::from_serialize(&args),
1451 }
1452 }
1453
1454 pub fn timer(duration: Duration) -> Self {
1455 Self::Timer(duration)
1456 }
1457
1458 pub fn group(operations: Vec<ParallelOperation>) -> Self {
1459 Self::Group(operations)
1460 }
1461}
1462
1463#[derive(Clone, Debug)]
1464struct ValidatedActivityOptions {
1465 task_queue: Option<String>,
1466 retry_policy: Option<Value>,
1467 start_to_close_timeout: Option<u64>,
1468 schedule_to_start_timeout: Option<u64>,
1469 schedule_to_close_timeout: Option<u64>,
1470 heartbeat_timeout: Option<u64>,
1471}
1472
1473fn validate_timeout_order(
1474 smaller_name: &'static str,
1475 smaller: Option<Duration>,
1476 larger_name: &'static str,
1477 larger: Option<Duration>,
1478) -> std::result::Result<(), ActivityOptionsError> {
1479 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1480 return Err(ActivityOptionsError::new(
1481 ActivityOptionsErrorKind::TimeoutOrder,
1482 Some(smaller_name),
1483 format!("{smaller_name} must be <= {larger_name}"),
1484 ));
1485 }
1486 Ok(())
1487}
1488
1489fn timeout_seconds(
1490 field: &'static str,
1491 value: Option<Duration>,
1492) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1493 value
1494 .map(|value| {
1495 activity_protocol_seconds(value).ok_or_else(|| {
1496 ActivityOptionsError::new(
1497 ActivityOptionsErrorKind::TimeoutOverflow,
1498 Some(field),
1499 format!("{field} is too large for the worker protocol"),
1500 )
1501 })
1502 })
1503 .transpose()
1504}
1505
1506fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1507 value
1508 .as_secs()
1509 .checked_add(u64::from(value.subsec_nanos() > 0))
1510}
1511
1512fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1513 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1514}
1515
1516fn validate_activity_retry_policy(
1517 policy: &ActivityRetryPolicy,
1518) -> std::result::Result<Value, ActivityOptionsError> {
1519 if policy.max_attempts.is_none()
1520 && policy.backoff.is_none()
1521 && policy.non_retryable_error_types.is_empty()
1522 {
1523 return Err(ActivityOptionsError::new(
1524 ActivityOptionsErrorKind::EmptyRetryPolicy,
1525 Some("retry_policy"),
1526 "retry_policy must configure at least one field",
1527 ));
1528 }
1529 if policy.max_attempts == Some(0) {
1530 return Err(ActivityOptionsError::new(
1531 ActivityOptionsErrorKind::InvalidMaxAttempts,
1532 Some("retry_policy.max_attempts"),
1533 "max_attempts must be >= 1",
1534 ));
1535 }
1536 if policy
1537 .non_retryable_error_types
1538 .iter()
1539 .any(|error_type| error_type.trim().is_empty())
1540 {
1541 return Err(ActivityOptionsError::new(
1542 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1543 Some("retry_policy.non_retryable_error_types"),
1544 "non_retryable_error_types must not contain empty values",
1545 ));
1546 }
1547
1548 let backoff_seconds = match &policy.backoff {
1549 None => None,
1550 Some(backoff) => {
1551 let max_attempts = policy.max_attempts.ok_or_else(|| {
1552 ActivityOptionsError::new(
1553 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1554 Some("retry_policy.backoff"),
1555 "backoff requires max_attempts",
1556 )
1557 })?;
1558 let retry_count = max_attempts.saturating_sub(1) as usize;
1559 let intervals = match backoff {
1560 ActivityBackoff::Explicit(intervals) => {
1561 if intervals.len() > retry_count {
1562 return Err(ActivityOptionsError::new(
1563 ActivityOptionsErrorKind::TooManyBackoffIntervals,
1564 Some("retry_policy.backoff"),
1565 "backoff interval count must not exceed max_attempts - 1",
1566 ));
1567 }
1568 intervals.clone()
1569 }
1570 ActivityBackoff::Exponential {
1571 initial_interval,
1572 coefficient,
1573 maximum_interval,
1574 } => {
1575 if *coefficient < 1 {
1576 return Err(ActivityOptionsError::new(
1577 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1578 Some("retry_policy.backoff.coefficient"),
1579 "backoff coefficient must be >= 1",
1580 ));
1581 }
1582 if retry_count > 10_000 {
1583 return Err(ActivityOptionsError::new(
1584 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1585 Some("retry_policy.max_attempts"),
1586 "generated backoff supports at most 10000 retry intervals",
1587 ));
1588 }
1589 let mut current = *initial_interval;
1590 let mut intervals = Vec::with_capacity(retry_count);
1591 for _ in 0..retry_count {
1592 let interval = maximum_interval
1593 .map(|maximum| current.min(maximum))
1594 .unwrap_or(current);
1595 intervals.push(interval);
1596 if maximum_interval.is_some_and(|maximum| interval == maximum) {
1597 break;
1598 }
1599 current = current.checked_mul(*coefficient).ok_or_else(|| {
1600 ActivityOptionsError::new(
1601 ActivityOptionsErrorKind::BackoffOverflow,
1602 Some("retry_policy.backoff"),
1603 "generated backoff interval overflowed",
1604 )
1605 })?;
1606 }
1607 intervals
1608 }
1609 };
1610 Some(
1611 intervals
1612 .into_iter()
1613 .map(|interval| {
1614 activity_protocol_seconds(interval).ok_or_else(|| {
1615 ActivityOptionsError::new(
1616 ActivityOptionsErrorKind::BackoffOverflow,
1617 Some("retry_policy.backoff"),
1618 "backoff interval is too large for the worker protocol",
1619 )
1620 })
1621 })
1622 .collect::<std::result::Result<Vec<_>, _>>()?,
1623 )
1624 }
1625 };
1626
1627 let mut encoded = serde_json::Map::new();
1628 if let Some(max_attempts) = policy.max_attempts {
1629 encoded.insert("max_attempts".to_string(), json!(max_attempts));
1630 }
1631 if let Some(backoff_seconds) = backoff_seconds {
1632 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1633 }
1634 if !policy.non_retryable_error_types.is_empty() {
1635 let mut canonical_error_types = Vec::new();
1636 for error_type in policy
1637 .non_retryable_error_types
1638 .iter()
1639 .map(|error_type| error_type.trim())
1640 {
1641 if !canonical_error_types.contains(&error_type) {
1642 canonical_error_types.push(error_type);
1643 }
1644 }
1645 encoded.insert(
1646 "non_retryable_error_types".to_string(),
1647 json!(canonical_error_types),
1648 );
1649 }
1650 Ok(Value::Object(encoded))
1651}
1652
1653#[derive(Clone, Debug, Error)]
1656#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1657pub struct ReplayFailure {
1658 pub reason: String,
1659 pub sequence: Option<u64>,
1660 pub expected: Option<String>,
1661 pub actual: Option<String>,
1662 pub message: String,
1663}
1664
1665impl ReplayFailure {
1666 fn new(
1667 reason: impl Into<String>,
1668 sequence: Option<u64>,
1669 expected: Option<String>,
1670 actual: Option<String>,
1671 message: impl Into<String>,
1672 ) -> Self {
1673 Self {
1674 reason: reason.into(),
1675 sequence,
1676 expected,
1677 actual,
1678 message: message.into(),
1679 }
1680 }
1681}
1682
1683#[derive(Clone, Debug, Error)]
1685#[error("query failed ({reason}, HTTP {status}): {message}")]
1686pub struct QueryFailure {
1687 pub status: u16,
1688 pub reason: String,
1689 pub message: String,
1690 pub body: Value,
1691}
1692
1693#[derive(Clone, Debug, Error)]
1695#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1696pub struct ProtocolFailure {
1697 pub status: u16,
1698 pub reason: String,
1699 pub message: String,
1700 pub supported_version: Option<String>,
1701 pub requested_version: Option<String>,
1702 pub body: Value,
1703}
1704
1705#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1706pub struct PayloadEnvelope {
1707 pub codec: String,
1708 pub blob: String,
1709}
1710
1711impl PayloadEnvelope {
1712 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1713 encode_payload(value, DEFAULT_CODEC)
1714 }
1715
1716 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1719 encode_avro_value(value)
1720 }
1721}
1722
1723#[derive(Clone, Debug)]
1725pub enum AvroValue {
1726 Null,
1727 Boolean(bool),
1728 Long(i64),
1729 Double(f64),
1730 Bytes(Vec<u8>),
1731 String(String),
1732 Array(Vec<AvroValue>),
1733 Map(BTreeMap<String, AvroValue>),
1734}
1735
1736impl PartialEq for AvroValue {
1737 fn eq(&self, other: &Self) -> bool {
1738 match (self, other) {
1739 (Self::Null, Self::Null) => true,
1740 (Self::Boolean(left), Self::Boolean(right)) => left == right,
1741 (Self::Long(left), Self::Long(right)) => left == right,
1742 (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1743 (Self::Bytes(left), Self::Bytes(right)) => left == right,
1744 (Self::String(left), Self::String(right)) => left == right,
1745 (Self::Array(left), Self::Array(right)) => left == right,
1746 (Self::Map(left), Self::Map(right)) => left == right,
1747 _ => false,
1748 }
1749 }
1750}
1751
1752impl AvroValue {
1753 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1754 Self::from_serde_value(
1755 serde_value::to_value(value).map_err(|error| {
1756 Error::Codec(format!("could not adapt value for Avro: {error}"))
1757 })?,
1758 )
1759 }
1760
1761 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1762 use serde_value::Value as SerdeValue;
1763
1764 match value {
1765 SerdeValue::Unit => Ok(Self::Null),
1766 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1767 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1768 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1769 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1770 SerdeValue::I64(value) => Ok(Self::Long(value)),
1771 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1772 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1773 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1774 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1775 Error::Codec(
1776 "integer_overflow: Avro Value long must be within signed 64-bit range"
1777 .to_string(),
1778 )
1779 }),
1780 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1781 SerdeValue::F64(value) => Self::finite_double(value),
1782 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1783 SerdeValue::String(value) => Ok(Self::String(value)),
1784 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1785 SerdeValue::Option(None) => Ok(Self::Null),
1786 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1787 Self::from_serde_value(*value)
1788 }
1789 SerdeValue::Seq(values) => values
1790 .into_iter()
1791 .map(Self::from_serde_value)
1792 .collect::<Result<Vec<_>>>()
1793 .map(Self::Array),
1794 SerdeValue::Map(values) => values
1795 .into_iter()
1796 .map(|(key, value)| {
1797 let SerdeValue::String(key) = key else {
1798 return Err(Error::Codec(
1799 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1800 ));
1801 };
1802
1803 Ok((key, Self::from_serde_value(value)?))
1804 })
1805 .collect::<Result<BTreeMap<_, _>>>()
1806 .map(Self::Map),
1807 }
1808 }
1809
1810 fn finite_double(value: f64) -> Result<Self> {
1811 if !value.is_finite() {
1812 return Err(Error::Codec(
1813 "non_finite_float: Avro Value doubles must be finite".to_string(),
1814 ));
1815 }
1816
1817 Ok(Self::Double(value))
1818 }
1819
1820 fn into_json(self) -> Result<Value> {
1821 match self {
1822 Self::Null => Ok(Value::Null),
1823 Self::Boolean(value) => Ok(Value::Bool(value)),
1824 Self::Long(value) => Ok(Value::Number(value.into())),
1825 Self::Double(value) => serde_json::Number::from_f64(value)
1826 .map(Value::Number)
1827 .ok_or_else(|| {
1828 Error::Codec(
1829 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1830 )
1831 }),
1832 Self::Bytes(value) => Ok(json!({
1833 "$type": "bytes",
1834 "base64": BASE64.encode(value),
1835 })),
1836 Self::String(value) => Ok(Value::String(value)),
1837 Self::Array(values) => values
1838 .into_iter()
1839 .map(Self::into_json)
1840 .collect::<Result<Vec<_>>>()
1841 .map(Value::Array),
1842 Self::Map(values) => values
1843 .into_iter()
1844 .map(|(key, value)| Ok((key, value.into_json()?)))
1845 .collect::<Result<serde_json::Map<_, _>>>()
1846 .map(Value::Object),
1847 }
1848 }
1849
1850 fn into_serde_value(self) -> serde_value::Value {
1851 use serde_value::Value as SerdeValue;
1852
1853 match self {
1854 Self::Null => SerdeValue::Unit,
1855 Self::Boolean(value) => SerdeValue::Bool(value),
1856 Self::Long(value) => SerdeValue::I64(value),
1857 Self::Double(value) => SerdeValue::F64(value),
1858 Self::Bytes(value) => SerdeValue::Bytes(value),
1859 Self::String(value) => SerdeValue::String(value),
1860 Self::Array(values) => {
1861 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1862 }
1863 Self::Map(values) => SerdeValue::Map(
1864 values
1865 .into_iter()
1866 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1867 .collect(),
1868 ),
1869 }
1870 }
1871
1872 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1873 self.into_serde_value().deserialize_into().map_err(|error| {
1874 Error::Codec(format!(
1875 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1876 ))
1877 })
1878 }
1879}
1880
1881impl Serialize for AvroValue {
1882 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1883 where
1884 S: Serializer,
1885 {
1886 match self {
1887 Self::Null => serializer.serialize_unit(),
1888 Self::Boolean(value) => serializer.serialize_bool(*value),
1889 Self::Long(value) => serializer.serialize_i64(*value),
1890 Self::Double(value) => serializer.serialize_f64(*value),
1891 Self::Bytes(value) => serializer.serialize_bytes(value),
1892 Self::String(value) => serializer.serialize_str(value),
1893 Self::Array(values) => {
1894 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1895 for value in values {
1896 sequence.serialize_element(value)?;
1897 }
1898 sequence.end()
1899 }
1900 Self::Map(values) => {
1901 let mut map = serializer.serialize_map(Some(values.len()))?;
1902 for (key, value) in values {
1903 map.serialize_entry(key, value)?;
1904 }
1905 map.end()
1906 }
1907 }
1908 }
1909}
1910
1911pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1912 let datum = avro_value_to_datum(value)?;
1913 let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
1914 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1915 let mut bytes = Vec::with_capacity(datum.len() + 10);
1916 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1917 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1918 bytes.extend_from_slice(&datum);
1919 Ok(PayloadEnvelope {
1920 codec: DEFAULT_CODEC.to_string(),
1921 blob: BASE64.encode(bytes),
1922 })
1923}
1924
1925pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1926 if envelope.codec != DEFAULT_CODEC {
1927 return Err(unsupported_payload_codec(&envelope.codec));
1928 }
1929 decode_avro_value_blob(&envelope.blob)
1930}
1931
1932pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1933 let blob = match codec {
1934 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1935 other => return Err(unsupported_payload_codec(other)),
1936 };
1937
1938 Ok(PayloadEnvelope {
1939 codec: codec.to_string(),
1940 blob,
1941 })
1942}
1943
1944pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1945 match envelope.codec.as_str() {
1946 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1947 other => Err(unsupported_payload_codec(other)),
1948 }
1949}
1950
1951fn handler_type_error<T>(
1952 handler_kind: HandlerKind,
1953 handler_name: &str,
1954 value_kind: HandlerValueKind,
1955 message: impl Into<String>,
1956) -> Error {
1957 Error::HandlerType {
1958 handler_kind,
1959 handler_name: handler_name.to_string(),
1960 value_kind,
1961 rust_type: type_name::<T>(),
1962 message: message.into(),
1963 }
1964}
1965
1966fn decode_handler_input<T: DeserializeOwned>(
1967 arguments: AvroValue,
1968 handler_kind: HandlerKind,
1969 handler_name: &str,
1970) -> Result<T> {
1971 let argument = match arguments {
1972 AvroValue::Array(mut arguments) if arguments.len() == 1 => {
1973 arguments.pop().expect("one typed handler argument")
1974 }
1975 AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
1976 AvroValue::Array(arguments) => {
1977 return Err(handler_type_error::<T>(
1978 handler_kind,
1979 handler_name,
1980 HandlerValueKind::Input,
1981 format!(
1982 "typed handlers accept one request value, but the task carried {} arguments",
1983 arguments.len()
1984 ),
1985 ));
1986 }
1987 argument => argument,
1988 };
1989
1990 argument.deserialize().map_err(|error| {
1991 handler_type_error::<T>(
1992 handler_kind,
1993 handler_name,
1994 HandlerValueKind::Input,
1995 error.to_string(),
1996 )
1997 })
1998}
1999
2000fn encode_handler_result<T: Serialize>(
2001 result: &T,
2002 handler_kind: HandlerKind,
2003 handler_name: &str,
2004) -> Result<AvroValue> {
2005 AvroValue::from_serialize(result).map_err(|error| {
2006 handler_type_error::<T>(
2007 handler_kind,
2008 handler_name,
2009 HandlerValueKind::Result,
2010 error.to_string(),
2011 )
2012 })
2013}
2014
2015fn decode_handler_result<T: DeserializeOwned>(
2016 result: AvroValue,
2017 handler_kind: HandlerKind,
2018 handler_name: &str,
2019) -> Result<T> {
2020 result.deserialize().map_err(|error| {
2021 handler_type_error::<T>(
2022 handler_kind,
2023 handler_name,
2024 HandlerValueKind::Result,
2025 error.to_string(),
2026 )
2027 })
2028}
2029
2030#[cfg(test)]
2031fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2032 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2033}
2034
2035fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2036 validate_payload_codec(fallback_codec)?;
2037
2038 if value.is_null() {
2039 return Ok(Value::Null);
2040 }
2041
2042 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2043 return decode_blob(blob, codec);
2044 }
2045
2046 if let Some(blob) = value.as_str() {
2047 return decode_blob(blob, fallback_codec);
2048 }
2049
2050 Err(untagged_payload_value())
2051}
2052
2053fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2054 let envelope = match codec {
2055 DEFAULT_CODEC => encode_avro_value(value)?,
2056 other => return Err(unsupported_payload_codec(other)),
2057 };
2058 Ok(serde_json::to_value(envelope)?)
2059}
2060
2061fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2062 validate_payload_codec(fallback_codec)?;
2063
2064 if value.is_null() {
2065 return Ok(AvroValue::Null);
2066 }
2067
2068 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2069 validate_payload_codec(codec)?;
2070 return decode_avro_value_blob(blob);
2071 }
2072
2073 if let Some(blob) = value.as_str() {
2074 return match fallback_codec {
2075 DEFAULT_CODEC => decode_avro_value_blob(blob),
2076 other => Err(unsupported_payload_codec(other)),
2077 };
2078 }
2079
2080 Err(untagged_payload_value())
2081}
2082
2083fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2084 match value {
2085 AvroValue::Null => AvroValue::Array(Vec::new()),
2086 AvroValue::Array(_) => value,
2087 other => AvroValue::Array(vec![other]),
2088 }
2089}
2090
2091fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2092 match codec {
2093 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2094 other => Err(unsupported_payload_codec(other)),
2095 }
2096}
2097
2098fn validate_payload_codec(codec: &str) -> Result<()> {
2099 match codec {
2100 DEFAULT_CODEC => Ok(()),
2101 MISSING_TASK_PAYLOAD_CODEC => {
2102 Err(invalid_task_payload_codec("task payload_codec is missing"))
2103 }
2104 NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2105 NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2106 "task payload_codec must be a string",
2107 )),
2108 other => Err(unsupported_payload_codec(other)),
2109 }
2110}
2111
2112fn invalid_task_payload_codec(reason: &str) -> Error {
2113 Error::Codec(format!(
2114 "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2115 ))
2116}
2117
2118fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2119 let Some(object) = value.as_object() else {
2120 return Ok(None);
2121 };
2122 if !object.contains_key("codec") && !object.contains_key("blob") {
2123 return Ok(None);
2124 }
2125
2126 let codec = object
2127 .get("codec")
2128 .and_then(Value::as_str)
2129 .ok_or_else(invalid_payload_envelope)?;
2130 validate_payload_codec(codec)?;
2131 let blob = object
2132 .get("blob")
2133 .and_then(Value::as_str)
2134 .ok_or_else(invalid_payload_envelope)?;
2135 Ok(Some((codec, blob)))
2136}
2137
2138fn invalid_payload_envelope() -> Error {
2139 Error::Codec(
2140 "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2141 .to_string(),
2142 )
2143}
2144
2145fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2146 for command in commands {
2147 let Some(command) = command.as_object() else {
2148 continue;
2149 };
2150 let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2151 continue;
2152 };
2153 let Some(payload_field) = workflow_command_payload_field(command_type) else {
2154 continue;
2155 };
2156
2157 if let Some(codec) = command.get("payload_codec") {
2158 let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2159 validate_payload_codec(codec)?;
2160 }
2161
2162 let payload = command
2163 .get(payload_field)
2164 .ok_or_else(invalid_payload_envelope)?;
2165 validate_outbound_payload_envelope(payload)?;
2166 }
2167 Ok(())
2168}
2169
2170fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2171 if commands.iter().any(|command| {
2172 command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2173 && command
2174 .get("condition_wait_occurrence_id")
2175 .and_then(Value::as_str)
2176 .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2177 }) {
2178 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2179 } else if commands.iter().any(|command| {
2180 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2181 && command.get("attribute_types").is_some()
2182 }) {
2183 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2184 } else if commands
2185 .iter()
2186 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2187 {
2188 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2189 } else if commands
2190 .iter()
2191 .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2192 {
2193 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2194 } else if commands.iter().any(|command| {
2195 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2196 }) {
2197 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2198 } else {
2199 WORKER_PROTOCOL_VERSION
2200 }
2201}
2202
2203fn workflow_completion_protocol_version_with_message_streams(
2204 commands: &[Value],
2205 has_message_stream_metadata: bool,
2206) -> &'static str {
2207 let command_protocol = workflow_completion_protocol_version(commands);
2208 if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2209 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2210 } else {
2211 command_protocol
2212 }
2213}
2214
2215fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2216 match command_type {
2217 "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2218 "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2219 "start_service_operation" => Some("request_payload"),
2220 "upsert_memo" => Some("entries"),
2221 _ => None,
2222 }
2223}
2224
2225fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2226 let Some((codec, blob)) = payload_envelope_parts(value)? else {
2227 return Err(untagged_payload_value());
2228 };
2229 validate_payload_codec(codec)?;
2230 decode_avro_value_blob(blob)?;
2231 Ok(())
2232}
2233
2234fn unsupported_payload_codec(codec: &str) -> Error {
2235 Error::Codec(format!(
2236 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2237 ))
2238}
2239
2240fn untagged_payload_value() -> Error {
2241 Error::Codec(
2242 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2243 .to_string(),
2244 )
2245}
2246
2247fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2248 let bytes = BASE64.decode(blob).map_err(|err| {
2249 Error::Codec(format!(
2250 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2251 ))
2252 })?;
2253
2254 if serde_json::from_slice::<Value>(&bytes).is_ok() {
2255 return Err(unsupported_payload_codec("json"));
2256 }
2257
2258 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2259 return Err(Error::Codec(
2260 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2261 ));
2262 }
2263
2264 let fingerprint: [u8; 8] = bytes[2..10]
2265 .try_into()
2266 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2267 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2268 return Err(Error::Codec(format!(
2269 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2270 fingerprint
2271 .iter()
2272 .map(|byte| format!("{byte:02x}"))
2273 .collect::<String>()
2274 )));
2275 }
2276
2277 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2278 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2282 if datum_reader.truncated {
2283 return Err(Error::Codec(
2284 "invalid_payload_framing: truncated Avro Value datum".to_string(),
2285 ));
2286 }
2287 let datum = datum.map_err(|err| {
2288 Error::Codec(format!(
2289 "invalid_payload_framing: malformed Avro Value datum: {err}"
2290 ))
2291 })?;
2292 if datum_reader.remaining() != 0 {
2293 return Err(Error::Codec(format!(
2294 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2295 datum_reader.remaining()
2296 )));
2297 }
2298 avro_value_from_datum(datum)
2299}
2300
2301struct StrictAvroDatumReader<'a> {
2302 bytes: &'a [u8],
2303 offset: usize,
2304 truncated: bool,
2305}
2306
2307impl<'a> StrictAvroDatumReader<'a> {
2308 fn new(bytes: &'a [u8]) -> Self {
2309 Self {
2310 bytes,
2311 offset: 0,
2312 truncated: false,
2313 }
2314 }
2315
2316 fn remaining(&self) -> usize {
2317 self.bytes.len() - self.offset
2318 }
2319}
2320
2321impl Read for StrictAvroDatumReader<'_> {
2322 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2323 let count = buffer.len().min(self.remaining());
2324 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2325 self.offset += count;
2326 if count < buffer.len() {
2327 self.truncated = true;
2328 }
2329
2330 Ok(count)
2331 }
2332}
2333
2334fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2335 let branch = match value {
2336 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2337 AvroValue::Boolean(value) => AvroDatum::Union(
2338 1,
2339 Box::new(AvroDatum::Record(vec![(
2340 "boolean".to_string(),
2341 AvroDatum::Boolean(*value),
2342 )])),
2343 ),
2344 AvroValue::Long(value) => AvroDatum::Union(
2345 2,
2346 Box::new(AvroDatum::Record(vec![(
2347 "long".to_string(),
2348 AvroDatum::Long(*value),
2349 )])),
2350 ),
2351 AvroValue::Double(value) => {
2352 if !value.is_finite() {
2353 return Err(Error::Codec(
2354 "non_finite_float: Avro Value doubles must be finite".to_string(),
2355 ));
2356 }
2357 AvroDatum::Union(
2358 3,
2359 Box::new(AvroDatum::Record(vec![(
2360 "double".to_string(),
2361 AvroDatum::Double(*value),
2362 )])),
2363 )
2364 }
2365 AvroValue::Bytes(value) => AvroDatum::Union(
2366 4,
2367 Box::new(AvroDatum::Record(vec![(
2368 "bytes".to_string(),
2369 AvroDatum::Bytes(value.clone()),
2370 )])),
2371 ),
2372 AvroValue::String(value) => AvroDatum::Union(
2373 5,
2374 Box::new(AvroDatum::Record(vec![(
2375 "string".to_string(),
2376 AvroDatum::String(value.clone()),
2377 )])),
2378 ),
2379 AvroValue::Array(values) => AvroDatum::Union(
2380 6,
2381 Box::new(AvroDatum::Record(vec![(
2382 "items".to_string(),
2383 AvroDatum::Array(
2384 values
2385 .iter()
2386 .map(avro_value_to_datum)
2387 .collect::<Result<Vec<_>>>()?,
2388 ),
2389 )])),
2390 ),
2391 AvroValue::Map(values) => AvroDatum::Union(
2392 7,
2393 Box::new(AvroDatum::Record(vec![(
2394 "entries".to_string(),
2395 AvroDatum::Array(
2396 values
2397 .iter()
2398 .map(|(key, value)| {
2399 Ok(AvroDatum::Record(vec![
2400 ("key".to_string(), AvroDatum::String(key.clone())),
2401 ("value".to_string(), avro_value_to_datum(value)?),
2402 ]))
2403 })
2404 .collect::<Result<Vec<_>>>()?,
2405 ),
2406 )])),
2407 ),
2408 };
2409 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2410}
2411
2412fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2413 let AvroDatum::Record(mut outer) = datum else {
2414 return Err(Error::Codec(
2415 "invalid_payload_framing: datum is not a Value record".to_string(),
2416 ));
2417 };
2418 let (_, branch) = outer
2419 .pop()
2420 .filter(|(name, _)| name == "value")
2421 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2422 let AvroDatum::Union(_, branch) = branch else {
2423 return Err(Error::Codec(
2424 "invalid_payload_framing: invalid Value union".to_string(),
2425 ));
2426 };
2427 match *branch {
2428 AvroDatum::Null => Ok(AvroValue::Null),
2429 AvroDatum::Record(mut fields) => {
2430 let (name, value) = fields.pop().ok_or_else(|| {
2431 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2432 })?;
2433 match (name.as_str(), value) {
2434 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2435 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2436 ("double", AvroDatum::Double(value)) if value.is_finite() => {
2437 Ok(AvroValue::Double(value))
2438 }
2439 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2440 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2441 ("items", AvroDatum::Array(values)) => values
2442 .into_iter()
2443 .map(avro_value_from_datum)
2444 .collect::<Result<Vec<_>>>()
2445 .map(AvroValue::Array),
2446 ("entries", AvroDatum::Map(values)) => values
2447 .into_iter()
2448 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2449 .collect::<Result<BTreeMap<_, _>>>()
2450 .map(AvroValue::Map),
2451 _ => Err(Error::Codec(
2452 "invalid_payload_framing: unknown Value branch".to_string(),
2453 )),
2454 }
2455 }
2456 _ => Err(Error::Codec(
2457 "invalid_payload_framing: invalid Value branch".to_string(),
2458 )),
2459 }
2460}
2461
2462fn avro_value_schema() -> Result<&'static Schema> {
2463 match AVRO_VALUE_SCHEMA.get_or_init(|| {
2464 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2465 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2466 }) {
2467 Ok(schema) => Ok(schema),
2468 Err(message) => Err(Error::Codec(message.clone())),
2469 }
2470}
2471
2472fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2473 match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2474 let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2479 .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2480 let entries_schema = schema
2481 .pointer_mut("/fields/0/type/7/fields/0/type")
2482 .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2483 if *entries_schema != json!({"type": "map", "values": "Value"}) {
2484 return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2485 }
2486 *entries_schema = json!({
2487 "type": "array",
2488 "items": {
2489 "type": "record",
2490 "name": "MapEntry",
2491 "fields": [
2492 {"name": "key", "type": "string"},
2493 {"name": "value", "type": "Value"}
2494 ]
2495 }
2496 });
2497 Schema::parse_str(&schema.to_string())
2498 .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2499 }) {
2500 Ok(schema) => Ok(schema),
2501 Err(message) => Err(Error::Codec(message.clone())),
2502 }
2503}
2504
2505#[derive(Clone, Debug)]
2506pub struct Client {
2507 http: reqwest::Client,
2508 base_url: String,
2509 token: Option<String>,
2510 control_token: Option<String>,
2511 worker_token: Option<String>,
2512 namespace: String,
2513}
2514
2515impl Client {
2516 pub fn new(base_url: impl Into<String>) -> Result<Self> {
2517 Self::builder(base_url).build()
2518 }
2519
2520 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2521 ClientBuilder {
2522 base_url: base_url.into(),
2523 token: None,
2524 control_token: None,
2525 worker_token: None,
2526 namespace: "default".to_string(),
2527 timeout: Duration::from_secs(60),
2528 }
2529 }
2530
2531 pub async fn health(&self) -> Result<Value> {
2532 self.request_json(
2533 reqwest::Method::GET,
2534 "/health",
2535 RequestProtocol::ControlPlane,
2536 Option::<&Value>::None,
2537 )
2538 .await
2539 }
2540
2541 pub async fn cluster_info(&self) -> Result<Value> {
2542 self.request_json(
2543 reqwest::Method::GET,
2544 "/cluster/info",
2545 RequestProtocol::ControlPlane,
2546 Option::<&Value>::None,
2547 )
2548 .await
2549 }
2550
2551 pub async fn start_workflow<T: Serialize>(
2552 &self,
2553 workflow_type: &str,
2554 task_queue: &str,
2555 workflow_id: &str,
2556 input: T,
2557 ) -> Result<WorkflowHandle> {
2558 self.start_workflow_with_options(
2559 workflow_type,
2560 task_queue,
2561 workflow_id,
2562 WorkflowStartOptions::default(),
2563 input,
2564 )
2565 .await
2566 }
2567
2568 pub async fn start_workflow_with_options<T: Serialize>(
2571 &self,
2572 workflow_type: &str,
2573 task_queue: &str,
2574 workflow_id: &str,
2575 options: WorkflowStartOptions,
2576 input: T,
2577 ) -> Result<WorkflowHandle> {
2578 options.validate()?;
2579 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2580 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2581 let body = json!({
2582 "workflow_id": workflow_id,
2583 "workflow_type": workflow_type,
2584 "task_queue": task_queue,
2585 "input": input_envelope,
2586 "execution_timeout_seconds": options.execution_timeout_seconds,
2587 "run_timeout_seconds": options.run_timeout_seconds
2588 });
2589
2590 let data: Value = self
2591 .request_json(
2592 reqwest::Method::POST,
2593 "/workflows",
2594 RequestProtocol::ControlPlane,
2595 Some(&body),
2596 )
2597 .await?;
2598
2599 Ok(WorkflowHandle {
2600 client: self.clone(),
2601 workflow_id: data
2602 .get("workflow_id")
2603 .and_then(Value::as_str)
2604 .unwrap_or(workflow_id)
2605 .to_string(),
2606 run_id: data
2607 .get("run_id")
2608 .and_then(Value::as_str)
2609 .map(str::to_string),
2610 workflow_type: data
2611 .get("workflow_type")
2612 .and_then(Value::as_str)
2613 .unwrap_or(workflow_type)
2614 .to_string(),
2615 })
2616 }
2617
2618 pub async fn signal_workflow<T: Serialize>(
2619 &self,
2620 workflow_id: &str,
2621 signal_name: &str,
2622 input: T,
2623 ) -> Result<Value> {
2624 self.signal_workflow_target(workflow_id, None, signal_name, input)
2625 .await
2626 }
2627
2628 pub async fn append_message_stream<T: Serialize>(
2630 &self,
2631 workflow_id: &str,
2632 stream_name: &str,
2633 message_id: &str,
2634 input: T,
2635 ) -> Result<Value> {
2636 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2637 let body = json!({
2638 "message_id": message_id,
2639 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2640 });
2641 self.request_json(
2642 reqwest::Method::POST,
2643 &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2644 RequestProtocol::ControlPlane,
2645 Some(&body),
2646 )
2647 .await
2648 }
2649
2650 pub async fn signal_workflow_run<T: Serialize>(
2652 &self,
2653 workflow_id: &str,
2654 run_id: &str,
2655 signal_name: &str,
2656 input: T,
2657 ) -> Result<Value> {
2658 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2659 .await
2660 }
2661
2662 async fn signal_workflow_target<T: Serialize>(
2663 &self,
2664 workflow_id: &str,
2665 run_id: Option<&str>,
2666 signal_name: &str,
2667 input: T,
2668 ) -> Result<Value> {
2669 validate_user_signal_name(signal_name)?;
2670 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2671 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2672 let body = json!({
2673 "input": input_envelope
2674 });
2675 let path = match run_id {
2676 Some(run_id) => {
2677 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2678 }
2679 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2680 };
2681 self.request_json(
2682 reqwest::Method::POST,
2683 &path,
2684 RequestProtocol::ControlPlane,
2685 Some(&body),
2686 )
2687 .await
2688 }
2689
2690 pub async fn cancel_workflow(
2692 &self,
2693 workflow_id: &str,
2694 options: WorkflowCommandOptions,
2695 ) -> Result<WorkflowCommandResult> {
2696 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2697 .await
2698 }
2699
2700 pub async fn cancel_workflow_run(
2702 &self,
2703 workflow_id: &str,
2704 run_id: &str,
2705 options: WorkflowCommandOptions,
2706 ) -> Result<WorkflowCommandResult> {
2707 self.workflow_command(
2708 workflow_id,
2709 Some(run_id),
2710 WorkflowCommandKind::Cancel,
2711 options,
2712 )
2713 .await
2714 }
2715
2716 pub async fn terminate_workflow(
2718 &self,
2719 workflow_id: &str,
2720 options: WorkflowCommandOptions,
2721 ) -> Result<WorkflowCommandResult> {
2722 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2723 .await
2724 }
2725
2726 pub async fn terminate_workflow_run(
2728 &self,
2729 workflow_id: &str,
2730 run_id: &str,
2731 options: WorkflowCommandOptions,
2732 ) -> Result<WorkflowCommandResult> {
2733 self.workflow_command(
2734 workflow_id,
2735 Some(run_id),
2736 WorkflowCommandKind::Terminate,
2737 options,
2738 )
2739 .await
2740 }
2741
2742 async fn workflow_command(
2743 &self,
2744 workflow_id: &str,
2745 run_id: Option<&str>,
2746 command: WorkflowCommandKind,
2747 options: WorkflowCommandOptions,
2748 ) -> Result<WorkflowCommandResult> {
2749 let path = match run_id {
2750 Some(run_id) => format!(
2751 "/workflows/{workflow_id}/runs/{run_id}/{}",
2752 command.as_str()
2753 ),
2754 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2755 };
2756 let data = match self
2757 .request_json(
2758 reqwest::Method::POST,
2759 &path,
2760 RequestProtocol::ControlPlane,
2761 Some(&options),
2762 )
2763 .await
2764 {
2765 Ok(data) => data,
2766 Err(Error::Http { status, body }) => {
2767 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2768 command,
2769 status,
2770 body,
2771 workflow_id,
2772 run_id,
2773 )));
2774 }
2775 Err(error) => return Err(error),
2776 };
2777
2778 Ok(workflow_command_result(command, data, workflow_id, run_id))
2779 }
2780
2781 pub async fn query_workflow<T: Serialize>(
2787 &self,
2788 workflow_id: &str,
2789 query_name: &str,
2790 input: T,
2791 ) -> Result<Value> {
2792 self.query_workflow_target(workflow_id, None, query_name, input)
2793 .await
2794 }
2795
2796 pub async fn query_workflow_run<T: Serialize>(
2798 &self,
2799 workflow_id: &str,
2800 run_id: &str,
2801 query_name: &str,
2802 input: T,
2803 ) -> Result<Value> {
2804 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2805 .await
2806 }
2807
2808 pub async fn query_workflow_avro_value<T: Serialize>(
2810 &self,
2811 workflow_id: &str,
2812 query_name: &str,
2813 input: T,
2814 ) -> Result<AvroValue> {
2815 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2816 .await
2817 }
2818
2819 pub async fn query_workflow_run_avro_value<T: Serialize>(
2821 &self,
2822 workflow_id: &str,
2823 run_id: &str,
2824 query_name: &str,
2825 input: T,
2826 ) -> Result<AvroValue> {
2827 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2828 .await
2829 }
2830
2831 async fn query_workflow_avro_value_target<T: Serialize>(
2832 &self,
2833 workflow_id: &str,
2834 run_id: Option<&str>,
2835 query_name: &str,
2836 input: T,
2837 ) -> Result<AvroValue> {
2838 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2839 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2840 let path = match run_id {
2841 Some(run_id) => {
2842 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2843 }
2844 None => format!("/workflows/{workflow_id}/query/{query_name}"),
2845 };
2846 let response: Value = match self
2847 .request_json(
2848 reqwest::Method::POST,
2849 &path,
2850 RequestProtocol::ControlPlane,
2851 Some(&body),
2852 )
2853 .await
2854 {
2855 Ok(response) => response,
2856 Err(Error::Http { status, body }) => {
2857 return Err(Error::QueryFailed(query_failure(status, body)));
2858 }
2859 Err(error) => return Err(error),
2860 };
2861
2862 let envelope = response
2863 .get("result_envelope")
2864 .filter(|envelope| !envelope.is_null())
2865 .ok_or_else(|| {
2866 Error::Codec(
2867 "missing_payload_envelope: typed query result requires result_envelope"
2868 .to_string(),
2869 )
2870 })?;
2871 decode_wire_avro_value(envelope, DEFAULT_CODEC)
2872 }
2873
2874 async fn query_workflow_target<T: Serialize>(
2875 &self,
2876 workflow_id: &str,
2877 run_id: Option<&str>,
2878 query_name: &str,
2879 input: T,
2880 ) -> Result<Value> {
2881 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2882 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2883 let body = json!({
2884 "input": input_envelope
2885 });
2886 let path = match run_id {
2887 Some(run_id) => {
2888 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2889 }
2890 None => format!("/workflows/{workflow_id}/query/{query_name}"),
2891 };
2892 let response: Value = match self
2893 .request_json(
2894 reqwest::Method::POST,
2895 &path,
2896 RequestProtocol::ControlPlane,
2897 Some(&body),
2898 )
2899 .await
2900 {
2901 Ok(response) => response,
2902 Err(Error::Http { status, body }) => {
2903 return Err(Error::QueryFailed(query_failure(status, body)));
2904 }
2905 Err(error) => return Err(error),
2906 };
2907
2908 if let Some(envelope) = response
2909 .get("result_envelope")
2910 .filter(|envelope| !envelope.is_null())
2911 {
2912 return decode_wire_value(envelope, DEFAULT_CODEC);
2913 }
2914
2915 Ok(response.get("result").cloned().unwrap_or(Value::Null))
2916 }
2917
2918 pub async fn update_workflow<T: Serialize>(
2920 &self,
2921 workflow_id: &str,
2922 update_name: &str,
2923 input: T,
2924 request_id: Option<&str>,
2925 ) -> Result<Value> {
2926 let response = self
2927 .update_workflow_response(workflow_id, update_name, input, request_id)
2928 .await?;
2929 if let Some(envelope) = response
2930 .get("result_envelope")
2931 .filter(|envelope| !envelope.is_null())
2932 {
2933 return decode_wire_value(envelope, DEFAULT_CODEC);
2934 }
2935 Ok(response.get("result").cloned().unwrap_or(response))
2936 }
2937
2938 pub async fn update_workflow_avro_value<T: Serialize>(
2940 &self,
2941 workflow_id: &str,
2942 update_name: &str,
2943 input: T,
2944 request_id: Option<&str>,
2945 ) -> Result<AvroValue> {
2946 let response = self
2947 .update_workflow_response(workflow_id, update_name, input, request_id)
2948 .await?;
2949 let envelope = response
2950 .get("result_envelope")
2951 .filter(|envelope| !envelope.is_null())
2952 .ok_or_else(|| {
2953 Error::Codec(
2954 "missing_payload_envelope: typed update result requires result_envelope"
2955 .to_string(),
2956 )
2957 })?;
2958 decode_wire_avro_value(envelope, DEFAULT_CODEC)
2959 }
2960
2961 async fn update_workflow_response<T: Serialize>(
2962 &self,
2963 workflow_id: &str,
2964 update_name: &str,
2965 input: T,
2966 request_id: Option<&str>,
2967 ) -> Result<Value> {
2968 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2969 let mut body = json!({
2970 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
2971 "wait_for": "completed",
2972 });
2973 if let Some(request_id) = request_id {
2974 body["request_id"] = json!(request_id);
2975 }
2976 self.request_json(
2977 reqwest::Method::POST,
2978 &format!("/workflows/{workflow_id}/update/{update_name}"),
2979 RequestProtocol::ControlPlane,
2980 Some(&body),
2981 )
2982 .await
2983 }
2984
2985 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2986 let path = format!("/workflows/{workflow_id}");
2987 let mut data: WorkflowDescription = self
2988 .request_json(
2989 reqwest::Method::GET,
2990 &path,
2991 RequestProtocol::ControlPlane,
2992 Option::<&Value>::None,
2993 )
2994 .await?;
2995 data.decode_payloads()?;
2996 Ok(data)
2997 }
2998
2999 pub async fn describe_workflow_run(
3001 &self,
3002 workflow_id: &str,
3003 run_id: &str,
3004 ) -> Result<WorkflowDescription> {
3005 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3006 let mut data: WorkflowDescription = self
3007 .request_json(
3008 reqwest::Method::GET,
3009 &path,
3010 RequestProtocol::ControlPlane,
3011 Option::<&Value>::None,
3012 )
3013 .await?;
3014 data.decode_payloads()?;
3015 Ok(data)
3016 }
3017
3018 fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3019 let mut path = format!(
3020 "/workflows/{}/runs/{}/streams",
3021 percent_encode_path_segment(workflow_id),
3022 percent_encode_path_segment(run_id),
3023 );
3024 if let Some(stream_name) = stream_name {
3025 path.push('/');
3026 path.push_str(&percent_encode_path_segment(stream_name));
3027 }
3028 path
3029 }
3030
3031 pub async fn list_workflow_streams(
3033 &self,
3034 workflow_id: &str,
3035 run_id: &str,
3036 ) -> Result<Vec<WorkflowStreamDescription>> {
3037 let response: WorkflowStreamListResponse = self
3038 .request_json(
3039 reqwest::Method::GET,
3040 &Self::workflow_stream_path(workflow_id, run_id, None),
3041 RequestProtocol::ControlPlane,
3042 Option::<&Value>::None,
3043 )
3044 .await?;
3045 Ok(response.streams)
3046 }
3047
3048 pub async fn describe_workflow_stream(
3050 &self,
3051 workflow_id: &str,
3052 run_id: &str,
3053 stream_name: &str,
3054 ) -> Result<WorkflowStreamDescription> {
3055 let response: WorkflowStreamDescriptionResponse = self
3056 .request_json(
3057 reqwest::Method::GET,
3058 &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3059 RequestProtocol::ControlPlane,
3060 Option::<&Value>::None,
3061 )
3062 .await?;
3063 Ok(response.stream)
3064 }
3065
3066 pub async fn subscribe_workflow_stream(
3073 &self,
3074 workflow_id: &str,
3075 run_id: &str,
3076 stream_name: &str,
3077 from_offset: u64,
3078 max_items: usize,
3079 wait: Duration,
3080 ) -> Result<WorkflowStreamPage> {
3081 let max_items = max_items.clamp(1, 500);
3082 let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3083 let path = format!(
3084 "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3085 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3086 );
3087 let response: WorkflowStreamPageResponse = self
3088 .request_json_with_timeout(
3089 reqwest::Method::GET,
3090 &path,
3091 RequestProtocol::ControlPlane,
3092 Option::<&Value>::None,
3093 Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3094 )
3095 .await?;
3096
3097 let items = response
3098 .items
3099 .into_iter()
3100 .map(|raw| {
3101 let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3102 let envelope = raw.get("payload").cloned();
3103 let payload = envelope
3104 .as_ref()
3105 .filter(|value| value.get("blob").is_some())
3106 .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3107 .transpose()?
3108 .map(AvroValue::into_json)
3109 .transpose()?;
3110 Ok(WorkflowStreamItem {
3111 offset,
3112 payload,
3113 payload_envelope: envelope,
3114 payload_reference: raw
3115 .get("payload_reference")
3116 .and_then(Value::as_str)
3117 .map(str::to_string),
3118 payload_codec: raw
3119 .get("payload_codec")
3120 .and_then(Value::as_str)
3121 .map(str::to_string),
3122 idempotency_key: raw
3123 .get("idempotency_key")
3124 .and_then(Value::as_str)
3125 .map(str::to_string),
3126 item_type: raw
3127 .get("item_type")
3128 .and_then(Value::as_str)
3129 .map(str::to_string),
3130 content_type: raw
3131 .get("content_type")
3132 .and_then(Value::as_str)
3133 .map(str::to_string),
3134 origin: raw
3135 .get("origin")
3136 .and_then(Value::as_str)
3137 .map(str::to_string),
3138 origin_reference: raw
3139 .get("origin_reference")
3140 .and_then(Value::as_str)
3141 .map(str::to_string),
3142 emitted_at: raw
3143 .get("emitted_at")
3144 .and_then(Value::as_str)
3145 .map(str::to_string),
3146 raw,
3147 })
3148 })
3149 .collect::<Result<Vec<_>>>()?;
3150 Ok(WorkflowStreamPage {
3151 stream: response.stream,
3152 items,
3153 next_offset: response.next_offset,
3154 terminal: response.terminal,
3155 })
3156 }
3157
3158 pub async fn append_workflow_stream(
3160 &self,
3161 workflow_id: &str,
3162 run_id: &str,
3163 stream_name: &str,
3164 items: &[WorkflowStreamAppendItem],
3165 max_pending_items: Option<u64>,
3166 ) -> Result<WorkflowStreamAppendResult> {
3167 if items.is_empty() {
3168 return Err(Error::Codec(
3169 "workflow_stream_items_empty: append requires at least one item".to_string(),
3170 ));
3171 }
3172 let mut body = json!({
3173 "items": items
3174 .iter()
3175 .map(|item| item.wire_value(None))
3176 .collect::<Vec<_>>(),
3177 });
3178 if let Some(max_pending_items) = max_pending_items {
3179 if max_pending_items == 0 {
3180 return Err(Error::Codec(
3181 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3182 .to_string(),
3183 ));
3184 }
3185 body["max_pending_items"] = json!(max_pending_items);
3186 }
3187 let response: WorkflowStreamAppendResponse = self
3188 .request_json(
3189 reqwest::Method::POST,
3190 &format!(
3191 "{}/items",
3192 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3193 ),
3194 RequestProtocol::ControlPlane,
3195 Some(&body),
3196 )
3197 .await?;
3198 Ok(WorkflowStreamAppendResult {
3199 stream: response.stream,
3200 accepted_offsets: response.accepted_offsets,
3201 accepted: response.accepted,
3202 deduped: response.deduped,
3203 })
3204 }
3205
3206 pub async fn close_workflow_stream(
3208 &self,
3209 workflow_id: &str,
3210 run_id: &str,
3211 stream_name: &str,
3212 error_reason: Option<&str>,
3213 retention_seconds: Option<u64>,
3214 ) -> Result<WorkflowStreamDescription> {
3215 let mut body = json!({});
3216 if let Some(error_reason) = error_reason {
3217 body["error_reason"] = json!(error_reason);
3218 }
3219 if let Some(retention_seconds) = retention_seconds {
3220 if retention_seconds == 0 {
3221 return Err(Error::Codec(
3222 "workflow_stream_retention_invalid: retention_seconds must be positive"
3223 .to_string(),
3224 ));
3225 }
3226 body["retention_seconds"] = json!(retention_seconds);
3227 }
3228 let response: WorkflowStreamDescriptionResponse = self
3229 .request_json(
3230 reqwest::Method::POST,
3231 &format!(
3232 "{}/close",
3233 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3234 ),
3235 RequestProtocol::ControlPlane,
3236 Some(&body),
3237 )
3238 .await?;
3239 Ok(response.stream)
3240 }
3241
3242 pub async fn register_worker(
3243 &self,
3244 worker_id: &str,
3245 task_queue: &str,
3246 supported_workflow_types: Vec<String>,
3247 supported_activity_types: Vec<String>,
3248 max_concurrent_workflow_tasks: usize,
3249 max_concurrent_activity_tasks: usize,
3250 ) -> Result<RegisterWorkerResponse> {
3251 self.register_worker_with_capabilities(
3252 worker_id,
3253 task_queue,
3254 supported_workflow_types,
3255 supported_activity_types,
3256 max_concurrent_workflow_tasks,
3257 max_concurrent_activity_tasks,
3258 Vec::new(),
3259 )
3260 .await
3261 }
3262
3263 pub async fn register_worker_with_capabilities(
3265 &self,
3266 worker_id: &str,
3267 task_queue: &str,
3268 supported_workflow_types: Vec<String>,
3269 supported_activity_types: Vec<String>,
3270 max_concurrent_workflow_tasks: usize,
3271 max_concurrent_activity_tasks: usize,
3272 capabilities: Vec<String>,
3273 ) -> Result<RegisterWorkerResponse> {
3274 self.register_worker_with_command_contracts(
3275 worker_id,
3276 task_queue,
3277 supported_workflow_types,
3278 supported_activity_types,
3279 max_concurrent_workflow_tasks,
3280 max_concurrent_activity_tasks,
3281 capabilities,
3282 Value::Object(serde_json::Map::new()),
3283 )
3284 .await
3285 }
3286
3287 #[allow(clippy::too_many_arguments)]
3294 pub async fn register_worker_with_command_contracts(
3295 &self,
3296 worker_id: &str,
3297 task_queue: &str,
3298 supported_workflow_types: Vec<String>,
3299 supported_activity_types: Vec<String>,
3300 max_concurrent_workflow_tasks: usize,
3301 max_concurrent_activity_tasks: usize,
3302 capabilities: Vec<String>,
3303 workflow_command_contracts: Value,
3304 ) -> Result<RegisterWorkerResponse> {
3305 if let Some(contracts) = workflow_command_contracts.as_object() {
3306 for (workflow_type, contract) in contracts {
3307 let Some(update_validators) = contract.get("update_validators") else {
3308 continue;
3309 };
3310 if !update_validators
3311 .as_array()
3312 .is_some_and(|validators| validators.is_empty())
3313 {
3314 return Err(Error::UnsupportedUpdateValidators {
3315 workflow_type: workflow_type.clone(),
3316 });
3317 }
3318 }
3319 }
3320
3321 let mut body = json!({
3322 "worker_id": worker_id,
3323 "task_queue": task_queue,
3324 "runtime": "rust",
3325 "sdk_version": SDK_VERSION,
3326 "supported_workflow_types": supported_workflow_types,
3327 "supported_activity_types": supported_activity_types,
3328 "capabilities": capabilities,
3329 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3330 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3331 });
3332 if workflow_command_contracts
3333 .as_object()
3334 .is_some_and(|contracts| !contracts.is_empty())
3335 {
3336 body["workflow_command_contracts"] = workflow_command_contracts;
3337 }
3338
3339 self.request_json(
3340 reqwest::Method::POST,
3341 "/worker/register",
3342 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3343 Some(&body),
3344 )
3345 .await
3346 }
3347
3348 pub async fn deregister_worker_registration(
3354 &self,
3355 worker_id: &str,
3356 ) -> Result<WorkerDeregistrationEnvelope> {
3357 let path = format!(
3358 "/worker/registrations/{}",
3359 percent_encode_path_segment(worker_id)
3360 );
3361 self.request_json(
3362 reqwest::Method::DELETE,
3363 &path,
3364 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3365 Option::<&Value>::None,
3366 )
3367 .await
3368 }
3369
3370 pub async fn poll_query_task(
3372 &self,
3373 worker_id: &str,
3374 task_queue: &str,
3375 timeout: Duration,
3376 ) -> Result<Option<QueryTask>> {
3377 Ok(self
3378 .poll_query_task_response(worker_id, task_queue, timeout)
3379 .await?
3380 .task)
3381 }
3382
3383 pub async fn poll_query_task_response(
3385 &self,
3386 worker_id: &str,
3387 task_queue: &str,
3388 timeout: Duration,
3389 ) -> Result<PollQueryTaskResponse> {
3390 let poll_request_id = unique_request_id("rust-query-poll");
3391 self.poll_query_task_response_with_request_id(
3392 worker_id,
3393 task_queue,
3394 timeout,
3395 &poll_request_id,
3396 1,
3397 )
3398 .await
3399 }
3400
3401 async fn poll_query_task_response_with_request_id(
3402 &self,
3403 worker_id: &str,
3404 task_queue: &str,
3405 timeout: Duration,
3406 poll_request_id: &str,
3407 transport_retries: usize,
3408 ) -> Result<PollQueryTaskResponse> {
3409 let timeout_seconds = long_poll_timeout_seconds(timeout);
3410 let body = json!({
3411 "worker_id": worker_id,
3412 "task_queue": task_queue,
3413 "poll_request_id": poll_request_id,
3414 "timeout_seconds": timeout_seconds,
3415 });
3416 self.poll_request_json(
3417 "/worker/query-tasks/poll",
3418 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3419 &body,
3420 timeout + Duration::from_secs(5),
3421 transport_retries,
3422 )
3423 .await
3424 }
3425
3426 pub async fn complete_query_task<T: Serialize>(
3428 &self,
3429 query_task_id: &str,
3430 lease_owner: &str,
3431 query_task_attempt: u64,
3432 result: T,
3433 codec: &str,
3434 ) -> Result<Value> {
3435 let typed_result = AvroValue::from_serialize(&result)?;
3436 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3437 self.complete_query_task_with_envelope(
3438 query_task_id,
3439 lease_owner,
3440 query_task_attempt,
3441 typed_result.into_json()?,
3442 result_envelope,
3443 )
3444 .await
3445 }
3446
3447 async fn complete_query_task_with_envelope(
3448 &self,
3449 query_task_id: &str,
3450 lease_owner: &str,
3451 query_task_attempt: u64,
3452 result: Value,
3453 result_envelope: Value,
3454 ) -> Result<Value> {
3455 let body = json!({
3456 "lease_owner": lease_owner,
3457 "query_task_attempt": query_task_attempt,
3458 "result": result,
3459 "result_envelope": result_envelope,
3460 });
3461 let path = format!("/worker/query-tasks/{query_task_id}/complete");
3462 let response = self
3463 .request_json(
3464 reqwest::Method::POST,
3465 &path,
3466 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3467 Some(&body),
3468 )
3469 .await;
3470 query_task_response(response)
3471 }
3472
3473 pub async fn fail_query_task(
3475 &self,
3476 query_task_id: &str,
3477 lease_owner: &str,
3478 query_task_attempt: u64,
3479 message: impl Into<String>,
3480 reason: impl Into<String>,
3481 failure_type: impl Into<String>,
3482 ) -> Result<Value> {
3483 let body = json!({
3484 "lease_owner": lease_owner,
3485 "query_task_attempt": query_task_attempt,
3486 "failure": {
3487 "message": message.into(),
3488 "reason": reason.into(),
3489 "type": failure_type.into(),
3490 }
3491 });
3492 let path = format!("/worker/query-tasks/{query_task_id}/fail");
3493 let response = self
3494 .request_json(
3495 reqwest::Method::POST,
3496 &path,
3497 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3498 Some(&body),
3499 )
3500 .await;
3501 query_task_response(response)
3502 }
3503
3504 pub async fn heartbeat_worker(
3505 &self,
3506 worker_id: &str,
3507 workflow_available: usize,
3508 activity_available: usize,
3509 ) -> Result<Value> {
3510 let body = json!({
3511 "worker_id": worker_id,
3512 "task_slots": {
3513 "workflow_available": workflow_available,
3514 "activity_available": activity_available
3515 },
3516 "process_metrics": {
3517 "process_id": std::process::id(),
3518 "process_uptime_seconds": 0
3519 }
3520 });
3521
3522 self.request_json(
3523 reqwest::Method::POST,
3524 "/worker/heartbeat",
3525 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3526 Some(&body),
3527 )
3528 .await
3529 }
3530
3531 pub async fn poll_workflow_task(
3532 &self,
3533 worker_id: &str,
3534 task_queue: &str,
3535 timeout: Duration,
3536 ) -> Result<Option<WorkflowTask>> {
3537 Ok(self
3538 .poll_workflow_task_response(worker_id, task_queue, timeout)
3539 .await?
3540 .task)
3541 }
3542
3543 pub async fn poll_workflow_task_response(
3544 &self,
3545 worker_id: &str,
3546 task_queue: &str,
3547 timeout: Duration,
3548 ) -> Result<PollWorkflowTaskResponse> {
3549 let poll_request_id = unique_request_id("rust-workflow-poll");
3550 self.poll_workflow_task_response_with_request_id(
3551 worker_id,
3552 task_queue,
3553 timeout,
3554 &poll_request_id,
3555 1,
3556 )
3557 .await
3558 }
3559
3560 async fn poll_workflow_task_response_with_request_id(
3561 &self,
3562 worker_id: &str,
3563 task_queue: &str,
3564 timeout: Duration,
3565 poll_request_id: &str,
3566 transport_retries: usize,
3567 ) -> Result<PollWorkflowTaskResponse> {
3568 let body = json!({
3569 "worker_id": worker_id,
3570 "task_queue": task_queue,
3571 "poll_request_id": poll_request_id,
3572 "timeout_seconds": long_poll_timeout_seconds(timeout),
3573 });
3574 let mut data: PollWorkflowTaskResponse = self
3575 .poll_request_json(
3576 "/worker/workflow-tasks/poll",
3577 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3578 &body,
3579 timeout + Duration::from_secs(5),
3580 transport_retries,
3581 )
3582 .await?;
3583
3584 if let Some(task) = data.task.as_mut() {
3585 self.fetch_remaining_workflow_history(worker_id, task)
3586 .await?;
3587 }
3588
3589 Ok(data)
3590 }
3591
3592 async fn fetch_remaining_workflow_history(
3593 &self,
3594 worker_id: &str,
3595 task: &mut WorkflowTask,
3596 ) -> Result<()> {
3597 let mut next_token = task.next_history_page_token.clone();
3598
3599 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3600 let lease_owner = task
3601 .lease_owner
3602 .clone()
3603 .unwrap_or_else(|| worker_id.to_string());
3604 let page = self
3605 .workflow_task_history_page(
3606 &task.task_id,
3607 &lease_owner,
3608 task.workflow_task_attempt,
3609 &token,
3610 )
3611 .await?;
3612
3613 task.append_history_page(page);
3614
3615 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3616 return Err(Error::Codec(
3617 "workflow history pagination returned the same page token".to_string(),
3618 ));
3619 }
3620
3621 next_token = task.next_history_page_token.clone();
3622 }
3623
3624 Ok(())
3625 }
3626
3627 async fn workflow_task_history_page(
3628 &self,
3629 task_id: &str,
3630 lease_owner: &str,
3631 workflow_task_attempt: u64,
3632 next_history_page_token: &str,
3633 ) -> Result<WorkflowTaskHistoryPage> {
3634 let body = json!({
3635 "lease_owner": lease_owner,
3636 "workflow_task_attempt": workflow_task_attempt,
3637 "next_history_page_token": next_history_page_token
3638 });
3639 let path = format!("/worker/workflow-tasks/{task_id}/history");
3640
3641 self.request_json(
3642 reqwest::Method::POST,
3643 &path,
3644 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3645 Some(&body),
3646 )
3647 .await
3648 }
3649
3650 pub async fn complete_workflow_task(
3651 &self,
3652 task_id: &str,
3653 lease_owner: &str,
3654 workflow_task_attempt: u64,
3655 commands: Vec<Value>,
3656 ) -> Result<Value> {
3657 self.complete_workflow_task_with_message_streams(
3658 task_id,
3659 lease_owner,
3660 workflow_task_attempt,
3661 commands,
3662 Vec::new(),
3663 Vec::new(),
3664 )
3665 .await
3666 }
3667
3668 async fn complete_workflow_task_with_message_streams(
3669 &self,
3670 task_id: &str,
3671 lease_owner: &str,
3672 workflow_task_attempt: u64,
3673 commands: Vec<Value>,
3674 message_stream_cursors: Vec<Value>,
3675 message_stream_waits: Vec<Value>,
3676 ) -> Result<Value> {
3677 validate_workflow_task_commands(&commands)?;
3678 let has_message_stream_metadata =
3679 !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3680 if has_message_stream_metadata
3681 && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3682 {
3683 return Err(Error::Codec(
3684 "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3685 .to_string(),
3686 ));
3687 }
3688 let protocol_version = workflow_completion_protocol_version_with_message_streams(
3689 &commands,
3690 has_message_stream_metadata,
3691 );
3692 let mut body = json!({
3693 "lease_owner": lease_owner,
3694 "workflow_task_attempt": workflow_task_attempt,
3695 "commands": commands
3696 });
3697 if !message_stream_cursors.is_empty() {
3698 body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3699 }
3700 if !message_stream_waits.is_empty() {
3701 body["message_stream_waits"] = Value::Array(message_stream_waits);
3702 }
3703 let path = format!("/worker/workflow-tasks/{task_id}/complete");
3704 self.request_json(
3705 reqwest::Method::POST,
3706 &path,
3707 RequestProtocol::Worker(protocol_version),
3708 Some(&body),
3709 )
3710 .await
3711 }
3712
3713 pub async fn fail_workflow_task(
3714 &self,
3715 task_id: &str,
3716 lease_owner: &str,
3717 workflow_task_attempt: u64,
3718 message: impl Into<String>,
3719 ) -> Result<Value> {
3720 self.fail_workflow_task_with_type(
3721 task_id,
3722 lease_owner,
3723 workflow_task_attempt,
3724 message,
3725 "RustWorkflowTaskFailure",
3726 )
3727 .await
3728 }
3729
3730 async fn fail_workflow_task_with_type(
3731 &self,
3732 task_id: &str,
3733 lease_owner: &str,
3734 workflow_task_attempt: u64,
3735 message: impl Into<String>,
3736 failure_type: &str,
3737 ) -> Result<Value> {
3738 let body = json!({
3739 "lease_owner": lease_owner,
3740 "workflow_task_attempt": workflow_task_attempt,
3741 "failure": {
3742 "message": message.into(),
3743 "type": failure_type
3744 }
3745 });
3746 let path = format!("/worker/workflow-tasks/{task_id}/fail");
3747 self.request_json(
3748 reqwest::Method::POST,
3749 &path,
3750 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3751 Some(&body),
3752 )
3753 .await
3754 }
3755
3756 pub async fn poll_activity_task(
3757 &self,
3758 worker_id: &str,
3759 task_queue: &str,
3760 timeout: Duration,
3761 ) -> Result<Option<ActivityTask>> {
3762 Ok(self
3763 .poll_activity_task_response(worker_id, task_queue, timeout)
3764 .await?
3765 .task)
3766 }
3767
3768 pub async fn poll_activity_task_response(
3770 &self,
3771 worker_id: &str,
3772 task_queue: &str,
3773 timeout: Duration,
3774 ) -> Result<PollActivityTaskResponse> {
3775 let poll_request_id = unique_request_id("rust-activity-poll");
3776 self.poll_activity_task_response_with_request_id(
3777 worker_id,
3778 task_queue,
3779 timeout,
3780 &poll_request_id,
3781 1,
3782 )
3783 .await
3784 }
3785
3786 async fn poll_activity_task_response_with_request_id(
3787 &self,
3788 worker_id: &str,
3789 task_queue: &str,
3790 timeout: Duration,
3791 poll_request_id: &str,
3792 transport_retries: usize,
3793 ) -> Result<PollActivityTaskResponse> {
3794 let body = json!({
3795 "worker_id": worker_id,
3796 "task_queue": task_queue,
3797 "poll_request_id": poll_request_id,
3798 "timeout_seconds": long_poll_timeout_seconds(timeout),
3799 });
3800 let data: PollActivityTaskResponse = self
3801 .poll_request_json(
3802 "/worker/activity-tasks/poll",
3803 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3804 &body,
3805 timeout + Duration::from_secs(5),
3806 transport_retries,
3807 )
3808 .await?;
3809 Ok(data)
3810 }
3811
3812 pub async fn complete_activity_task<T: Serialize>(
3813 &self,
3814 task_id: &str,
3815 activity_attempt_id: &str,
3816 lease_owner: &str,
3817 result: T,
3818 codec: &str,
3819 ) -> Result<Value> {
3820 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3821 let body = json!({
3822 "activity_attempt_id": activity_attempt_id,
3823 "lease_owner": lease_owner,
3824 "result": result
3825 });
3826 let path = format!("/worker/activity-tasks/{task_id}/complete");
3827 activity_task_response(
3828 self.request_json(
3829 reqwest::Method::POST,
3830 &path,
3831 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3832 Some(&body),
3833 )
3834 .await,
3835 "complete",
3836 task_id,
3837 activity_attempt_id,
3838 )
3839 }
3840
3841 pub async fn fail_activity_task(
3842 &self,
3843 task_id: &str,
3844 activity_attempt_id: &str,
3845 lease_owner: &str,
3846 message: impl Into<String>,
3847 non_retryable: bool,
3848 ) -> Result<Value> {
3849 let body = json!({
3850 "activity_attempt_id": activity_attempt_id,
3851 "lease_owner": lease_owner,
3852 "failure": {
3853 "message": message.into(),
3854 "type": "RustActivityFailure",
3855 "non_retryable": non_retryable
3856 }
3857 });
3858 let path = format!("/worker/activity-tasks/{task_id}/fail");
3859 activity_task_response(
3860 self.request_json(
3861 reqwest::Method::POST,
3862 &path,
3863 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3864 Some(&body),
3865 )
3866 .await,
3867 "fail",
3868 task_id,
3869 activity_attempt_id,
3870 )
3871 }
3872
3873 pub async fn heartbeat_activity_task<T: Serialize>(
3874 &self,
3875 task_id: &str,
3876 activity_attempt_id: &str,
3877 lease_owner: &str,
3878 details: T,
3879 ) -> Result<ActivityHeartbeatResponse> {
3880 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3881 let body = json!({
3882 "activity_attempt_id": activity_attempt_id,
3883 "lease_owner": lease_owner,
3884 "details": details
3885 });
3886 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
3887 activity_task_response(
3888 self.request_json(
3889 reqwest::Method::POST,
3890 &path,
3891 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3892 Some(&body),
3893 )
3894 .await,
3895 "heartbeat",
3896 task_id,
3897 activity_attempt_id,
3898 )
3899 }
3900
3901 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
3902 &self,
3903 method: reqwest::Method,
3904 path: &str,
3905 protocol: RequestProtocol,
3906 body: Option<&B>,
3907 ) -> Result<T> {
3908 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
3909 .await
3910 }
3911
3912 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
3913 &self,
3914 method: reqwest::Method,
3915 path: &str,
3916 protocol: RequestProtocol,
3917 body: Option<&B>,
3918 timeout: Duration,
3919 ) -> Result<T> {
3920 let auth_token = self.auth_token(protocol)?;
3921 let mut request = self
3922 .http
3923 .request(method, format!("{}/api{}", self.base_url, path))
3924 .timeout(timeout)
3925 .header(reqwest::header::ACCEPT, "application/json")
3926 .header(reqwest::header::CONTENT_TYPE, "application/json")
3927 .header("X-Namespace", &self.namespace);
3928
3929 match protocol {
3930 RequestProtocol::Worker(version) => {
3931 request = request.header("X-Durable-Workflow-Protocol-Version", version);
3932 }
3933 RequestProtocol::ControlPlane => {
3934 request = request.header(
3935 "X-Durable-Workflow-Control-Plane-Version",
3936 CONTROL_PLANE_VERSION,
3937 );
3938 }
3939 }
3940
3941 if let Some(token) = auth_token {
3942 request = request.bearer_auth(token);
3943 }
3944
3945 if let Some(body) = body {
3946 request = request.json(body);
3947 }
3948
3949 let response = request.send().await?;
3950 let status = response.status();
3951 let bytes = response.bytes().await?;
3952
3953 if !status.is_success() {
3954 let body = String::from_utf8_lossy(&bytes).to_string();
3955 if let Some(protocol) = protocol_failure(status, &body) {
3956 return Err(Error::Protocol(protocol));
3957 }
3958 return Err(Error::Http { status, body });
3959 }
3960
3961 if bytes.is_empty() {
3962 return Ok(serde_json::from_value(Value::Null)?);
3963 }
3964
3965 Ok(serde_json::from_slice(&bytes)?)
3966 }
3967
3968 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
3969 &self,
3970 path: &str,
3971 protocol: RequestProtocol,
3972 body: &B,
3973 timeout: Duration,
3974 max_retries: usize,
3975 ) -> Result<T> {
3976 let mut retries = 0;
3977
3978 loop {
3979 let response = self
3980 .request_json_with_timeout(
3981 reqwest::Method::POST,
3982 path,
3983 protocol,
3984 Some(body),
3985 timeout,
3986 )
3987 .await;
3988
3989 match response {
3990 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
3991 response => return worker_poll_response(response),
3992 }
3993 }
3994 }
3995
3996 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
3997 match protocol {
3998 RequestProtocol::Worker(_) => {
3999 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4000 return Ok(Some(token));
4001 }
4002 if self.control_token.is_some() {
4003 return Err(Error::MissingRoleCredentials {
4004 role: "worker",
4005 opposite_role: "control",
4006 });
4007 }
4008 Ok(None)
4009 }
4010 RequestProtocol::ControlPlane => {
4011 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4012 return Ok(Some(token));
4013 }
4014 if self.worker_token.is_some() {
4015 return Err(Error::MissingRoleCredentials {
4016 role: "control",
4017 opposite_role: "worker",
4018 });
4019 }
4020 Ok(None)
4021 }
4022 }
4023 }
4024}
4025
4026fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4027 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4028 let reason = body
4029 .get("reason")
4030 .and_then(Value::as_str)
4031 .unwrap_or("query_rejected")
4032 .to_string();
4033 let message = body
4034 .get("message")
4035 .or_else(|| body.get("error"))
4036 .and_then(Value::as_str)
4037 .unwrap_or("workflow query was rejected")
4038 .to_string();
4039
4040 QueryFailure {
4041 status: status.as_u16(),
4042 reason,
4043 message,
4044 body,
4045 }
4046}
4047
4048fn workflow_command_result(
4049 command: WorkflowCommandKind,
4050 data: Value,
4051 workflow_id: &str,
4052 run_id: Option<&str>,
4053) -> WorkflowCommandResult {
4054 WorkflowCommandResult {
4055 command,
4056 workflow_id: data
4057 .get("workflow_id")
4058 .and_then(Value::as_str)
4059 .unwrap_or(workflow_id)
4060 .to_string(),
4061 run_id: data
4062 .get("run_id")
4063 .and_then(Value::as_str)
4064 .or(run_id)
4065 .map(str::to_string),
4066 outcome: data
4067 .get("outcome")
4068 .and_then(Value::as_str)
4069 .map(str::to_string),
4070 reason: data
4071 .get("reason")
4072 .and_then(Value::as_str)
4073 .map(str::to_string),
4074 command_status: data
4075 .get("command_status")
4076 .and_then(Value::as_str)
4077 .map(str::to_string),
4078 raw: data,
4079 }
4080}
4081
4082fn workflow_command_rejection(
4083 command: WorkflowCommandKind,
4084 status: reqwest::StatusCode,
4085 raw_body: String,
4086 workflow_id: &str,
4087 run_id: Option<&str>,
4088) -> WorkflowCommandRejection {
4089 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4090 WorkflowCommandRejection {
4091 command,
4092 status: status.as_u16(),
4093 reason: body
4094 .get("reason")
4095 .and_then(Value::as_str)
4096 .unwrap_or("workflow_command_rejected")
4097 .to_string(),
4098 message: body
4099 .get("message")
4100 .or_else(|| body.get("error"))
4101 .and_then(Value::as_str)
4102 .unwrap_or("workflow lifecycle command was rejected")
4103 .to_string(),
4104 workflow_id: body
4105 .get("workflow_id")
4106 .and_then(Value::as_str)
4107 .unwrap_or(workflow_id)
4108 .to_string(),
4109 run_id: body
4110 .get("run_id")
4111 .and_then(Value::as_str)
4112 .or(run_id)
4113 .map(str::to_string),
4114 target_scope: body
4115 .get("target_scope")
4116 .and_then(Value::as_str)
4117 .map(str::to_string),
4118 body,
4119 }
4120}
4121
4122fn query_task_response(response: Result<Value>) -> Result<Value> {
4123 match response {
4124 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4125 response => response,
4126 }
4127}
4128
4129fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4130 match response {
4131 Err(Error::Http { status, body })
4132 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4133 {
4134 Ok(serde_json::from_str(&body)?)
4135 }
4136 response => response,
4137 }
4138}
4139
4140fn worker_poll_body_is_stop(body: &str) -> bool {
4141 serde_json::from_str::<Value>(body)
4142 .ok()
4143 .is_some_and(|body| {
4144 worker_poll_is_stop(
4145 body.get("poll_status").and_then(Value::as_str),
4146 body.get("reason").and_then(Value::as_str),
4147 )
4148 })
4149}
4150
4151fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4152 matches!(poll_status, Some("draining" | "stopped"))
4153 || matches!(reason, Some("worker_draining" | "worker_stopped"))
4154}
4155
4156fn query_task_rejection_is_final(error: &Error) -> bool {
4157 matches!(
4158 error,
4159 Error::QueryFailed(failure)
4160 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4161 )
4162}
4163
4164fn activity_task_response<T>(
4165 response: Result<T>,
4166 operation: &str,
4167 task_id: &str,
4168 activity_attempt_id: &str,
4169) -> Result<T> {
4170 match response {
4171 Err(Error::Http { status, body }) => {
4172 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4173 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4174 operation: operation.to_string(),
4175 status: status.as_u16(),
4176 reason: body
4177 .get("reason")
4178 .and_then(Value::as_str)
4179 .unwrap_or("activity_task_rejected")
4180 .to_string(),
4181 task_id: body
4182 .get("task_id")
4183 .and_then(Value::as_str)
4184 .unwrap_or(task_id)
4185 .to_string(),
4186 activity_attempt_id: body
4187 .get("activity_attempt_id")
4188 .and_then(Value::as_str)
4189 .unwrap_or(activity_attempt_id)
4190 .to_string(),
4191 cancel_requested: body
4192 .get("cancel_requested")
4193 .and_then(Value::as_bool)
4194 .unwrap_or(false),
4195 can_continue: body.get("can_continue").and_then(Value::as_bool),
4196 run_closed_reason: body
4197 .get("run_closed_reason")
4198 .and_then(Value::as_str)
4199 .map(str::to_string),
4200 body,
4201 }))
4202 }
4203 response => response,
4204 }
4205}
4206
4207fn activity_task_rejection_is_final(error: &Error) -> bool {
4208 matches!(
4209 error,
4210 Error::ActivityTaskRejected(rejection)
4211 if matches!(
4212 rejection.reason.as_str(),
4213 "run_cancelled"
4214 | "run_terminated"
4215 | "attempt_closed"
4216 | "stale_attempt"
4217 | "activity_cancelled"
4218 | "task_cancelled"
4219 | "run_closed"
4220 | "activity_not_running"
4221 | "attempt_not_found"
4222 )
4223 )
4224}
4225
4226fn workflow_task_completion_is_terminal_timeout(
4227 error: &Error,
4228 task_id: &str,
4229 workflow_task_attempt: u64,
4230 run_id: Option<&str>,
4231) -> bool {
4232 let Error::Http { status, body } = error else {
4233 return false;
4234 };
4235 if *status != reqwest::StatusCode::CONFLICT {
4236 return false;
4237 }
4238
4239 let Some(run_id) = run_id else {
4240 return false;
4241 };
4242 let Ok(body) = serde_json::from_str::<Value>(body) else {
4243 return false;
4244 };
4245
4246 body.get("recorded").and_then(Value::as_bool) == Some(false)
4247 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4248 && body.get("run_status").and_then(Value::as_str) == Some("failed")
4249 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4250 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4251 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4252}
4253
4254fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4255 let body: Value = serde_json::from_str(raw_body).ok()?;
4256 let reason = body.get("reason")?.as_str()?;
4257 if !matches!(
4258 reason,
4259 "missing_protocol_version"
4260 | "unsupported_protocol_version"
4261 | "missing_control_plane_version"
4262 | "unsupported_control_plane_version"
4263 ) {
4264 return None;
4265 }
4266
4267 Some(ProtocolFailure {
4268 status: status.as_u16(),
4269 reason: reason.to_string(),
4270 message: body
4271 .get("message")
4272 .or_else(|| body.get("error"))
4273 .and_then(Value::as_str)
4274 .unwrap_or("protocol version rejected")
4275 .to_string(),
4276 supported_version: body
4277 .get("supported_version")
4278 .and_then(Value::as_str)
4279 .map(str::to_string),
4280 requested_version: body
4281 .get("requested_version")
4282 .and_then(Value::as_str)
4283 .map(str::to_string),
4284 body,
4285 })
4286}
4287
4288fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4289 timeout
4290 .as_secs()
4291 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4292 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4293}
4294
4295fn worker_operation_is_retryable(error: &Error) -> bool {
4296 match error {
4297 Error::Transport(error) => {
4298 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4299 }
4300 Error::Http { status, .. } => {
4301 matches!(
4302 *status,
4303 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4304 ) || status.is_server_error()
4305 }
4306 _ => false,
4307 }
4308}
4309
4310fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4311 let exponent = retry.saturating_sub(1).min(31) as u32;
4312 policy
4313 .initial_backoff
4314 .saturating_mul(1_u32 << exponent)
4315 .min(policy.max_backoff)
4316}
4317
4318#[derive(Debug)]
4319pub struct ClientBuilder {
4320 base_url: String,
4321 token: Option<String>,
4322 control_token: Option<String>,
4323 worker_token: Option<String>,
4324 namespace: String,
4325 timeout: Duration,
4326}
4327
4328impl ClientBuilder {
4329 pub fn token(mut self, token: Option<String>) -> Self {
4330 self.token = token;
4331 self
4332 }
4333
4334 pub fn control_token(mut self, token: Option<String>) -> Self {
4335 self.control_token = token;
4336 self
4337 }
4338
4339 pub fn worker_token(mut self, token: Option<String>) -> Self {
4340 self.worker_token = token;
4341 self
4342 }
4343
4344 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4345 self.namespace = namespace.into();
4346 self
4347 }
4348
4349 pub fn timeout(mut self, timeout: Duration) -> Self {
4350 self.timeout = timeout;
4351 self
4352 }
4353
4354 pub fn build(self) -> Result<Client> {
4355 let base_url = self.base_url.trim_end_matches('/').to_string();
4356 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4357 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4358 .unwrap_or_else(|_| base_url.ends_with("/api"));
4359
4360 if has_sdk_api_suffix {
4361 return Err(Error::InvalidBaseUrl);
4362 }
4363
4364 Ok(Client {
4365 http: reqwest::Client::builder().timeout(self.timeout).build()?,
4366 base_url,
4367 token: self.token,
4368 control_token: self.control_token,
4369 worker_token: self.worker_token,
4370 namespace: self.namespace,
4371 })
4372 }
4373}
4374
4375#[derive(Clone, Debug)]
4376pub struct WorkflowHandle {
4377 client: Client,
4378 pub workflow_id: String,
4379 pub run_id: Option<String>,
4380 pub workflow_type: String,
4381}
4382
4383impl WorkflowHandle {
4384 pub async fn describe(&self) -> Result<WorkflowDescription> {
4386 self.client.describe_workflow(&self.workflow_id).await
4387 }
4388
4389 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4391 let run_id = self.run_id.as_deref().ok_or_else(|| {
4392 Error::Codec("run_id is required for selected-run description".to_string())
4393 })?;
4394 self.client
4395 .describe_workflow_run(&self.workflow_id, run_id)
4396 .await
4397 }
4398
4399 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4400 self.client
4401 .signal_workflow(&self.workflow_id, signal_name, input)
4402 .await
4403 }
4404
4405 pub async fn append_message<T: Serialize>(
4406 &self,
4407 stream_name: &str,
4408 message_id: &str,
4409 input: T,
4410 ) -> Result<Value> {
4411 self.client
4412 .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4413 .await
4414 }
4415
4416 pub async fn signal_selected_run<T: Serialize>(
4418 &self,
4419 signal_name: &str,
4420 input: T,
4421 ) -> Result<Value> {
4422 let run_id = self.run_id.as_deref().ok_or_else(|| {
4423 Error::Codec("run_id is required for selected-run signaling".to_string())
4424 })?;
4425 self.client
4426 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4427 .await
4428 }
4429
4430 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4432 self.client
4433 .cancel_workflow(&self.workflow_id, options)
4434 .await
4435 }
4436
4437 pub async fn cancel_selected_run(
4439 &self,
4440 options: WorkflowCommandOptions,
4441 ) -> Result<WorkflowCommandResult> {
4442 let run_id = self.run_id.as_deref().ok_or_else(|| {
4443 Error::Codec("run_id is required for selected-run cancellation".to_string())
4444 })?;
4445 self.client
4446 .cancel_workflow_run(&self.workflow_id, run_id, options)
4447 .await
4448 }
4449
4450 pub async fn terminate(
4452 &self,
4453 options: WorkflowCommandOptions,
4454 ) -> Result<WorkflowCommandResult> {
4455 self.client
4456 .terminate_workflow(&self.workflow_id, options)
4457 .await
4458 }
4459
4460 pub async fn terminate_selected_run(
4462 &self,
4463 options: WorkflowCommandOptions,
4464 ) -> Result<WorkflowCommandResult> {
4465 let run_id = self.run_id.as_deref().ok_or_else(|| {
4466 Error::Codec("run_id is required for selected-run termination".to_string())
4467 })?;
4468 self.client
4469 .terminate_workflow_run(&self.workflow_id, run_id, options)
4470 .await
4471 }
4472
4473 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4475 self.client
4476 .query_workflow(&self.workflow_id, query_name, input)
4477 .await
4478 }
4479
4480 pub async fn query_avro_value<T: Serialize>(
4481 &self,
4482 query_name: &str,
4483 input: T,
4484 ) -> Result<AvroValue> {
4485 self.client
4486 .query_workflow_avro_value(&self.workflow_id, query_name, input)
4487 .await
4488 }
4489
4490 pub async fn update<T: Serialize>(
4491 &self,
4492 update_name: &str,
4493 input: T,
4494 request_id: Option<&str>,
4495 ) -> Result<Value> {
4496 self.client
4497 .update_workflow(&self.workflow_id, update_name, input, request_id)
4498 .await
4499 }
4500
4501 pub async fn update_avro_value<T: Serialize>(
4502 &self,
4503 update_name: &str,
4504 input: T,
4505 request_id: Option<&str>,
4506 ) -> Result<AvroValue> {
4507 self.client
4508 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4509 .await
4510 }
4511
4512 pub async fn query_selected_run<T: Serialize>(
4514 &self,
4515 query_name: &str,
4516 input: T,
4517 ) -> Result<Value> {
4518 let run_id = self
4519 .run_id
4520 .as_deref()
4521 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4522 self.client
4523 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4524 .await
4525 }
4526
4527 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4529 self.result_target(options, None).await
4530 }
4531
4532 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4534 self.result_avro_value_target(options, None).await
4535 }
4536
4537 pub async fn result_typed<T: DeserializeOwned>(
4539 &self,
4540 options: WorkflowResultOptions,
4541 ) -> Result<T> {
4542 let result = self.result_avro_value(options).await?;
4543 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4544 }
4545
4546 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4548 let run_id = self.run_id.as_deref().ok_or_else(|| {
4549 Error::Codec("run_id is required for selected-run result".to_string())
4550 })?;
4551 self.result_target(options, Some(run_id)).await
4552 }
4553
4554 pub async fn result_selected_run_avro_value(
4556 &self,
4557 options: WorkflowResultOptions,
4558 ) -> Result<AvroValue> {
4559 let run_id = self.run_id.as_deref().ok_or_else(|| {
4560 Error::Codec("run_id is required for selected-run result".to_string())
4561 })?;
4562 self.result_avro_value_target(options, Some(run_id)).await
4563 }
4564
4565 pub async fn result_selected_run_typed<T: DeserializeOwned>(
4567 &self,
4568 options: WorkflowResultOptions,
4569 ) -> Result<T> {
4570 let result = self.result_selected_run_avro_value(options).await?;
4571 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4572 }
4573
4574 async fn result_avro_value_target(
4575 &self,
4576 options: WorkflowResultOptions,
4577 selected_run_id: Option<&str>,
4578 ) -> Result<AvroValue> {
4579 let started = Instant::now();
4580
4581 loop {
4582 let description = match selected_run_id {
4583 Some(run_id) => {
4584 self.client
4585 .describe_workflow_run(&self.workflow_id, run_id)
4586 .await?
4587 }
4588 None => self.describe().await?,
4589 };
4590 if description.is_completed() {
4591 return description.output_avro_value.ok_or_else(|| {
4592 Error::Codec(
4593 "missing_payload_envelope: typed workflow result requires output_envelope"
4594 .to_string(),
4595 )
4596 });
4597 }
4598 if description.is_terminal() {
4599 let outcome =
4600 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4601 return Err(match outcome.kind {
4602 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4603 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4604 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4605 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4606 });
4607 }
4608 if started.elapsed() >= options.timeout {
4609 return Err(Error::Timeout);
4610 }
4611 tokio::time::sleep(options.poll_interval).await;
4612 }
4613 }
4614
4615 async fn result_target(
4616 &self,
4617 options: WorkflowResultOptions,
4618 selected_run_id: Option<&str>,
4619 ) -> Result<Value> {
4620 let started = Instant::now();
4621
4622 loop {
4623 let description = match selected_run_id {
4624 Some(run_id) => {
4625 self.client
4626 .describe_workflow_run(&self.workflow_id, run_id)
4627 .await?
4628 }
4629 None => self.describe().await?,
4630 };
4631 if description.is_completed() {
4632 return Ok(description.output.unwrap_or(Value::Null));
4633 }
4634
4635 if description.is_terminal() {
4636 let outcome =
4637 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4638 return Err(match outcome.kind {
4639 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4640 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4641 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4642 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4643 });
4644 }
4645
4646 if started.elapsed() >= options.timeout {
4647 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4648 kind: WorkflowTerminalKind::TimedOut,
4649 workflow_id: description
4650 .workflow_id
4651 .clone()
4652 .unwrap_or_else(|| self.workflow_id.clone()),
4653 run_id: description
4654 .run_id
4655 .clone()
4656 .or_else(|| selected_run_id.map(str::to_string)),
4657 reason: "result_wait_timeout".to_string(),
4658 failure_category: Some("client_timeout".to_string()),
4659 failure_id: None,
4660 exception_type: None,
4661 exception_class: None,
4662 non_retryable: None,
4663 message: Some(format!(
4664 "workflow result was not terminal within {:?}",
4665 options.timeout
4666 )),
4667 exception: None,
4668 raw: description.raw_value(),
4669 }));
4670 }
4671
4672 tokio::time::sleep(options.poll_interval).await;
4673 }
4674 }
4675}
4676
4677#[derive(Clone, Copy, Debug)]
4678pub struct WorkflowResultOptions {
4679 pub poll_interval: Duration,
4680 pub timeout: Duration,
4681}
4682
4683impl Default for WorkflowResultOptions {
4684 fn default() -> Self {
4685 Self {
4686 poll_interval: Duration::from_millis(500),
4687 timeout: Duration::from_secs(30),
4688 }
4689 }
4690}
4691
4692#[derive(Clone, Debug, Deserialize)]
4693pub struct WorkflowDescription {
4694 pub workflow_id: Option<String>,
4695 pub run_id: Option<String>,
4696 pub workflow_type: Option<String>,
4697 pub status: Option<String>,
4698 #[serde(default)]
4699 pub closed_reason: Option<String>,
4700 #[serde(default)]
4701 pub error: Option<String>,
4702 #[serde(default)]
4703 pub failure: Option<Value>,
4704 #[serde(default)]
4705 pub exception: Option<Value>,
4706 #[serde(default)]
4707 pub failures: Vec<Value>,
4708 #[serde(default)]
4709 pub output: Option<Value>,
4710 #[serde(default)]
4711 pub output_envelope: Option<Value>,
4712 #[serde(skip)]
4713 pub output_avro_value: Option<AvroValue>,
4714 #[serde(flatten)]
4715 pub raw: HashMap<String, Value>,
4716}
4717
4718#[derive(Clone, Debug, Deserialize)]
4720pub struct WorkflowStreamDescription {
4721 pub stream_name: String,
4722 pub status: String,
4723 pub last_offset: i64,
4724 pub total_items: u64,
4725 pub pending_items: u64,
4726 #[serde(default)]
4727 pub opened_at: Option<String>,
4728 #[serde(default)]
4729 pub last_appended_at: Option<String>,
4730 #[serde(default)]
4731 pub closed_at: Option<String>,
4732 #[serde(default)]
4733 pub error_reason: Option<String>,
4734 #[serde(default)]
4735 pub retention_seconds: Option<u64>,
4736 #[serde(flatten)]
4737 pub raw: HashMap<String, Value>,
4738}
4739
4740impl WorkflowStreamDescription {
4741 pub fn is_terminal(&self) -> bool {
4742 matches!(self.status.as_str(), "closed" | "errored")
4743 }
4744}
4745
4746#[derive(Clone, Debug, Default)]
4748pub struct WorkflowStreamAppendItem {
4749 pub payload_envelope: Option<Value>,
4750 pub payload_reference: Option<String>,
4751 pub item_type: Option<String>,
4752 pub content_type: Option<String>,
4753 pub idempotency_key: Option<String>,
4754}
4755
4756impl WorkflowStreamAppendItem {
4757 pub fn new<T: Serialize>(payload: T) -> Result<Self> {
4759 let value = AvroValue::from_serialize(&payload)?;
4760 Ok(Self {
4761 payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
4762 ..Self::default()
4763 })
4764 }
4765
4766 pub fn from_reference(reference: impl Into<String>) -> Self {
4768 Self {
4769 payload_reference: Some(reference.into()),
4770 ..Self::default()
4771 }
4772 }
4773
4774 pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
4775 self.item_type = Some(item_type.into());
4776 self
4777 }
4778
4779 pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
4780 self.content_type = Some(content_type.into());
4781 self
4782 }
4783
4784 pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
4785 self.idempotency_key = Some(idempotency_key.into());
4786 self
4787 }
4788
4789 fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
4790 let mut item = serde_json::Map::new();
4791 if let Some(payload) = &self.payload_envelope {
4792 item.insert("payload".to_string(), payload.clone());
4793 item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
4794 }
4795 if let Some(reference) = &self.payload_reference {
4796 item.insert("payload_reference".to_string(), json!(reference));
4797 }
4798 if let Some(item_type) = &self.item_type {
4799 item.insert("item_type".to_string(), json!(item_type));
4800 }
4801 if let Some(content_type) = &self.content_type {
4802 item.insert("content_type".to_string(), json!(content_type));
4803 }
4804 if let Some(key) = derived_idempotency_key
4805 .as_ref()
4806 .or(self.idempotency_key.as_ref())
4807 {
4808 item.insert("idempotency_key".to_string(), json!(key));
4809 }
4810 Value::Object(item)
4811 }
4812}
4813
4814#[derive(Clone, Debug)]
4816pub struct WorkflowStreamItem {
4817 pub offset: u64,
4818 pub payload: Option<Value>,
4819 pub payload_envelope: Option<Value>,
4820 pub payload_reference: Option<String>,
4821 pub payload_codec: Option<String>,
4822 pub idempotency_key: Option<String>,
4823 pub item_type: Option<String>,
4824 pub content_type: Option<String>,
4825 pub origin: Option<String>,
4826 pub origin_reference: Option<String>,
4827 pub emitted_at: Option<String>,
4828 pub raw: Value,
4829}
4830
4831#[derive(Clone, Debug)]
4833pub struct WorkflowStreamPage {
4834 pub stream: WorkflowStreamDescription,
4835 pub items: Vec<WorkflowStreamItem>,
4836 pub next_offset: u64,
4837 pub terminal: bool,
4838}
4839
4840#[derive(Clone, Debug)]
4842pub struct WorkflowStreamAppendResult {
4843 pub stream: WorkflowStreamDescription,
4844 pub accepted_offsets: Vec<u64>,
4845 pub accepted: u64,
4846 pub deduped: u64,
4847}
4848
4849#[derive(Deserialize)]
4850struct WorkflowStreamListResponse {
4851 #[serde(default)]
4852 streams: Vec<WorkflowStreamDescription>,
4853}
4854
4855#[derive(Deserialize)]
4856struct WorkflowStreamDescriptionResponse {
4857 stream: WorkflowStreamDescription,
4858}
4859
4860#[derive(Deserialize)]
4861struct WorkflowStreamPageResponse {
4862 stream: WorkflowStreamDescription,
4863 #[serde(default)]
4864 items: Vec<Value>,
4865 next_offset: u64,
4866 terminal: bool,
4867}
4868
4869#[derive(Deserialize)]
4870struct WorkflowStreamAppendResponse {
4871 stream: WorkflowStreamDescription,
4872 #[serde(default)]
4873 accepted_offsets: Vec<u64>,
4874 accepted: u64,
4875 deduped: u64,
4876}
4877
4878impl WorkflowDescription {
4879 pub fn is_completed(&self) -> bool {
4880 matches!(self.status.as_deref(), Some("completed" | "Completed"))
4881 }
4882
4883 pub fn is_terminal(&self) -> bool {
4884 matches!(
4885 self.status.as_deref(),
4886 Some(
4887 "completed"
4888 | "Completed"
4889 | "failed"
4890 | "Failed"
4891 | "cancelled"
4892 | "Cancelled"
4893 | "terminated"
4894 | "Terminated"
4895 | "timed_out"
4896 | "TimedOut",
4897 )
4898 )
4899 }
4900
4901 fn decode_payloads(&mut self) -> Result<()> {
4902 if let Some(envelope) = &self.output_envelope {
4903 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
4904 self.output = Some(value.clone().into_json()?);
4905 self.output_avro_value = Some(value);
4906 }
4907
4908 Ok(())
4909 }
4910
4911 fn raw_value(&self) -> Value {
4912 let mut data = self.raw.clone();
4913 data.insert(
4914 "workflow_id".to_string(),
4915 self.workflow_id
4916 .clone()
4917 .map(Value::String)
4918 .unwrap_or(Value::Null),
4919 );
4920 data.insert(
4921 "run_id".to_string(),
4922 self.run_id
4923 .clone()
4924 .map(Value::String)
4925 .unwrap_or(Value::Null),
4926 );
4927 data.insert(
4928 "workflow_type".to_string(),
4929 self.workflow_type
4930 .clone()
4931 .map(Value::String)
4932 .unwrap_or(Value::Null),
4933 );
4934 data.insert(
4935 "status".to_string(),
4936 self.status
4937 .clone()
4938 .map(Value::String)
4939 .unwrap_or(Value::Null),
4940 );
4941 data.insert(
4942 "closed_reason".to_string(),
4943 self.closed_reason
4944 .clone()
4945 .map(Value::String)
4946 .unwrap_or(Value::Null),
4947 );
4948 if let Some(failure) = &self.failure {
4949 data.insert("failure".to_string(), failure.clone());
4950 }
4951 if let Some(exception) = &self.exception {
4952 data.insert("exception".to_string(), exception.clone());
4953 }
4954 Value::Object(data.into_iter().collect())
4955 }
4956}
4957
4958fn workflow_terminal_outcome(
4959 description: &WorkflowDescription,
4960 workflow_id: &str,
4961 run_id: Option<&str>,
4962) -> WorkflowTerminalOutcome {
4963 let terminal_kind = description
4964 .closed_reason
4965 .as_deref()
4966 .or(description.status.as_deref())
4967 .unwrap_or("failed")
4968 .to_ascii_lowercase();
4969 let kind = match terminal_kind.as_str() {
4970 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
4971 "terminated" => WorkflowTerminalKind::Terminated,
4972 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
4973 _ => WorkflowTerminalKind::Failed,
4974 };
4975 let default_reason = match kind {
4976 WorkflowTerminalKind::Failed => "workflow_failed",
4977 WorkflowTerminalKind::Cancelled => "cancelled",
4978 WorkflowTerminalKind::Terminated => "terminated",
4979 WorkflowTerminalKind::TimedOut => "timed_out",
4980 };
4981 let failure = description
4982 .failure
4983 .as_ref()
4984 .filter(|value| value.is_object());
4985 let nested_failure = failure
4986 .and_then(|value| value.get("failures"))
4987 .and_then(Value::as_array)
4988 .and_then(|failures| failures.last())
4989 .or_else(|| description.failures.last());
4990 let exception = description
4991 .exception
4992 .clone()
4993 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
4994 .or_else(|| {
4995 nested_failure
4996 .and_then(|value| value.get("exception_payload"))
4997 .cloned()
4998 });
4999 let string_field = |name: &str| {
5000 failure
5001 .and_then(|value| value.get(name))
5002 .and_then(Value::as_str)
5003 .or_else(|| {
5004 nested_failure
5005 .and_then(|value| value.get(name))
5006 .and_then(Value::as_str)
5007 })
5008 .map(str::to_string)
5009 };
5010 let exception_field = |name: &str| {
5011 exception
5012 .as_ref()
5013 .and_then(|value| value.get(name))
5014 .and_then(Value::as_str)
5015 .map(str::to_string)
5016 };
5017 let message = description
5018 .error
5019 .clone()
5020 .or_else(|| string_field("message"))
5021 .or_else(|| exception_field("message"));
5022 let reason = description
5023 .raw
5024 .get("reason")
5025 .and_then(Value::as_str)
5026 .map(str::to_string)
5027 .or_else(|| {
5028 failure
5029 .and_then(|value| value.get("reason"))
5030 .and_then(Value::as_str)
5031 .map(str::to_string)
5032 })
5033 .or_else(|| description.closed_reason.clone())
5034 .unwrap_or_else(|| default_reason.to_string());
5035 let failure_id = string_field("failure_id").or_else(|| {
5036 nested_failure
5037 .and_then(|value| value.get("id"))
5038 .and_then(Value::as_str)
5039 .map(str::to_string)
5040 });
5041
5042 WorkflowTerminalOutcome {
5043 kind,
5044 workflow_id: description
5045 .workflow_id
5046 .clone()
5047 .unwrap_or_else(|| workflow_id.to_string()),
5048 run_id: description
5049 .run_id
5050 .clone()
5051 .or_else(|| run_id.map(str::to_string)),
5052 reason,
5053 failure_category: string_field("failure_category")
5054 .or_else(|| Some(default_reason.to_string())),
5055 failure_id,
5056 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5057 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5058 non_retryable: failure
5059 .and_then(|value| value.get("non_retryable"))
5060 .and_then(Value::as_bool)
5061 .or_else(|| {
5062 nested_failure
5063 .and_then(|value| value.get("non_retryable"))
5064 .and_then(Value::as_bool)
5065 }),
5066 message,
5067 exception,
5068 raw: description.raw_value(),
5069 }
5070}
5071
5072#[derive(Clone, Debug, Deserialize)]
5073pub struct RegisterWorkerResponse {
5074 pub worker_id: String,
5075 pub registered: bool,
5076 #[serde(default)]
5077 pub heartbeat_interval_seconds: Option<u64>,
5078 #[serde(default)]
5079 pub protocol_version: Option<String>,
5080 #[serde(default)]
5081 pub server_capabilities: Option<Value>,
5082}
5083
5084#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5086pub struct WorkerDeregistrationEnvelope {
5087 pub worker_id: String,
5088 pub outcome: String,
5089 pub recovered_workflow_task_count: u64,
5090}
5091
5092#[derive(Clone, Debug, Deserialize)]
5093pub struct PollWorkflowTaskResponse {
5094 #[serde(default)]
5095 pub task: Option<WorkflowTask>,
5096 #[serde(default)]
5097 pub poll_status: Option<String>,
5098 #[serde(default)]
5099 pub reason: Option<String>,
5100 #[serde(default)]
5101 pub protocol_version: Option<String>,
5102 #[serde(default)]
5103 pub server_capabilities: Option<Value>,
5104}
5105
5106impl PollWorkflowTaskResponse {
5107 pub fn outcome(&self) -> WorkerPollOutcome {
5109 worker_poll_outcome(
5110 self.task.is_some(),
5111 self.poll_status.as_deref(),
5112 self.reason.as_deref(),
5113 )
5114 }
5115}
5116
5117fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5118 let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5119 return false;
5120 };
5121 let supported = capabilities
5122 .get("workflow_memo_updates")
5123 .and_then(Value::as_object)
5124 .and_then(|memo| memo.get("supported"))
5125 .and_then(Value::as_bool)
5126 == Some(true);
5127 let command_advertised = capabilities
5128 .get("supported_workflow_task_commands")
5129 .and_then(Value::as_array)
5130 .is_some_and(|commands| {
5131 commands
5132 .iter()
5133 .any(|command| command.as_str() == Some("upsert_memo"))
5134 });
5135 supported && command_advertised
5136}
5137
5138fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5139 commands
5140 .iter()
5141 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5142}
5143
5144#[derive(Clone, Debug, Deserialize)]
5145pub struct PollActivityTaskResponse {
5146 #[serde(default)]
5147 pub task: Option<ActivityTask>,
5148 #[serde(default)]
5149 pub poll_status: Option<String>,
5150 #[serde(default)]
5151 pub reason: Option<String>,
5152}
5153
5154impl PollActivityTaskResponse {
5155 pub fn outcome(&self) -> WorkerPollOutcome {
5157 worker_poll_outcome(
5158 self.task.is_some(),
5159 self.poll_status.as_deref(),
5160 self.reason.as_deref(),
5161 )
5162 }
5163}
5164
5165#[derive(Clone, Debug, Deserialize)]
5166pub struct PollQueryTaskResponse {
5167 #[serde(default)]
5168 pub task: Option<QueryTask>,
5169 #[serde(default)]
5170 pub poll_status: Option<String>,
5171 #[serde(default)]
5172 pub reason: Option<String>,
5173}
5174
5175impl PollQueryTaskResponse {
5176 pub fn outcome(&self) -> WorkerPollOutcome {
5178 worker_poll_outcome(
5179 self.task.is_some(),
5180 self.poll_status.as_deref(),
5181 self.reason.as_deref(),
5182 )
5183 }
5184}
5185
5186#[derive(Clone, Debug, PartialEq, Eq)]
5188pub enum WorkerPollOutcome {
5189 Task,
5191 Idle {
5193 poll_status: Option<String>,
5194 reason: Option<String>,
5195 },
5196 Stop {
5198 poll_status: Option<String>,
5199 reason: Option<String>,
5200 },
5201}
5202
5203impl WorkerPollOutcome {
5204 pub fn should_stop(&self) -> bool {
5205 matches!(self, Self::Stop { .. })
5206 }
5207}
5208
5209fn worker_poll_outcome(
5210 has_task: bool,
5211 poll_status: Option<&str>,
5212 reason: Option<&str>,
5213) -> WorkerPollOutcome {
5214 if worker_poll_is_stop(poll_status, reason) {
5215 return WorkerPollOutcome::Stop {
5216 poll_status: poll_status.map(str::to_string),
5217 reason: reason.map(str::to_string),
5218 };
5219 }
5220
5221 if has_task {
5222 WorkerPollOutcome::Task
5223 } else {
5224 WorkerPollOutcome::Idle {
5225 poll_status: poll_status.map(str::to_string),
5226 reason: reason.map(str::to_string),
5227 }
5228 }
5229}
5230
5231#[derive(Clone, Debug, Deserialize)]
5233pub struct QueryTask {
5234 pub query_task_id: String,
5235 #[serde(default = "default_workflow_task_attempt")]
5236 pub query_task_attempt: u64,
5237 #[serde(default)]
5238 pub lease_owner: Option<String>,
5239 #[serde(default)]
5240 pub workflow_id: Option<String>,
5241 #[serde(default)]
5242 pub run_id: Option<String>,
5243 pub workflow_type: String,
5244 pub query_name: String,
5245 #[serde(
5246 default = "missing_task_payload_codec",
5247 deserialize_with = "deserialize_task_payload_codec"
5248 )]
5249 pub payload_codec: String,
5250 #[serde(default)]
5251 pub workflow_arguments: Option<Value>,
5252 #[serde(default)]
5253 pub query_arguments: Option<Value>,
5254 #[serde(default)]
5255 pub history_events: Vec<HistoryEvent>,
5256 #[serde(default)]
5257 pub history_export: Option<Value>,
5258 #[serde(default)]
5259 pub run_status: Option<String>,
5260}
5261
5262#[derive(Clone, Debug, Deserialize)]
5263pub struct WorkflowTask {
5264 pub task_id: String,
5265 #[serde(default)]
5266 pub workflow_command_id: Option<String>,
5267 #[serde(default)]
5268 pub workflow_id: Option<String>,
5269 #[serde(default)]
5270 pub run_id: Option<String>,
5271 pub workflow_type: String,
5272 #[serde(default)]
5273 pub cancel_requested: bool,
5274 #[serde(
5275 default = "missing_task_payload_codec",
5276 deserialize_with = "deserialize_task_payload_codec"
5277 )]
5278 pub payload_codec: String,
5279 #[serde(default)]
5280 pub arguments: Option<Value>,
5281 #[serde(default)]
5282 pub history_events: Vec<HistoryEvent>,
5283 #[serde(default)]
5284 pub total_history_events: Option<u64>,
5285 #[serde(default)]
5286 pub history_size_bytes: Option<u64>,
5287 #[serde(default)]
5288 pub continue_as_new_recommended: Option<bool>,
5289 #[serde(default)]
5290 pub history_budget_pressure: Option<String>,
5291 #[serde(default)]
5292 pub next_history_page_token: Option<String>,
5293 #[serde(default = "default_workflow_task_attempt")]
5294 pub workflow_task_attempt: u64,
5295 #[serde(default)]
5296 pub workflow_signal_id: Option<String>,
5297 #[serde(default)]
5298 pub signal_name: Option<String>,
5299 #[serde(default)]
5300 pub signal_arguments: Option<Value>,
5301 #[serde(default)]
5302 pub workflow_update_id: Option<String>,
5303 #[serde(default)]
5304 pub update_name: Option<String>,
5305 #[serde(default)]
5306 pub lease_owner: Option<String>,
5307}
5308
5309impl WorkflowTask {
5310 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5311 self.history_events.extend(page.history_events);
5312
5313 if page.total_history_events.is_some() {
5314 self.total_history_events = page.total_history_events;
5315 }
5316
5317 self.next_history_page_token = page
5318 .next_history_page_token
5319 .filter(|token| !token.is_empty());
5320 }
5321}
5322
5323#[derive(Clone, Debug, Deserialize)]
5324struct WorkflowTaskHistoryPage {
5325 #[serde(default)]
5326 history_events: Vec<HistoryEvent>,
5327 #[serde(default)]
5328 total_history_events: Option<u64>,
5329 #[serde(default)]
5330 next_history_page_token: Option<String>,
5331}
5332
5333#[derive(Clone, Debug, Deserialize)]
5334pub struct ActivityTask {
5335 pub task_id: String,
5336 #[serde(default)]
5337 pub activity_attempt_id: Option<String>,
5338 #[serde(default)]
5339 pub attempt_id: Option<String>,
5340 pub activity_type: String,
5341 #[serde(
5342 default = "missing_task_payload_codec",
5343 deserialize_with = "deserialize_task_payload_codec"
5344 )]
5345 pub payload_codec: String,
5346 #[serde(default)]
5347 pub arguments: Option<Value>,
5348 #[serde(default = "default_attempt_number")]
5349 pub attempt_number: u64,
5350 #[serde(default)]
5351 pub lease_owner: Option<String>,
5352}
5353
5354#[derive(Clone, Debug, Deserialize)]
5355pub struct HistoryEvent {
5356 #[serde(alias = "type")]
5357 pub event_type: String,
5358 #[serde(default)]
5359 pub payload: Value,
5360 #[serde(flatten)]
5361 pub raw: HashMap<String, Value>,
5362}
5363
5364#[derive(Clone, Debug, PartialEq)]
5366pub struct QuerySignal {
5367 pub id: Option<String>,
5368 pub name: String,
5369 pub arguments: Vec<Value>,
5370 avro_arguments: Vec<AvroValue>,
5371 pub workflow_sequence: Option<u64>,
5372}
5373
5374impl QuerySignal {
5375 pub fn arguments_avro_value(&self) -> &[AvroValue] {
5377 &self.avro_arguments
5378 }
5379}
5380
5381#[derive(Clone, Debug)]
5387pub struct QueryContext {
5388 pub workflow_id: Option<String>,
5389 pub run_id: Option<String>,
5390 pub workflow_type: String,
5391 pub run_status: Option<String>,
5392 workflow_input: Value,
5393 workflow_input_avro_value: AvroValue,
5394 history_events: Arc<Vec<HistoryEvent>>,
5395 signal_events: Arc<Vec<QuerySignal>>,
5396}
5397
5398impl QueryContext {
5399 pub fn workflow_input(&self) -> &Value {
5401 &self.workflow_input
5402 }
5403
5404 pub fn workflow_input_avro_value(&self) -> &AvroValue {
5406 &self.workflow_input_avro_value
5407 }
5408
5409 pub fn history_events(&self) -> &[HistoryEvent] {
5411 self.history_events.as_slice()
5412 }
5413
5414 pub fn signal_events(&self) -> &[QuerySignal] {
5416 self.signal_events.as_slice()
5417 }
5418
5419 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5421 self.signal_events
5422 .iter()
5423 .filter(|signal| signal.name == signal_name)
5424 .map(|signal| signal.arguments.clone())
5425 .collect()
5426 }
5427
5428 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5430 self.signal_events
5431 .iter()
5432 .filter(|signal| signal.name == signal_name)
5433 .map(|signal| signal.avro_arguments.clone())
5434 .collect()
5435 }
5436}
5437
5438#[derive(Clone, Debug, Deserialize)]
5439pub struct ActivityHeartbeatResponse {
5440 #[serde(default)]
5441 pub cancel_requested: bool,
5442 #[serde(default)]
5443 pub heartbeat_recorded: bool,
5444 #[serde(default)]
5445 pub can_continue: Option<bool>,
5446 #[serde(default)]
5447 pub reason: Option<String>,
5448 #[serde(default)]
5449 pub run_closed_reason: Option<String>,
5450 #[serde(default)]
5451 pub run_closed_at: Option<String>,
5452 #[serde(default)]
5453 pub lease_expires_at: Option<String>,
5454 #[serde(default)]
5455 pub last_heartbeat_at: Option<String>,
5456}
5457
5458impl ActivityHeartbeatResponse {
5459 pub fn should_stop(&self) -> bool {
5461 self.cancel_requested || self.can_continue == Some(false)
5462 }
5463}
5464
5465fn missing_task_payload_codec() -> String {
5466 MISSING_TASK_PAYLOAD_CODEC.to_string()
5467}
5468
5469fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5470where
5471 D: Deserializer<'de>,
5472{
5473 Ok(match Value::deserialize(deserializer)? {
5474 Value::String(codec) => codec,
5475 Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5476 _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5477 })
5478}
5479
5480fn default_workflow_task_attempt() -> u64 {
5481 1
5482}
5483
5484fn default_attempt_number() -> u64 {
5485 1
5486}
5487
5488type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5489type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5490type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5491type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5492type ReplayedWorkflowHandler =
5493 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5494type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5495type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5496type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5497type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5498type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5499type ReplayedQueryHandler = Arc<
5500 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5501 + Send
5502 + Sync,
5503>;
5504type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5505
5506struct ReplayedWorkflowInvocation {
5507 future: WorkflowFuture,
5508 snapshot: WorkflowStateSnapshot,
5509}
5510
5511#[derive(Clone)]
5512struct RegisteredWorkflow {
5513 execute: WorkflowHandler,
5514 replay: Option<ReplayedWorkflowHandler>,
5515 state_type: Option<TypeId>,
5516}
5517
5518#[derive(Debug)]
5519struct WorkflowTaskDecision {
5520 commands: Vec<Value>,
5521 message_stream_cursors: Vec<Value>,
5522 message_stream_waits: Vec<Value>,
5523}
5524
5525impl WorkflowTaskDecision {
5526 fn without_message_streams(commands: Vec<Value>) -> Self {
5527 Self {
5528 commands,
5529 message_stream_cursors: Vec::new(),
5530 message_stream_waits: Vec::new(),
5531 }
5532 }
5533}
5534
5535#[derive(Clone)]
5536enum RegisteredQuery {
5537 Snapshot(QueryHandler),
5538 Replayed {
5539 state_type: TypeId,
5540 handler: ReplayedQueryHandler,
5541 },
5542}
5543
5544#[derive(Clone, Debug)]
5545pub struct WorkerHeartbeatObservation {
5546 pub worker_id: String,
5547 pub task_queue: String,
5548 pub acknowledged_at_unix_millis: u64,
5549 pub acknowledgement: Value,
5550}
5551
5552#[derive(Clone, Copy, Debug)]
5559pub struct WorkerRetryPolicy {
5560 pub max_retries: usize,
5562 pub initial_backoff: Duration,
5564 pub max_backoff: Duration,
5566}
5567
5568impl Default for WorkerRetryPolicy {
5569 fn default() -> Self {
5570 Self {
5571 max_retries: 5,
5572 initial_backoff: Duration::from_millis(100),
5573 max_backoff: Duration::from_secs(5),
5574 }
5575 }
5576}
5577
5578#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5579enum ManagedPollOutcome {
5580 Idle,
5581 Handled,
5582 Stop,
5583}
5584
5585#[derive(Clone)]
5586pub struct Worker {
5587 client: Client,
5588 worker_id: String,
5589 task_queue: String,
5590 workflows: HashMap<String, RegisteredWorkflow>,
5591 activities: HashMap<String, ActivityHandler>,
5592 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5593 updates: HashMap<String, HashMap<String, UpdateHandler>>,
5594 max_concurrent_workflow_tasks: usize,
5595 max_concurrent_activity_tasks: usize,
5596 poll_timeout: Duration,
5597 heartbeat_interval: Duration,
5598 retry_policy: WorkerRetryPolicy,
5599 heartbeat_observer: Option<WorkerHeartbeatObserver>,
5600}
5601
5602impl Worker {
5603 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5604 Self {
5605 client,
5606 worker_id: default_worker_id(),
5607 task_queue: task_queue.into(),
5608 workflows: HashMap::new(),
5609 activities: HashMap::new(),
5610 queries: HashMap::new(),
5611 updates: HashMap::new(),
5612 max_concurrent_workflow_tasks: 10,
5613 max_concurrent_activity_tasks: 10,
5614 poll_timeout: Duration::from_secs(30),
5615 heartbeat_interval: Duration::from_secs(60),
5616 retry_policy: WorkerRetryPolicy::default(),
5617 heartbeat_observer: None,
5618 }
5619 }
5620
5621 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5622 self.worker_id = worker_id.into();
5623 self
5624 }
5625
5626 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5627 self.poll_timeout = timeout;
5628 self
5629 }
5630
5631 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5632 self.heartbeat_interval = interval;
5633 self
5634 }
5635
5636 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5638 self.retry_policy = policy;
5639 self
5640 }
5641
5642 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5643 where
5644 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5645 {
5646 self.heartbeat_observer = Some(Arc::new(observer));
5647 self
5648 }
5649
5650 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5651 self.max_concurrent_workflow_tasks = count.max(1);
5652 self
5653 }
5654
5655 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5656 self.max_concurrent_activity_tasks = count.max(1);
5657 self
5658 }
5659
5660 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5667 where
5668 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5669 Fut: Future<Output = Result<Value>> + Send + 'static,
5670 {
5671 let handler = Arc::new(handler);
5672 self.workflows.insert(
5673 workflow_type.into(),
5674 RegisteredWorkflow {
5675 execute: Arc::new(move |ctx, input| {
5676 let handler = Arc::clone(&handler);
5677 Box::pin(async move {
5678 let result = handler(ctx, input.into_json()?).await?;
5679 AvroValue::from_serialize(&result)
5680 })
5681 }),
5682 replay: None,
5683 state_type: None,
5684 },
5685 );
5686 }
5687
5688 pub fn register_typed_workflow<I, O, F, Fut>(
5699 &mut self,
5700 workflow_type: impl Into<String>,
5701 handler: F,
5702 ) where
5703 I: DeserializeOwned + Send + 'static,
5704 O: Serialize + Send + 'static,
5705 F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
5706 Fut: Future<Output = Result<O>> + Send + 'static,
5707 {
5708 let workflow_type = workflow_type.into();
5709 let handler_name = workflow_type.clone();
5710 let handler = Arc::new(handler);
5711 self.workflows.insert(
5712 workflow_type,
5713 RegisteredWorkflow {
5714 execute: Arc::new(move |ctx, input| {
5715 let handler = Arc::clone(&handler);
5716 let handler_name = handler_name.clone();
5717 Box::pin(async move {
5718 let input =
5719 decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5720 let result = handler(ctx, input).await?;
5721 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5722 })
5723 }),
5724 replay: None,
5725 state_type: None,
5726 },
5727 );
5728 }
5729
5730 pub fn register_workflow_avro_value<F, Fut>(
5732 &mut self,
5733 workflow_type: impl Into<String>,
5734 handler: F,
5735 ) where
5736 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
5737 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5738 {
5739 self.workflows.insert(
5740 workflow_type.into(),
5741 RegisteredWorkflow {
5742 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
5743 replay: None,
5744 state_type: None,
5745 },
5746 );
5747 }
5748
5749 pub fn register_replayed_workflow<S, Factory, F, Fut>(
5757 &mut self,
5758 workflow_type: impl Into<String>,
5759 state_factory: Factory,
5760 handler: F,
5761 ) where
5762 S: Clone + Send + Sync + 'static,
5763 Factory: Fn() -> S + Send + Sync + 'static,
5764 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5765 Fut: Future<Output = Result<Value>> + Send + 'static,
5766 {
5767 let state_factory = Arc::new(state_factory);
5768 let handler = Arc::new(handler);
5769
5770 let execute_factory = Arc::clone(&state_factory);
5771 let execute_handler = Arc::clone(&handler);
5772 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5773 let state = WorkflowInstance::new(execute_factory());
5774 let handler = Arc::clone(&execute_handler);
5775 Box::pin(async move {
5776 let result = handler(ctx, input.into_json()?, state).await?;
5777 AvroValue::from_serialize(&result)
5778 }) as WorkflowFuture
5779 });
5780
5781 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5782 let state = WorkflowInstance::new(state_factory());
5783 let snapshot_state = state.clone();
5784 let snapshot: WorkflowStateSnapshot =
5785 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5786 let replay_handler = Arc::clone(&handler);
5787 let future = async move {
5788 let result = replay_handler(ctx, input.into_json()?, state).await?;
5789 AvroValue::from_serialize(&result)
5790 };
5791 ReplayedWorkflowInvocation {
5792 future: Box::pin(future),
5793 snapshot,
5794 }
5795 });
5796
5797 self.workflows.insert(
5798 workflow_type.into(),
5799 RegisteredWorkflow {
5800 execute,
5801 replay: Some(replay),
5802 state_type: Some(TypeId::of::<S>()),
5803 },
5804 );
5805 }
5806
5807 pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
5813 &mut self,
5814 workflow_type: impl Into<String>,
5815 state_factory: Factory,
5816 handler: F,
5817 ) where
5818 I: DeserializeOwned + Send + 'static,
5819 O: Serialize + Send + 'static,
5820 S: Clone + Send + Sync + 'static,
5821 Factory: Fn() -> S + Send + Sync + 'static,
5822 F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5823 Fut: Future<Output = Result<O>> + Send + 'static,
5824 {
5825 let workflow_type = workflow_type.into();
5826 let state_factory = Arc::new(state_factory);
5827 let handler = Arc::new(handler);
5828
5829 let execute_name = workflow_type.clone();
5830 let execute_factory = Arc::clone(&state_factory);
5831 let execute_handler = Arc::clone(&handler);
5832 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5833 let state = WorkflowInstance::new(execute_factory());
5834 let handler = Arc::clone(&execute_handler);
5835 let handler_name = execute_name.clone();
5836 Box::pin(async move {
5837 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5838 let result = handler(ctx, input, state).await?;
5839 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5840 }) as WorkflowFuture
5841 });
5842
5843 let replay_name = workflow_type.clone();
5844 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5845 let state = WorkflowInstance::new(state_factory());
5846 let snapshot_state = state.clone();
5847 let snapshot: WorkflowStateSnapshot =
5848 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5849 let handler = Arc::clone(&handler);
5850 let handler_name = replay_name.clone();
5851 let future = async move {
5852 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5853 let result = handler(ctx, input, state).await?;
5854 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5855 };
5856 ReplayedWorkflowInvocation {
5857 future: Box::pin(future),
5858 snapshot,
5859 }
5860 });
5861
5862 self.workflows.insert(
5863 workflow_type,
5864 RegisteredWorkflow {
5865 execute,
5866 replay: Some(replay),
5867 state_type: Some(TypeId::of::<S>()),
5868 },
5869 );
5870 }
5871
5872 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
5874 &mut self,
5875 workflow_type: impl Into<String>,
5876 state_factory: Factory,
5877 handler: F,
5878 ) where
5879 S: Clone + Send + Sync + 'static,
5880 Factory: Fn() -> S + Send + Sync + 'static,
5881 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5882 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5883 {
5884 let state_factory = Arc::new(state_factory);
5885 let handler = Arc::new(handler);
5886
5887 let execute_factory = Arc::clone(&state_factory);
5888 let execute_handler = Arc::clone(&handler);
5889 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5890 let state = WorkflowInstance::new(execute_factory());
5891 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
5892 });
5893
5894 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5895 let state = WorkflowInstance::new(state_factory());
5896 let snapshot_state = state.clone();
5897 let snapshot: WorkflowStateSnapshot =
5898 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5899 ReplayedWorkflowInvocation {
5900 future: Box::pin(handler(ctx, input, state)),
5901 snapshot,
5902 }
5903 });
5904
5905 self.workflows.insert(
5906 workflow_type.into(),
5907 RegisteredWorkflow {
5908 execute,
5909 replay: Some(replay),
5910 state_type: Some(TypeId::of::<S>()),
5911 },
5912 );
5913 }
5914
5915 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
5916 where
5917 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
5918 Fut: Future<Output = Result<Value>> + Send + 'static,
5919 {
5920 let handler = Arc::new(handler);
5921 self.activities.insert(
5922 activity_type.into(),
5923 Arc::new(move |ctx, args| {
5924 let handler = Arc::clone(&handler);
5925 Box::pin(async move {
5926 let result = handler(ctx, args.into_json()?).await?;
5927 AvroValue::from_serialize(&result)
5928 })
5929 }),
5930 );
5931 }
5932
5933 pub fn register_typed_activity<I, O, F, Fut>(
5939 &mut self,
5940 activity_type: impl Into<String>,
5941 handler: F,
5942 ) where
5943 I: DeserializeOwned + Send + 'static,
5944 O: Serialize + Send + 'static,
5945 F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
5946 Fut: Future<Output = Result<O>> + Send + 'static,
5947 {
5948 let activity_type = activity_type.into();
5949 let handler_name = activity_type.clone();
5950 let handler = Arc::new(handler);
5951 self.activities.insert(
5952 activity_type,
5953 Arc::new(move |ctx, input| {
5954 let handler = Arc::clone(&handler);
5955 let handler_name = handler_name.clone();
5956 Box::pin(async move {
5957 let input =
5958 decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
5959 let result = handler(ctx, input).await?;
5960 encode_handler_result(&result, HandlerKind::Activity, &handler_name)
5961 })
5962 }),
5963 );
5964 }
5965
5966 pub fn register_activity_avro_value<F, Fut>(
5968 &mut self,
5969 activity_type: impl Into<String>,
5970 handler: F,
5971 ) where
5972 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
5973 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5974 {
5975 self.activities.insert(
5976 activity_type.into(),
5977 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
5978 );
5979 }
5980
5981 pub fn register_query<F, Fut>(
5987 &mut self,
5988 workflow_type: impl Into<String>,
5989 query_name: impl Into<String>,
5990 handler: F,
5991 ) where
5992 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
5993 Fut: Future<Output = Result<Value>> + Send + 'static,
5994 {
5995 let handler = Arc::new(handler);
5996 self.queries
5997 .entry(workflow_type.into())
5998 .or_default()
5999 .insert(
6000 query_name.into(),
6001 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6002 let handler = Arc::clone(&handler);
6003 Box::pin(async move {
6004 let result = handler(ctx, args.into_json()?).await?;
6005 AvroValue::from_serialize(&result)
6006 })
6007 })),
6008 );
6009 }
6010
6011 pub fn register_query_avro_value<F, Fut>(
6013 &mut self,
6014 workflow_type: impl Into<String>,
6015 query_name: impl Into<String>,
6016 handler: F,
6017 ) where
6018 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6019 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6020 {
6021 self.queries
6022 .entry(workflow_type.into())
6023 .or_default()
6024 .insert(
6025 query_name.into(),
6026 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6027 );
6028 }
6029
6030 pub fn register_replayed_query<S, F, Fut>(
6037 &mut self,
6038 workflow_type: impl Into<String>,
6039 query_name: impl Into<String>,
6040 handler: F,
6041 ) where
6042 S: Clone + Send + Sync + 'static,
6043 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6044 Fut: Future<Output = Result<Value>> + Send + 'static,
6045 {
6046 let handler = Arc::new(handler);
6047 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6048 let state = state.downcast::<S>().map_err(|_| {
6049 "registered query state type does not match the replayed workflow state".to_string()
6050 })?;
6051 let handler = Arc::clone(&handler);
6052 Ok(Box::pin(async move {
6053 let result = handler(ctx, state, args.into_json()?).await?;
6054 AvroValue::from_serialize(&result)
6055 }))
6056 });
6057
6058 self.queries
6059 .entry(workflow_type.into())
6060 .or_default()
6061 .insert(
6062 query_name.into(),
6063 RegisteredQuery::Replayed {
6064 state_type: TypeId::of::<S>(),
6065 handler: erased_handler,
6066 },
6067 );
6068 }
6069
6070 pub fn register_replayed_query_avro_value<S, F, Fut>(
6072 &mut self,
6073 workflow_type: impl Into<String>,
6074 query_name: impl Into<String>,
6075 handler: F,
6076 ) where
6077 S: Clone + Send + Sync + 'static,
6078 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6079 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6080 {
6081 let handler = Arc::new(handler);
6082 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6083 let state = state.downcast::<S>().map_err(|_| {
6084 "registered query state type does not match the replayed workflow state".to_string()
6085 })?;
6086 Ok(Box::pin(handler(ctx, state, args)))
6087 });
6088
6089 self.queries
6090 .entry(workflow_type.into())
6091 .or_default()
6092 .insert(
6093 query_name.into(),
6094 RegisteredQuery::Replayed {
6095 state_type: TypeId::of::<S>(),
6096 handler: erased_handler,
6097 },
6098 );
6099 }
6100
6101 pub fn register_update<F, Fut>(
6103 &mut self,
6104 workflow_type: impl Into<String>,
6105 update_name: impl Into<String>,
6106 handler: F,
6107 ) where
6108 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6109 Fut: Future<Output = Result<Value>> + Send + 'static,
6110 {
6111 let handler = Arc::new(handler);
6112 self.updates
6113 .entry(workflow_type.into())
6114 .or_default()
6115 .insert(
6116 update_name.into(),
6117 Arc::new(move |ctx, args| {
6118 let handler = Arc::clone(&handler);
6119 Box::pin(async move {
6120 let result = handler(ctx, args.into_json()?).await?;
6121 AvroValue::from_serialize(&result)
6122 })
6123 }),
6124 );
6125 }
6126
6127 pub fn register_update_avro_value<F, Fut>(
6129 &mut self,
6130 workflow_type: impl Into<String>,
6131 update_name: impl Into<String>,
6132 handler: F,
6133 ) where
6134 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6135 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6136 {
6137 self.updates
6138 .entry(workflow_type.into())
6139 .or_default()
6140 .insert(
6141 update_name.into(),
6142 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6143 );
6144 }
6145
6146 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6147 let mut command_contracts = serde_json::Map::new();
6148 for workflow_type in self.workflows.keys() {
6149 let mut queries = self
6150 .queries
6151 .get(workflow_type)
6152 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6153 .unwrap_or_default();
6154 queries.sort();
6155 let mut updates = self
6156 .updates
6157 .get(workflow_type)
6158 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6159 .unwrap_or_default();
6160 updates.sort();
6161 command_contracts.insert(
6162 workflow_type.clone(),
6163 json!({
6164 "queries": queries,
6165 "query_contracts": [],
6166 "signals": [],
6167 "signal_contracts": [],
6168 "updates": updates,
6169 "update_contracts": [],
6170 "update_validators": [],
6171 }),
6172 );
6173 }
6174
6175 self.client
6176 .register_worker_with_command_contracts(
6177 &self.worker_id,
6178 &self.task_queue,
6179 self.workflows.keys().cloned().collect(),
6180 self.activities.keys().cloned().collect(),
6181 self.max_concurrent_workflow_tasks,
6182 self.max_concurrent_activity_tasks,
6183 [
6184 Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6185 Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6186 Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6187 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6188 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6189 worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6190 .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6191 ]
6192 .into_iter()
6193 .flatten()
6194 .collect(),
6195 Value::Object(command_contracts),
6196 )
6197 .await
6198 }
6199
6200 pub async fn run(&self) -> Result<()> {
6206 self.run_until(std::future::pending::<()>()).await
6207 }
6208
6209 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6213 where
6214 F: Future<Output = ()>,
6215 {
6216 let registration = self.register().await?;
6217 if !registration.registered {
6218 return Err(Error::WorkerLoop(format!(
6219 "worker registration for {:?} was not accepted",
6220 self.worker_id
6221 )));
6222 }
6223 let registered_worker_id = registration.worker_id.clone();
6224 let primary = self.run_registered_until(shutdown, registration).await;
6225 let deregistration = self
6226 .client
6227 .deregister_worker_registration(®istered_worker_id)
6228 .await;
6229
6230 match (primary, deregistration) {
6231 (Ok(()), Ok(_)) => Ok(()),
6232 (Ok(()), Err(deregistration)) => Err(deregistration),
6233 (Err(primary), Ok(_)) => Err(primary),
6234 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6235 primary: Box::new(primary),
6236 deregistration: Box::new(deregistration),
6237 }),
6238 }
6239 }
6240
6241 async fn run_registered_until<F>(
6242 &self,
6243 shutdown: F,
6244 registration: RegisterWorkerResponse,
6245 ) -> Result<()>
6246 where
6247 F: Future<Output = ()>,
6248 {
6249 let heartbeat_interval = Duration::from_secs(
6250 registration
6251 .heartbeat_interval_seconds
6252 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6253 );
6254 let heartbeat = tokio::time::sleep(Duration::ZERO);
6260 tokio::pin!(heartbeat);
6261 tokio::pin!(shutdown);
6262 let stop = Arc::new(AtomicBool::new(false));
6263 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6267 let worker = self.clone();
6268 let stop = Arc::clone(&stop);
6269 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6270 });
6271 let mut activity_poller = (!self.activities.is_empty()).then(|| {
6272 let worker = self.clone();
6273 let stop = Arc::clone(&stop);
6274 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6275 });
6276 let mut query_poller = (!self.queries.is_empty()).then(|| {
6277 let worker = self.clone();
6278 let stop = Arc::clone(&stop);
6279 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6280 });
6281
6282 loop {
6283 tokio::select! {
6284 _ = &mut shutdown => {
6285 stop.store(true, Ordering::SeqCst);
6286 break;
6287 }
6288 _ = &mut heartbeat => {
6289 let result = self.retry_worker_operation(|| {
6290 self.client.heartbeat_worker(
6291 &self.worker_id,
6292 self.max_concurrent_workflow_tasks,
6293 self.max_concurrent_activity_tasks,
6294 )
6295 }).await;
6296 heartbeat
6297 .as_mut()
6298 .reset(tokio::time::Instant::now() + heartbeat_interval);
6299 match result {
6300 Ok(acknowledgement) => {
6301 if let Some(observer) = &self.heartbeat_observer {
6302 observer(&WorkerHeartbeatObservation {
6303 worker_id: self.worker_id.clone(),
6304 task_queue: self.task_queue.clone(),
6305 acknowledged_at_unix_millis: SystemTime::now()
6306 .duration_since(UNIX_EPOCH)
6307 .unwrap_or_default()
6308 .as_millis()
6309 .min(u64::MAX as u128)
6310 as u64,
6311 acknowledgement,
6312 });
6313 }
6314 }
6315 Err(error) => {
6316 stop.store(true, Ordering::SeqCst);
6317 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6318 return Err(error);
6319 }
6320 }
6321 }
6322 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6323 workflow_poller = None;
6324 let stopped_by_server = stop.load(Ordering::SeqCst);
6325 stop.store(true, Ordering::SeqCst);
6326 let poller_result = optional_poller_result("workflow", result);
6327 let join_result =
6328 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6329 poller_result?;
6330 join_result?;
6331 if stopped_by_server {
6332 return Ok(());
6333 }
6334 return Err(Error::WorkerLoop(
6335 "workflow poller stopped unexpectedly".to_string(),
6336 ));
6337 }
6338 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6339 activity_poller = None;
6340 let stopped_by_server = stop.load(Ordering::SeqCst);
6341 stop.store(true, Ordering::SeqCst);
6342 let poller_result = optional_poller_result("activity", result);
6343 let join_result =
6344 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6345 poller_result?;
6346 join_result?;
6347 if stopped_by_server {
6348 return Ok(());
6349 }
6350 return Err(Error::WorkerLoop(
6351 "activity poller stopped unexpectedly".to_string(),
6352 ));
6353 }
6354 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6355 query_poller = None;
6356 let stopped_by_server = stop.load(Ordering::SeqCst);
6357 stop.store(true, Ordering::SeqCst);
6358 let poller_result = optional_poller_result("query", result);
6359 let join_result =
6360 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6361 poller_result?;
6362 join_result?;
6363 if stopped_by_server {
6364 return Ok(());
6365 }
6366 return Err(Error::WorkerLoop(
6367 "query poller stopped unexpectedly".to_string(),
6368 ));
6369 }
6370 }
6371 }
6372
6373 join_pollers(
6374 workflow_poller.take(),
6375 activity_poller.take(),
6376 query_poller.take(),
6377 )
6378 .await
6379 }
6380
6381 pub async fn run_once(&self) -> Result<usize> {
6395 let mut handled = 0;
6396 match self.poll_workflow_once().await? {
6397 ManagedPollOutcome::Handled => handled += 1,
6398 ManagedPollOutcome::Stop => return Ok(handled),
6399 ManagedPollOutcome::Idle => {}
6400 }
6401 match self.poll_activity_once().await? {
6402 ManagedPollOutcome::Handled => handled += 1,
6403 ManagedPollOutcome::Stop => return Ok(handled),
6404 ManagedPollOutcome::Idle => {}
6405 }
6406 if !self.queries.is_empty() {
6407 match self.poll_query_once().await? {
6408 ManagedPollOutcome::Handled => handled += 1,
6409 ManagedPollOutcome::Stop => return Ok(handled),
6410 ManagedPollOutcome::Idle => {}
6411 }
6412 }
6413 Ok(handled)
6414 }
6415
6416 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6417 let poll_request_id = unique_request_id("rust-workflow-poll");
6418 let response = self
6419 .retry_worker_operation(|| {
6420 self.client.poll_workflow_task_response_with_request_id(
6421 &self.worker_id,
6422 &self.task_queue,
6423 self.poll_timeout,
6424 &poll_request_id,
6425 0,
6426 )
6427 })
6428 .await?;
6429 if response.outcome().should_stop() {
6430 return Ok(ManagedPollOutcome::Stop);
6431 }
6432 let memo_updates_supported =
6433 runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6434 let Some(task) = response.task else {
6435 return Ok(ManagedPollOutcome::Idle);
6436 };
6437
6438 let task_id = task.task_id.clone();
6439 let attempt = task.workflow_task_attempt;
6440 let run_id = task.run_id.clone();
6441 let lease_owner = task
6442 .lease_owner
6443 .clone()
6444 .unwrap_or_else(|| self.worker_id.clone());
6445
6446 match self.execute_workflow_task_decision(task) {
6447 Ok(decision)
6448 if commands_use_workflow_memo_updates(&decision.commands)
6449 && !memo_updates_supported =>
6450 {
6451 self.client
6452 .fail_workflow_task(
6453 &task_id,
6454 &lease_owner,
6455 attempt,
6456 Error::WorkflowMemoUpdatesUnavailable.to_string(),
6457 )
6458 .await?;
6459 }
6460 Ok(decision) if decision.commands.is_empty() => {
6461 self.client
6467 .fail_workflow_task_with_type(
6468 &task_id,
6469 &lease_owner,
6470 attempt,
6471 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6472 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6473 )
6474 .await?;
6475 }
6476 Ok(decision) => {
6477 let completion = self
6478 .client
6479 .complete_workflow_task_with_message_streams(
6480 &task_id,
6481 &lease_owner,
6482 attempt,
6483 decision.commands,
6484 decision.message_stream_cursors,
6485 decision.message_stream_waits,
6486 )
6487 .await;
6488 if let Err(error) = completion {
6489 if !workflow_task_completion_is_terminal_timeout(
6490 &error,
6491 &task_id,
6492 attempt,
6493 run_id.as_deref(),
6494 ) {
6495 return Err(error);
6496 }
6497 }
6498 }
6499 Err(error) => {
6500 self.client
6501 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6502 .await?;
6503 }
6504 }
6505
6506 Ok(ManagedPollOutcome::Handled)
6507 }
6508
6509 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6510 while !stop.load(Ordering::SeqCst) {
6511 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6512 stop.store(true, Ordering::SeqCst);
6513 break;
6514 }
6515 }
6516
6517 Ok(())
6518 }
6519
6520 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6521 let poll_request_id = unique_request_id("rust-activity-poll");
6522 let response = self
6523 .retry_worker_operation(|| {
6524 self.client.poll_activity_task_response_with_request_id(
6525 &self.worker_id,
6526 &self.task_queue,
6527 self.poll_timeout,
6528 &poll_request_id,
6529 0,
6530 )
6531 })
6532 .await?;
6533 if response.outcome().should_stop() {
6534 return Ok(ManagedPollOutcome::Stop);
6535 }
6536 let Some(task) = response.task else {
6537 return Ok(ManagedPollOutcome::Idle);
6538 };
6539
6540 let task_id = task.task_id.clone();
6541 let attempt_id = task
6542 .activity_attempt_id
6543 .clone()
6544 .or(task.attempt_id.clone())
6545 .unwrap_or_default();
6546 let lease_owner = task
6547 .lease_owner
6548 .clone()
6549 .unwrap_or_else(|| self.worker_id.clone());
6550 let codec = task.payload_codec.clone();
6551 let result = self.execute_activity_task(task).await;
6552 match result {
6553 Ok(value) => {
6554 let completion = self
6555 .client
6556 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6557 .await;
6558 if let Err(error) = completion {
6559 if !activity_task_rejection_is_final(&error) {
6560 return Err(error);
6561 }
6562 }
6563 }
6564 Err(error) => {
6565 let failure = self
6566 .client
6567 .fail_activity_task(
6568 &task_id,
6569 &attempt_id,
6570 &lease_owner,
6571 error.to_string(),
6572 false,
6573 )
6574 .await;
6575 if let Err(error) = failure {
6576 if !activity_task_rejection_is_final(&error) {
6577 return Err(error);
6578 }
6579 }
6580 }
6581 }
6582
6583 Ok(ManagedPollOutcome::Handled)
6584 }
6585
6586 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6587 while !stop.load(Ordering::SeqCst) {
6588 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6589 stop.store(true, Ordering::SeqCst);
6590 break;
6591 }
6592 }
6593
6594 Ok(())
6595 }
6596
6597 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6598 let poll_request_id = unique_request_id("rust-query-poll");
6599 let response = self
6600 .retry_worker_operation(|| {
6601 self.client.poll_query_task_response_with_request_id(
6602 &self.worker_id,
6603 &self.task_queue,
6604 self.poll_timeout,
6605 &poll_request_id,
6606 0,
6607 )
6608 })
6609 .await?;
6610 if response.outcome().should_stop() {
6611 return Ok(ManagedPollOutcome::Stop);
6612 }
6613 let Some(task) = response.task else {
6614 return Ok(ManagedPollOutcome::Idle);
6615 };
6616
6617 let query_task_id = task.query_task_id.clone();
6618 let attempt = task.query_task_attempt;
6619 let lease_owner = task
6620 .lease_owner
6621 .clone()
6622 .unwrap_or_else(|| self.worker_id.clone());
6623 let codec = task.payload_codec.clone();
6624
6625 match self.execute_query_task(task).await {
6626 Ok(value) => {
6627 let result_envelope = match encode_typed_envelope(&value, &codec) {
6628 Ok(result_envelope) => result_envelope,
6629 Err(error) => {
6630 let failure = self
6631 .client
6632 .fail_query_task(
6633 &query_task_id,
6634 &lease_owner,
6635 attempt,
6636 error.to_string(),
6637 "query_result_encode_failed",
6638 "QueryResultEncodeFailed",
6639 )
6640 .await;
6641 if let Err(error) = failure {
6642 if !query_task_rejection_is_final(&error) {
6643 return Err(error);
6644 }
6645 }
6646 return Ok(ManagedPollOutcome::Handled);
6647 }
6648 };
6649
6650 if let Err(error) = self
6651 .client
6652 .complete_query_task_with_envelope(
6653 &query_task_id,
6654 &lease_owner,
6655 attempt,
6656 value.clone().into_json()?,
6657 result_envelope,
6658 )
6659 .await
6660 {
6661 if !query_task_rejection_is_final(&error) {
6662 return Err(error);
6663 }
6664 }
6665 }
6666 Err(failure) => {
6667 let result = self
6668 .client
6669 .fail_query_task(
6670 &query_task_id,
6671 &lease_owner,
6672 attempt,
6673 failure.message,
6674 failure.reason,
6675 failure.failure_type,
6676 )
6677 .await;
6678 if let Err(error) = result {
6679 if !query_task_rejection_is_final(&error) {
6680 return Err(error);
6681 }
6682 }
6683 }
6684 }
6685
6686 Ok(ManagedPollOutcome::Handled)
6687 }
6688
6689 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6690 while !stop.load(Ordering::SeqCst) {
6691 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
6692 stop.store(true, Ordering::SeqCst);
6693 break;
6694 }
6695 }
6696
6697 Ok(())
6698 }
6699
6700 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
6701 where
6702 F: FnMut() -> Fut,
6703 Fut: Future<Output = Result<T>>,
6704 {
6705 let mut retries = 0;
6706
6707 loop {
6708 match operation().await {
6709 Err(error)
6710 if worker_operation_is_retryable(&error)
6711 && retries < self.retry_policy.max_retries =>
6712 {
6713 retries += 1;
6714 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
6715 }
6716 result => return result,
6717 }
6718 }
6719 }
6720
6721 async fn execute_query_task(
6722 &self,
6723 mut task: QueryTask,
6724 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
6725 validate_query_task_payloads(&task).map_err(|error| {
6726 QueryTaskExecutionFailure::new(
6727 "query_payload_decode_failed",
6728 error.to_string(),
6729 "QueryPayloadDecodeFailed",
6730 )
6731 })?;
6732
6733 if !self.workflows.contains_key(&task.workflow_type) {
6734 return Err(QueryTaskExecutionFailure::new(
6735 "query_workflow_type_not_registered",
6736 format!("no workflow registered for type {:?}", task.workflow_type),
6737 "WorkflowTypeNotRegistered",
6738 ));
6739 }
6740
6741 let Some(handlers) = self.queries.get(&task.workflow_type) else {
6742 return Err(QueryTaskExecutionFailure::new(
6743 "query_handler_unavailable",
6744 format!(
6745 "query handlers are unavailable for workflow type {:?}",
6746 task.workflow_type
6747 ),
6748 "QueryHandlerUnavailable",
6749 ));
6750 };
6751 let Some(query) = handlers.get(&task.query_name) else {
6752 return Err(QueryTaskExecutionFailure::new(
6753 "rejected_unknown_query",
6754 format!("unknown query {:?}", task.query_name),
6755 "QueryFailed",
6756 ));
6757 };
6758
6759 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
6760 .map_err(|error| {
6761 QueryTaskExecutionFailure::new(
6762 "query_payload_decode_failed",
6763 format!("cannot decode query arguments: {error}"),
6764 "QueryPayloadDecodeFailed",
6765 )
6766 })?;
6767 let workflow_input_typed =
6768 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
6769 .map_err(|error| {
6770 QueryTaskExecutionFailure::new(
6771 "query_workflow_state_unavailable",
6772 format!("cannot decode workflow start input: {error}"),
6773 "QueryWorkflowStateUnavailable",
6774 )
6775 })?;
6776 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
6777 QueryTaskExecutionFailure::new(
6778 "query_workflow_state_unavailable",
6779 format!("cannot project workflow start input: {error}"),
6780 "QueryWorkflowStateUnavailable",
6781 )
6782 })?;
6783 hydrate_query_history_from_export(&mut task).map_err(|error| {
6784 QueryTaskExecutionFailure::new(
6785 "query_workflow_state_unavailable",
6786 format!("cannot restore query history snapshot: {error}"),
6787 "QueryWorkflowStateUnavailable",
6788 )
6789 })?;
6790 enrich_query_history_from_export(&mut task).map_err(|error| {
6791 QueryTaskExecutionFailure::new(
6792 "query_workflow_state_unavailable",
6793 format!("cannot restore compact query history payloads: {error}"),
6794 "QueryWorkflowStateUnavailable",
6795 )
6796 })?;
6797 let signal_events = query_signal_events(&task).map_err(|error| {
6798 QueryTaskExecutionFailure::new(
6799 "query_workflow_state_unavailable",
6800 format!("cannot decode committed workflow signals: {error}"),
6801 "QueryWorkflowStateUnavailable",
6802 )
6803 })?;
6804 let history_events = Arc::new(std::mem::take(&mut task.history_events));
6805 let context = QueryContext {
6806 workflow_id: task.workflow_id,
6807 run_id: task.run_id,
6808 workflow_type: task.workflow_type.clone(),
6809 run_status: task.run_status,
6810 workflow_input,
6811 workflow_input_avro_value: workflow_input_typed.clone(),
6812 history_events: Arc::clone(&history_events),
6813 signal_events: Arc::new(signal_events),
6814 };
6815
6816 let future = match query {
6817 RegisteredQuery::Snapshot(handler) => handler(context, args),
6818 RegisteredQuery::Replayed {
6819 state_type,
6820 handler,
6821 } => {
6822 let workflow = self
6823 .workflows
6824 .get(&task.workflow_type)
6825 .expect("workflow registration was checked above");
6826 if workflow.state_type != Some(*state_type) {
6827 return Err(QueryTaskExecutionFailure::new(
6828 "query_workflow_state_unavailable",
6829 "replayed query state type does not match its workflow registration",
6830 "QueryWorkflowStateUnavailable",
6831 ));
6832 }
6833 let replay = workflow.replay.as_ref().ok_or_else(|| {
6834 QueryTaskExecutionFailure::new(
6835 "query_workflow_state_unavailable",
6836 format!(
6837 "workflow type {:?} is not registered for instance-state replay",
6838 task.workflow_type
6839 ),
6840 "QueryWorkflowStateUnavailable",
6841 )
6842 })?;
6843 let workflow_state = Arc::new(Mutex::new(
6844 WorkflowState::new_with_identity(
6845 history_events.as_ref().clone(),
6846 context.workflow_id.clone(),
6847 context.run_id.clone(),
6848 self.task_queue.clone(),
6849 task.payload_codec,
6850 None,
6851 )
6852 .map_err(|error| {
6853 QueryTaskExecutionFailure::new(
6854 "query_workflow_state_unavailable",
6855 format!("workflow replay failed before query: {error}"),
6856 "QueryWorkflowStateUnavailable",
6857 )
6858 })?,
6859 ));
6860 let workflow_context = WorkflowContext {
6861 state: workflow_state,
6862 };
6863 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
6864 let mut cx = TaskContext::from_waker(noop_waker_ref());
6865 match invocation.future.as_mut().poll(&mut cx) {
6866 Poll::Ready(Ok(_)) => {
6867 workflow_context
6868 .ensure_history_consumed()
6869 .map_err(|error| {
6870 QueryTaskExecutionFailure::new(
6871 "query_workflow_state_unavailable",
6872 format!("workflow replay failed before query: {error}"),
6873 "QueryWorkflowStateUnavailable",
6874 )
6875 })?;
6876 }
6877 Poll::Ready(Err(error)) => {
6878 return Err(QueryTaskExecutionFailure::new(
6879 "query_workflow_state_unavailable",
6880 format!("workflow replay failed before query: {error}"),
6881 "QueryWorkflowStateUnavailable",
6882 ));
6883 }
6884 Poll::Pending => {
6885 let commands = workflow_context.take_commands().map_err(|error| {
6886 QueryTaskExecutionFailure::new(
6887 "query_workflow_state_unavailable",
6888 format!("workflow replay failed before query: {error}"),
6889 "QueryWorkflowStateUnavailable",
6890 )
6891 })?;
6892 if commands.is_empty()
6893 && !workflow_context
6894 .matched_recorded_pending()
6895 .map_err(|error| {
6896 QueryTaskExecutionFailure::new(
6897 "query_workflow_state_unavailable",
6898 format!("workflow replay failed before query: {error}"),
6899 "QueryWorkflowStateUnavailable",
6900 )
6901 })?
6902 {
6903 return Err(QueryTaskExecutionFailure::new(
6904 "query_workflow_state_unavailable",
6905 "workflow replay yielded without a durable command",
6906 "QueryWorkflowStateUnavailable",
6907 ));
6908 }
6909 }
6910 }
6911 let state = (invocation.snapshot)().map_err(|error| {
6912 QueryTaskExecutionFailure::new(
6913 "query_workflow_state_unavailable",
6914 format!("cannot snapshot replayed workflow state: {error}"),
6915 "QueryWorkflowStateUnavailable",
6916 )
6917 })?;
6918 handler(context, state, args).map_err(|message| {
6919 QueryTaskExecutionFailure::new(
6920 "query_workflow_state_unavailable",
6921 message,
6922 "QueryWorkflowStateUnavailable",
6923 )
6924 })?
6925 }
6926 };
6927
6928 future.await.map_err(|error| {
6929 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
6930 })
6931 }
6932
6933 #[cfg(test)]
6934 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
6935 Ok(self.execute_workflow_task_decision(task)?.commands)
6936 }
6937
6938 fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
6939 validate_workflow_task_payloads(&task)?;
6940
6941 if let Some(update_id) = task
6942 .workflow_update_id
6943 .as_deref()
6944 .filter(|update_id| !update_id.is_empty())
6945 {
6946 return self
6947 .execute_update_task(&task, update_id)
6948 .map(WorkflowTaskDecision::without_message_streams);
6949 }
6950
6951 let workflow = self
6952 .workflows
6953 .get(&task.workflow_type)
6954 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
6955 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
6956 let resume_signal = decode_resume_signal(&task)?;
6957 let history_budget = WorkflowHistoryBudget {
6958 event_count: task
6959 .total_history_events
6960 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
6961 size_bytes: task.history_size_bytes,
6962 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
6963 pressure: task.history_budget_pressure.clone(),
6964 };
6965 let workflow_command_identity = task
6966 .workflow_command_id
6967 .clone()
6968 .filter(|identity| !identity.is_empty())
6969 .unwrap_or_default();
6970 let mut workflow_state = WorkflowState::new_with_identity(
6971 task.history_events,
6972 task.workflow_id,
6973 task.run_id,
6974 self.task_queue.clone(),
6975 task.payload_codec.clone(),
6976 resume_signal,
6977 )?;
6978 workflow_state.history_budget = history_budget;
6979 workflow_state.workflow_command_identity = workflow_command_identity;
6980 workflow_state.cancel_requested = task.cancel_requested;
6981 let state = Arc::new(Mutex::new(workflow_state));
6982 let ctx = WorkflowContext { state };
6983 let mut future = (workflow.execute)(ctx.clone(), input);
6984 let mut cx = TaskContext::from_waker(noop_waker_ref());
6985
6986 match future.as_mut().poll(&mut cx) {
6987 Poll::Ready(Ok(result)) => {
6988 ctx.ensure_history_consumed()?;
6989 let result = encode_typed_envelope(&result, &task.payload_codec)?;
6990 let mut commands = ctx.take_commands()?;
6991 commands.push(json!({
6992 "type": "complete_workflow",
6993 "result": result
6994 }));
6995 self.message_stream_decision(&ctx, commands)
6996 }
6997 Poll::Ready(Err(error)) => {
6998 if let Error::ContinueAsNew(request) = error {
6999 let mut commands = ctx.take_commands()?;
7000 if let Some(command) = ctx.continue_as_new_command(request)? {
7001 commands.push(command);
7002 }
7003 ctx.ensure_history_consumed()?;
7004 return self.message_stream_decision(&ctx, commands);
7005 }
7006 if workflow_task_integrity_error(&error) {
7007 return Err(error);
7012 }
7013 ctx.ensure_history_consumed()?;
7016 let mut commands = ctx.take_commands()?;
7017 commands.push(workflow_failure_command(&error));
7018 self.message_stream_decision(&ctx, commands)
7019 }
7020 Poll::Pending => {
7021 let commands = ctx.take_commands()?;
7022 if commands.is_empty() && !ctx.matched_recorded_pending()? {
7023 Err(Error::WorkflowYieldedWithoutCommand)
7024 } else {
7025 self.message_stream_decision(&ctx, commands)
7026 }
7027 }
7028 }
7029 }
7030
7031 fn message_stream_decision(
7032 &self,
7033 ctx: &WorkflowContext,
7034 commands: Vec<Value>,
7035 ) -> Result<WorkflowTaskDecision> {
7036 let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7037 Ok(WorkflowTaskDecision {
7038 commands,
7039 message_stream_cursors,
7040 message_stream_waits,
7041 })
7042 }
7043
7044 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7045 if !self.workflows.contains_key(&task.workflow_type) {
7046 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7047 }
7048
7049 let accepted = task.history_events.iter().rev().find_map(|event| {
7050 (event.event_type == "UpdateAccepted"
7051 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7052 .then_some(&event.payload)
7053 });
7054 let update_name = accepted
7055 .and_then(|payload| payload.get("update_name"))
7056 .and_then(Value::as_str)
7057 .or(task.update_name.as_deref())
7058 .unwrap_or_default();
7059 let Some(handler) = self
7060 .updates
7061 .get(&task.workflow_type)
7062 .and_then(|handlers| handlers.get(update_name))
7063 else {
7064 return Ok(vec![json!({
7065 "type": "fail_update",
7066 "update_id": update_id,
7067 "message": format!(
7068 "no update handler is registered for {}.{update_name}",
7069 task.workflow_type
7070 ),
7071 "exception_type": "UnknownUpdate",
7072 "non_retryable": true,
7073 })]);
7074 };
7075 let arguments = accepted
7076 .and_then(|payload| payload.get("arguments"))
7077 .or(task.arguments.as_ref());
7078 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7079 let context = QueryContext {
7080 workflow_id: task.workflow_id.clone(),
7081 run_id: task.run_id.clone(),
7082 workflow_type: task.workflow_type.clone(),
7083 run_status: Some("running".to_string()),
7084 workflow_input: Value::Null,
7085 workflow_input_avro_value: AvroValue::Null,
7086 history_events: Arc::new(task.history_events.clone()),
7087 signal_events: Arc::new(Vec::new()),
7088 };
7089 let mut future = handler(context, arguments);
7090 let mut cx = TaskContext::from_waker(noop_waker_ref());
7091
7092 match future.as_mut().poll(&mut cx) {
7093 Poll::Ready(Ok(result)) => Ok(vec![json!({
7094 "type": "complete_update",
7095 "update_id": update_id,
7096 "result": encode_typed_envelope(&result, &task.payload_codec)?,
7097 })]),
7098 Poll::Ready(Err(error)) => Ok(vec![json!({
7099 "type": "fail_update",
7100 "update_id": update_id,
7101 "message": error.to_string(),
7102 "exception_type": "UpdateFailed",
7103 "non_retryable": true,
7104 })]),
7105 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7106 }
7107 }
7108
7109 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7110 validate_activity_task_payloads(&task)?;
7111
7112 let handler = self
7113 .activities
7114 .get(&task.activity_type)
7115 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7116 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7117 let attempt_id = task
7118 .activity_attempt_id
7119 .clone()
7120 .or(task.attempt_id.clone())
7121 .unwrap_or_default();
7122 let lease_owner = task
7123 .lease_owner
7124 .clone()
7125 .unwrap_or_else(|| self.worker_id.clone());
7126 let ctx = ActivityContext {
7127 client: self.client.clone(),
7128 task_id: task.task_id,
7129 activity_attempt_id: attempt_id,
7130 lease_owner,
7131 activity_type: task.activity_type,
7132 attempt_number: task.attempt_number,
7133 task_queue: self.task_queue.clone(),
7134 worker_id: self.worker_id.clone(),
7135 };
7136
7137 handler(ctx, args).await
7138 }
7139}
7140
7141fn poller_result(
7142 kind: &str,
7143 result: std::result::Result<Result<()>, tokio::task::JoinError>,
7144) -> Result<()> {
7145 match result {
7146 Ok(result) => result,
7147 Err(error) => Err(Error::WorkerLoop(format!(
7148 "{kind} poller join error: {error}"
7149 ))),
7150 }
7151}
7152
7153fn optional_poller_result(
7154 kind: &str,
7155 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7156) -> Result<()> {
7157 match result {
7158 Some(result) => poller_result(kind, result),
7159 None => Ok(()),
7160 }
7161}
7162
7163async fn join_pollers(
7164 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7165 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7166 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7167) -> Result<()> {
7168 let mut first_error = None;
7169
7170 if let Some(handle) = workflow_poller {
7171 if let Err(error) = poller_result("workflow", handle.await) {
7172 first_error.get_or_insert(error);
7173 }
7174 }
7175
7176 if let Some(handle) = activity_poller {
7177 if let Err(error) = poller_result("activity", handle.await) {
7178 first_error.get_or_insert(error);
7179 }
7180 }
7181
7182 if let Some(handle) = query_poller {
7183 if let Err(error) = poller_result("query", handle.await) {
7184 first_error.get_or_insert(error);
7185 }
7186 }
7187
7188 if let Some(error) = first_error {
7189 Err(error)
7190 } else {
7191 Ok(())
7192 }
7193}
7194
7195fn default_worker_id() -> String {
7196 let millis = SystemTime::now()
7197 .duration_since(UNIX_EPOCH)
7198 .unwrap_or_default()
7199 .as_millis();
7200 format!("rust-worker-{}-{millis}", std::process::id())
7201}
7202
7203fn percent_encode_path_segment(segment: &str) -> String {
7204 const HEX: &[u8; 16] = b"0123456789ABCDEF";
7205 let mut encoded = String::with_capacity(segment.len());
7206
7207 for byte in segment.bytes() {
7208 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7209 encoded.push(char::from(byte));
7210 } else {
7211 encoded.push('%');
7212 encoded.push(char::from(HEX[(byte >> 4) as usize]));
7213 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7214 }
7215 }
7216
7217 encoded
7218}
7219
7220fn unique_request_id(prefix: &str) -> String {
7221 let nanos = SystemTime::now()
7222 .duration_since(UNIX_EPOCH)
7223 .unwrap_or_default()
7224 .as_nanos();
7225 format!("{prefix}-{}-{nanos}", std::process::id())
7226}
7227
7228#[derive(Debug)]
7229struct QueryTaskExecutionFailure {
7230 reason: String,
7231 message: String,
7232 failure_type: String,
7233}
7234
7235impl QueryTaskExecutionFailure {
7236 fn new(
7237 reason: impl Into<String>,
7238 message: impl Into<String>,
7239 failure_type: impl Into<String>,
7240 ) -> Self {
7241 Self {
7242 reason: reason.into(),
7243 message: message.into(),
7244 failure_type: failure_type.into(),
7245 }
7246 }
7247}
7248
7249#[derive(Clone, Debug)]
7255pub struct WorkflowInstance<S> {
7256 state: Arc<Mutex<S>>,
7257}
7258
7259impl<S> WorkflowInstance<S> {
7260 fn new(state: S) -> Self {
7261 Self {
7262 state: Arc::new(Mutex::new(state)),
7263 }
7264 }
7265
7266 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7268 let state = self
7269 .state
7270 .lock()
7271 .map_err(|_| Error::WorkflowStatePoisoned)?;
7272 Ok(reader(&state))
7273 }
7274
7275 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7277 let mut state = self
7278 .state
7279 .lock()
7280 .map_err(|_| Error::WorkflowStatePoisoned)?;
7281 Ok(transition(&mut state))
7282 }
7283}
7284
7285impl<S: Clone> WorkflowInstance<S> {
7286 fn snapshot(&self) -> Result<S> {
7287 self.read(Clone::clone)
7288 }
7289}
7290
7291#[derive(Clone, Debug, PartialEq)]
7292pub struct MessageStreamMessage {
7293 pub stream_name: String,
7294 pub message_id: String,
7295 pub position: u64,
7296 pub arguments: Vec<AvroValue>,
7297}
7298
7299#[derive(Clone, Debug)]
7300pub struct MessageStream {
7301 ctx: WorkflowContext,
7302 name: String,
7303}
7304
7305impl MessageStream {
7306 pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7308 if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7309 return Err(Error::Codec(format!(
7310 "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7311 )));
7312 }
7313 loop {
7314 if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7315 return Ok(batch);
7316 }
7317
7318 self.ctx.record_message_stream_wait(&self.name)?;
7319 let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7320 let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7321 self.ctx.buffer_message_stream_delivery(arguments)?;
7322 if let Some(sequence) = replay_wait_sequence {
7323 self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7324 }
7325 }
7326 }
7327
7328 pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7329 self.receive(1)
7330 .await?
7331 .into_iter()
7332 .next()
7333 .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7334 }
7335}
7336
7337#[derive(Clone, Debug)]
7338pub struct WorkflowContext {
7339 state: Arc<Mutex<WorkflowState>>,
7340}
7341
7342fn valid_memo_key(key: &str) -> bool {
7343 let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7344
7345 !key.is_empty()
7346 && key.len() <= 64
7347 && (numeric_candidate.is_empty()
7348 || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7349 && key
7350 .bytes()
7351 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7352}
7353
7354fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7355 BASE64
7356 .decode(encode_avro_value(value)?.blob)
7357 .map(|bytes| bytes.len())
7358 .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7359}
7360
7361fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7362 let AvroValue::Map(entries) = value else {
7363 return Err(Error::InvalidMemoUpdate(
7364 "entries must serialize to an Avro string-keyed map".to_string(),
7365 ));
7366 };
7367 if require_entries && entries.is_empty() {
7368 return Err(Error::InvalidMemoUpdate(
7369 "at least one entry is required".to_string(),
7370 ));
7371 }
7372 if entries.len() > MAX_MEMO_ENTRIES {
7373 return Err(Error::InvalidMemoUpdate(format!(
7374 "at most {MAX_MEMO_ENTRIES} entries are allowed"
7375 )));
7376 }
7377
7378 for (key, value) in &entries {
7379 if !valid_memo_key(&key) {
7380 return Err(Error::InvalidMemoUpdate(
7381 "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7382 ));
7383 }
7384 if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7385 return Err(Error::InvalidMemoUpdate(format!(
7386 "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7387 )));
7388 }
7389 }
7390
7391 let value = AvroValue::Map(entries);
7392 if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7393 return Err(Error::InvalidMemoUpdate(format!(
7394 "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7395 )));
7396 }
7397 Ok(value)
7398}
7399
7400fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7401 let object = envelope.as_object().ok_or_else(|| {
7402 Error::InvalidMemoUpdate(
7403 "history field must use the public {codec, blob} payload envelope".to_string(),
7404 )
7405 })?;
7406 if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7407 return Err(Error::InvalidMemoUpdate(
7408 "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7409 ));
7410 }
7411
7412 canonical_memo_entries(
7413 decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7414 require_entries,
7415 )
7416}
7417
7418impl WorkflowContext {
7419 pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7420 let name = name.into();
7421 if name.is_empty()
7422 || name.len() > 128
7423 || !name.bytes().all(|byte| {
7424 byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7425 })
7426 {
7427 return Err(Error::Codec(
7428 "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7429 .to_string(),
7430 ));
7431 }
7432 Ok(MessageStream {
7433 ctx: self.clone(),
7434 name,
7435 })
7436 }
7437
7438 fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7439 let mut state = self
7440 .state
7441 .lock()
7442 .map_err(|_| Error::WorkflowStatePoisoned)?;
7443 let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7444 state
7445 .message_stream_waits
7446 .insert(name.to_string(), position);
7447 Ok(())
7448 }
7449
7450 fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7451 let mut state = self
7452 .state
7453 .lock()
7454 .map_err(|_| Error::WorkflowStatePoisoned)?;
7455 let cursor = state
7456 .message_stream_cursors
7457 .get(&message.stream_name)
7458 .copied()
7459 .unwrap_or(0);
7460 if message.position <= cursor {
7461 return Ok(());
7462 }
7463 let pending = state
7464 .message_stream_messages
7465 .entry(message.stream_name.clone())
7466 .or_default();
7467 if pending.iter().any(|candidate| {
7468 candidate.position == message.position || candidate.message_id == message.message_id
7469 }) {
7470 return Ok(());
7471 }
7472 pending.push(message);
7473 pending.sort_by_key(|candidate| candidate.position);
7474 Ok(())
7475 }
7476
7477 fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7478 if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7479 match delivery {
7480 MessageStreamDelivery::Message(message) => {
7481 let stream_name = message.stream_name.clone();
7482 self.buffer_message_stream(message)?;
7483 return Ok(Some(stream_name));
7484 }
7485 MessageStreamDelivery::Cursor {
7486 stream_name,
7487 through_position,
7488 } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7489 }
7490 }
7491 Ok(None)
7492 }
7493
7494 fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7495 let state = self
7496 .state
7497 .lock()
7498 .map_err(|_| Error::WorkflowStatePoisoned)?;
7499 Ok(match state.recorded_commands.get(state.command_cursor) {
7500 Some(RecordedCommand::SignalWait {
7501 sequence,
7502 signal_name,
7503 ..
7504 }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7505 _ => None,
7506 })
7507 }
7508
7509 fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7510 let (history, payload_codec) = {
7511 let state = self
7512 .state
7513 .lock()
7514 .map_err(|_| Error::WorkflowStatePoisoned)?;
7515 (
7516 Arc::clone(&state.history_events),
7517 state.payload_codec.clone(),
7518 )
7519 };
7520
7521 let Some(opened_index) = history.iter().position(|event| {
7522 event.event_type == "SignalWaitOpened"
7523 && durable_event_sequence(event) == Some(wait_sequence)
7524 && event.payload.get("signal_name").and_then(Value::as_str)
7525 == Some(MESSAGE_STREAM_SIGNAL)
7526 }) else {
7527 return Ok(());
7528 };
7529 let boundary_index = history
7530 .iter()
7531 .enumerate()
7532 .skip(opened_index + 1)
7533 .find_map(|(index, event)| {
7534 (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7535 && is_authored_command_open_event(event))
7536 .then_some(index)
7537 })
7538 .unwrap_or(history.len());
7539
7540 for event in history[opened_index + 1..boundary_index]
7541 .iter()
7542 .filter(|event| {
7543 event.event_type == "SignalReceived"
7544 && event.payload.get("signal_name").and_then(Value::as_str)
7545 == Some(MESSAGE_STREAM_SIGNAL)
7546 })
7547 {
7548 let arguments = decode_signal_event_arguments(event, &payload_codec)?
7549 .into_iter()
7550 .map(AvroValue::into_json)
7551 .collect::<Result<Vec<_>>>()?;
7552 self.buffer_message_stream_delivery(arguments)?;
7553 }
7554 Ok(())
7555 }
7556
7557 fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7558 let mut state = self
7559 .state
7560 .lock()
7561 .map_err(|_| Error::WorkflowStatePoisoned)?;
7562 let cursor = state
7563 .message_stream_cursors
7564 .entry(name.to_string())
7565 .or_default();
7566 *cursor = (*cursor).max(through_position);
7567 if let Some(pending) = state.message_stream_messages.get_mut(name) {
7568 pending.retain(|message| message.position > through_position);
7569 }
7570 Ok(())
7571 }
7572
7573 fn take_message_stream_batch(
7574 &self,
7575 name: &str,
7576 max_items: usize,
7577 ) -> Result<Option<Vec<MessageStreamMessage>>> {
7578 let mut state = self
7579 .state
7580 .lock()
7581 .map_err(|_| Error::WorkflowStatePoisoned)?;
7582 let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7583 let pending = state
7584 .message_stream_messages
7585 .entry(name.to_string())
7586 .or_default();
7587 let count = contiguous_message_stream_count(pending, cursor, max_items);
7588 if count == 0 {
7589 return Ok(None);
7590 }
7591 let batch = pending.drain(..count).collect::<Vec<_>>();
7592 let position = batch.last().map(|message| message.position).unwrap_or(0);
7593 state
7594 .message_stream_cursors
7595 .insert(name.to_string(), position);
7596 state.message_stream_waits.remove(name);
7597 Ok(Some(batch))
7598 }
7599
7600 fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
7601 let state = self
7602 .state
7603 .lock()
7604 .map_err(|_| Error::WorkflowStatePoisoned)?;
7605 let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
7606 cursors.sort_by_key(|(name, _)| *name);
7607 let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
7608 waits.sort_by_key(|(name, _)| *name);
7609 Ok((
7610 cursors
7611 .into_iter()
7612 .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
7613 .collect(),
7614 waits
7615 .into_iter()
7616 .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
7617 .collect(),
7618 ))
7619 }
7620 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7622 let state = self
7623 .state
7624 .lock()
7625 .map_err(|_| Error::WorkflowStatePoisoned)?;
7626 Ok(WorkflowIdentity {
7627 workflow_id: state.workflow_id.clone(),
7628 run_id: state.run_id.clone(),
7629 })
7630 }
7631
7632 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
7634 let state = self
7635 .state
7636 .lock()
7637 .map_err(|_| Error::WorkflowStatePoisoned)?;
7638 Ok(state.history_budget.clone())
7639 }
7640
7641 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
7647 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
7648 }
7649
7650 pub fn continue_as_new_with_options<T: Serialize>(
7652 &self,
7653 options: ContinueAsNewOptions,
7654 args: T,
7655 ) -> Result<Value> {
7656 options.validate()?;
7657 Err(Error::ContinueAsNew(ContinueAsNewRequest {
7658 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
7659 options,
7660 }))
7661 }
7662
7663 pub fn activity<T: Serialize>(
7664 &self,
7665 activity_type: impl Into<String>,
7666 args: T,
7667 ) -> ActivityCall {
7668 self.activity_with_options(activity_type, ActivityOptions::new(), args)
7669 }
7670
7671 pub fn activity_on_queue<T, Q>(
7672 &self,
7673 activity_type: impl Into<String>,
7674 task_queue: Option<Q>,
7675 args: T,
7676 ) -> ActivityCall
7677 where
7678 T: Serialize,
7679 Q: Into<String>,
7680 {
7681 let mut options = ActivityOptions::new();
7682 options.task_queue = task_queue.map(Into::into);
7683 self.activity_with_options(activity_type, options, args)
7684 }
7685
7686 pub fn activity_with_options<T: Serialize>(
7724 &self,
7725 activity_type: impl Into<String>,
7726 options: ActivityOptions,
7727 args: T,
7728 ) -> ActivityCall {
7729 ActivityCall {
7730 ctx: self.clone(),
7731 activity_type: activity_type.into(),
7732 options,
7733 args: Some(AvroValue::from_serialize(&args)),
7734 scheduled: false,
7735 parallel_group_path: Vec::new(),
7736 }
7737 }
7738
7739 pub async fn activity_avro_value<T: Serialize>(
7740 &self,
7741 activity_type: impl Into<String>,
7742 args: T,
7743 ) -> Result<AvroValue> {
7744 let mut call = self.activity(activity_type, args);
7745 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7746 }
7747
7748 pub async fn activity_avro_value_with_options<T: Serialize>(
7749 &self,
7750 activity_type: impl Into<String>,
7751 options: ActivityOptions,
7752 args: T,
7753 ) -> Result<AvroValue> {
7754 let mut call = self.activity_with_options(activity_type, options, args);
7755 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7756 }
7757
7758 pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
7760 where
7761 I: Serialize,
7762 O: DeserializeOwned,
7763 {
7764 self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
7765 .await
7766 }
7767
7768 pub async fn activity_typed_with_options<I, O>(
7774 &self,
7775 activity_type: impl Into<String>,
7776 options: ActivityOptions,
7777 args: I,
7778 ) -> Result<O>
7779 where
7780 I: Serialize,
7781 O: DeserializeOwned,
7782 {
7783 let activity_type = activity_type.into();
7784 let encoded = AvroValue::from_serialize(&args).map_err(|error| {
7785 handler_type_error::<I>(
7786 HandlerKind::Activity,
7787 &activity_type,
7788 HandlerValueKind::Input,
7789 error.to_string(),
7790 )
7791 });
7792 let mut call = ActivityCall {
7793 ctx: self.clone(),
7794 activity_type: activity_type.clone(),
7795 options,
7796 args: Some(encoded),
7797 scheduled: false,
7798 parallel_group_path: Vec::new(),
7799 };
7800 let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
7801 decode_handler_result(result, HandlerKind::Activity, &activity_type)
7802 }
7803
7804 pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7811 ParallelCall::new(self.clone(), operations)
7812 }
7813
7814 pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7816 self.parallel(operations)
7817 }
7818
7819 pub async fn parallel_avro_value(
7821 &self,
7822 operations: Vec<ParallelOperation>,
7823 ) -> Result<Vec<ParallelAvroResult>> {
7824 let mut call = self.parallel(operations);
7825 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7826 }
7827
7828 pub fn saga(&self) -> Saga {
7830 Saga::new(self.clone())
7831 }
7832
7833 pub fn is_cancellation_requested(&self) -> Result<bool> {
7835 let state = self
7836 .state
7837 .lock()
7838 .map_err(|_| Error::WorkflowStatePoisoned)?;
7839 Ok(state.cancel_requested)
7840 }
7841
7842 pub fn throw_if_cancellation_requested(&self) -> Result<()> {
7847 if self.is_cancellation_requested()? {
7848 return Err(Error::WorkflowCancellationRequested(
7849 WorkflowCancellationRequested,
7850 ));
7851 }
7852 Ok(())
7853 }
7854
7855 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
7856 SignalCall {
7857 ctx: self.clone(),
7858 signal_name: signal_name.into(),
7859 runtime_reserved_allowed: false,
7860 opened_wait: false,
7861 matched_pending: false,
7862 }
7863 }
7864
7865 fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
7866 SignalCall {
7867 ctx: self.clone(),
7868 signal_name: signal_name.into(),
7869 runtime_reserved_allowed: true,
7870 opened_wait: false,
7871 matched_pending: false,
7872 }
7873 }
7874
7875 pub async fn wait_signal_avro_value(
7876 &self,
7877 signal_name: impl Into<String>,
7878 ) -> Result<Vec<AvroValue>> {
7879 let mut call = self.wait_signal(signal_name);
7880 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7881 }
7882
7883 pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
7889 self.signals_avro_value(signal_name)?
7890 .into_iter()
7891 .map(|arguments| {
7892 arguments
7893 .into_iter()
7894 .map(AvroValue::into_json)
7895 .collect::<Result<Vec<_>>>()
7896 })
7897 .collect()
7898 }
7899
7900 pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
7902 let state = self
7903 .state
7904 .lock()
7905 .map_err(|_| Error::WorkflowStatePoisoned)?;
7906 state
7907 .history_events
7908 .iter()
7909 .filter(|event| {
7910 event.event_type == "SignalReceived"
7911 && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
7912 })
7913 .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
7914 .collect()
7915 }
7916
7917 pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
7923 self.updates_avro_value(update_name)?
7924 .into_iter()
7925 .map(|arguments| {
7926 arguments
7927 .into_iter()
7928 .map(AvroValue::into_json)
7929 .collect::<Result<Vec<_>>>()
7930 })
7931 .collect()
7932 }
7933
7934 pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
7936 let state = self
7937 .state
7938 .lock()
7939 .map_err(|_| Error::WorkflowStatePoisoned)?;
7940 let mut seen = Vec::new();
7941 let mut updates = Vec::new();
7942 for event in state.history_events.iter() {
7943 if !matches!(
7944 event.event_type.as_str(),
7945 "UpdateAccepted" | "UpdateApplied"
7946 ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
7947 || event.payload.get("arguments").is_none()
7948 {
7949 continue;
7950 }
7951 if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
7952 if seen.iter().any(|recorded| recorded == update_id) {
7953 continue;
7954 }
7955 seen.push(update_id.to_string());
7956 }
7957 updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
7958 }
7959 Ok(updates)
7960 }
7961
7962 pub fn wait_condition<F>(
7970 &self,
7971 options: ConditionWaitOptions,
7972 predicate: F,
7973 ) -> ConditionWaitCall
7974 where
7975 F: Fn() -> Result<bool> + Send + 'static,
7976 {
7977 ConditionWaitCall {
7978 ctx: self.clone(),
7979 options,
7980 predicate: Box::new(predicate),
7981 occurrence_id: None,
7982 opened_wait: false,
7983 }
7984 }
7985
7986 pub fn sleep(&self, duration: Duration) -> TimerCall {
8007 let delay_seconds = duration
8008 .as_secs()
8009 .checked_add(u64::from(duration.subsec_nanos() > 0));
8010 TimerCall {
8011 ctx: self.clone(),
8012 delay_seconds,
8013 scheduled: false,
8014 matched_pending: false,
8015 parallel_group_path: Vec::new(),
8016 }
8017 }
8018
8019 pub fn start_timer(&self, duration: Duration) -> TimerCall {
8021 self.sleep(duration)
8022 }
8023
8024 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8031 where
8032 T: Serialize + DeserializeOwned,
8033 F: FnOnce() -> T,
8034 {
8035 {
8036 let mut state = self
8037 .state
8038 .lock()
8039 .map_err(|_| Error::WorkflowStatePoisoned)?;
8040 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8041 return match recorded {
8042 RecordedCommand::SideEffect { sequence, value } => {
8043 state.command_cursor += 1;
8044 value.deserialize().map_err(|error| {
8045 Error::NonDeterministicReplay(ReplayFailure::new(
8046 "side_effect_type_mismatch",
8047 Some(sequence),
8048 Some(std::any::type_name::<T>().to_string()),
8049 Some(error.to_string()),
8050 "recorded side-effect value is incompatible with the requested Rust type",
8051 ))
8052 })
8053 }
8054 other => Err(command_mismatch(&other, "side effect")),
8055 };
8056 }
8057 }
8058
8059 let value = callback();
8060 let avro_value = AvroValue::from_serialize(&value)?;
8061 let mut state = self
8062 .state
8063 .lock()
8064 .map_err(|_| Error::WorkflowStatePoisoned)?;
8065 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8066 state.commands.push(json!({
8067 "type": "record_side_effect",
8068 "result": result,
8069 }));
8070 Ok(value)
8071 }
8072
8073 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8075 where
8076 F: FnOnce() -> AvroValue,
8077 {
8078 {
8079 let mut state = self
8080 .state
8081 .lock()
8082 .map_err(|_| Error::WorkflowStatePoisoned)?;
8083 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8084 return match recorded {
8085 RecordedCommand::SideEffect { value, .. } => {
8086 state.command_cursor += 1;
8087 Ok(value)
8088 }
8089 other => Err(command_mismatch(&other, "side effect")),
8090 };
8091 }
8092 }
8093
8094 let value = callback();
8095 let mut state = self
8096 .state
8097 .lock()
8098 .map_err(|_| Error::WorkflowStatePoisoned)?;
8099 let result = encode_typed_envelope(&value, &state.payload_codec)?;
8100 state.commands.push(json!({
8101 "type": "record_side_effect",
8102 "result": result,
8103 }));
8104 Ok(value)
8105 }
8106
8107 pub fn append_workflow_stream(
8113 &self,
8114 stream_name: impl Into<String>,
8115 items: &[WorkflowStreamAppendItem],
8116 max_pending_items: Option<u64>,
8117 ) -> Result<()> {
8118 if items.is_empty() {
8119 return Err(Error::Codec(
8120 "workflow_stream_items_empty: append requires at least one item".to_string(),
8121 ));
8122 }
8123 if max_pending_items == Some(0) {
8124 return Err(Error::Codec(
8125 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8126 .to_string(),
8127 ));
8128 }
8129 let stream_name = stream_name.into();
8130 if stream_name.is_empty() {
8131 return Err(Error::Codec(
8132 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8133 ));
8134 }
8135
8136 let mut state = self
8137 .state
8138 .lock()
8139 .map_err(|_| Error::WorkflowStatePoisoned)?;
8140 let command_ordinal = state.workflow_stream_command_counter;
8141 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8142 state.workflow_stream_command_counter += 1;
8143 return match recorded {
8144 RecordedCommand::SideEffect { .. } => {
8145 state.command_cursor += 1;
8146 Ok(())
8147 }
8148 other => Err(command_mismatch(&other, "workflow stream append")),
8149 };
8150 }
8151
8152 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8153 state.workflow_stream_command_counter += 1;
8154 let wire_items = items
8155 .iter()
8156 .enumerate()
8157 .map(|(item_index, item)| {
8158 item.wire_value(Some(format!(
8159 "dw-stream:{identity}:{command_ordinal}:{item_index}"
8160 )))
8161 })
8162 .collect::<Vec<_>>();
8163 let mut directive = json!({
8164 "operation": "append",
8165 "stream_name": stream_name,
8166 "command_identity": identity,
8167 "command_ordinal": command_ordinal,
8168 "items": wire_items,
8169 });
8170 if let Some(max_pending_items) = max_pending_items {
8171 directive["max_pending_items"] = json!(max_pending_items);
8172 }
8173 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8174 state.commands.push(json!({
8175 "type": "record_side_effect",
8176 "result": result,
8177 "workflow_stream": directive,
8178 }));
8179 Ok(())
8180 }
8181
8182 pub fn close_workflow_stream(
8184 &self,
8185 stream_name: impl Into<String>,
8186 retention_seconds: Option<u64>,
8187 ) -> Result<()> {
8188 self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8189 }
8190
8191 pub fn error_workflow_stream(
8193 &self,
8194 stream_name: impl Into<String>,
8195 error_reason: impl Into<String>,
8196 retention_seconds: Option<u64>,
8197 ) -> Result<()> {
8198 let error_reason = error_reason.into();
8199 if error_reason.is_empty() {
8200 return Err(Error::Codec(
8201 "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8202 ));
8203 }
8204 self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8205 }
8206
8207 fn finish_workflow_stream(
8208 &self,
8209 stream_name: String,
8210 error_reason: Option<String>,
8211 retention_seconds: Option<u64>,
8212 ) -> Result<()> {
8213 if stream_name.is_empty() {
8214 return Err(Error::Codec(
8215 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8216 ));
8217 }
8218 if retention_seconds == Some(0) {
8219 return Err(Error::Codec(
8220 "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8221 ));
8222 }
8223 let mut state = self
8224 .state
8225 .lock()
8226 .map_err(|_| Error::WorkflowStatePoisoned)?;
8227 let command_ordinal = state.workflow_stream_command_counter;
8228 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8229 state.workflow_stream_command_counter += 1;
8230 return match recorded {
8231 RecordedCommand::SideEffect { .. } => {
8232 state.command_cursor += 1;
8233 Ok(())
8234 }
8235 other => Err(command_mismatch(&other, "workflow stream close")),
8236 };
8237 }
8238 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8239 state.workflow_stream_command_counter += 1;
8240 let mut directive = json!({
8241 "operation": if error_reason.is_some() { "error" } else { "close" },
8242 "stream_name": stream_name,
8243 "command_identity": identity,
8244 "command_ordinal": command_ordinal,
8245 });
8246 if let Some(error_reason) = error_reason {
8247 directive["error_reason"] = json!(error_reason);
8248 }
8249 if let Some(retention_seconds) = retention_seconds {
8250 directive["retention_seconds"] = json!(retention_seconds);
8251 }
8252 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8253 state.commands.push(json!({
8254 "type": "record_side_effect",
8255 "result": result,
8256 "workflow_stream": directive,
8257 }));
8258 Ok(())
8259 }
8260
8261 fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8262 let identity = state.workflow_command_identity.as_str();
8263 if identity.is_empty() {
8264 return Err(Error::MissingWorkflowCommandIdentity);
8265 }
8266 Ok(identity)
8267 }
8268
8269 pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8275 update.validate()?;
8276 let (attributes, attribute_types) = update.into_wire_parts();
8277 let mut state = self
8278 .state
8279 .lock()
8280 .map_err(|_| Error::WorkflowStatePoisoned)?;
8281
8282 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8283 return match recorded {
8284 RecordedCommand::SearchAttributes {
8285 sequence,
8286 attributes: recorded_attributes,
8287 attribute_types: recorded_attribute_types,
8288 } => {
8289 if recorded_attributes != attributes {
8290 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8291 "search_attribute_value_mismatch",
8292 Some(sequence),
8293 Some(recorded_attributes.to_string()),
8294 Some(attributes.to_string()),
8295 "search-attribute values differ from the recorded durable command",
8296 )));
8297 }
8298 if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8299 if recorded_types != attribute_types {
8300 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8301 "search_attribute_type_mismatch",
8302 Some(sequence),
8303 Some(json!(recorded_types).to_string()),
8304 Some(json!(attribute_types).to_string()),
8305 "search-attribute declared types differ from the recorded durable command",
8306 )));
8307 }
8308 }
8309 state.command_cursor += 1;
8310 Ok(())
8311 }
8312 other => Err(command_mismatch(&other, "search-attribute update")),
8313 };
8314 }
8315
8316 let mut command = serde_json::Map::from_iter([
8317 ("type".to_string(), json!("upsert_search_attributes")),
8318 ("attributes".to_string(), attributes),
8319 ]);
8320 if !attribute_types.is_empty() {
8321 command.insert("attribute_types".to_string(), json!(attribute_types));
8322 }
8323 state.commands.push(Value::Object(command));
8324 Ok(())
8325 }
8326
8327 pub fn uuid_v4(&self) -> Result<Uuid> {
8329 self.side_effect(Uuid::new_v4)
8330 }
8331
8332 pub fn get_version(
8335 &self,
8336 change_id: impl Into<String>,
8337 min_supported: i32,
8338 max_supported: i32,
8339 ) -> Result<i32> {
8340 let change_id = change_id.into();
8341 if change_id.trim().is_empty() {
8342 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8343 "version_change_id_invalid",
8344 None,
8345 Some("non-empty change ID".to_string()),
8346 Some(change_id),
8347 "version markers require a stable non-empty change ID",
8348 )));
8349 }
8350 if min_supported > max_supported {
8351 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8352 "version_range_invalid",
8353 None,
8354 Some("min_supported <= max_supported".to_string()),
8355 Some(format!("{min_supported}..={max_supported}")),
8356 "version marker supported range is invalid",
8357 )));
8358 }
8359
8360 let mut state = self
8361 .state
8362 .lock()
8363 .map_err(|_| Error::WorkflowStatePoisoned)?;
8364 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8365 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8366 return Ok(version);
8367 }
8368
8369 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8370 return match recorded {
8371 RecordedCommand::VersionMarker {
8372 sequence,
8373 change_id: recorded_change_id,
8374 version,
8375 ..
8376 } => {
8377 if recorded_change_id != change_id {
8378 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8379 "version_change_id_mismatch",
8380 Some(sequence),
8381 Some(recorded_change_id),
8382 Some(change_id),
8383 "recorded version marker change ID differs from current workflow code",
8384 )));
8385 }
8386 ensure_version_supported(
8387 &change_id,
8388 version,
8389 min_supported,
8390 max_supported,
8391 sequence,
8392 )?;
8393 state.command_cursor += 1;
8394 state.version_markers.insert(change_id, (version, sequence));
8395 Ok(version)
8396 }
8397 other => Err(command_mismatch(
8398 &other,
8399 format!("version marker:{change_id}"),
8400 )),
8401 };
8402 }
8403
8404 let version = max_supported;
8405 state.commands.push(json!({
8406 "type": "record_version_marker",
8407 "change_id": change_id,
8408 "version": version,
8409 "min_supported": min_supported,
8410 "max_supported": max_supported,
8411 }));
8412 state.version_markers.insert(change_id, (version, 0));
8415 Ok(version)
8416 }
8417
8418 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8420 Ok(self.get_version(change_id, -1, 1)? == 1)
8421 }
8422
8423 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8425 self.get_version(change_id, -1, 1).map(|_| ())
8426 }
8427
8428 pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8433 let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8434 let mut state = self
8435 .state
8436 .lock()
8437 .map_err(|_| Error::WorkflowStatePoisoned)?;
8438
8439 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8440 return match recorded {
8441 RecordedCommand::Memo {
8442 sequence,
8443 entries: recorded_entries,
8444 } => {
8445 if recorded_entries != entries {
8446 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8447 "memo_update_mismatch",
8448 Some(sequence),
8449 Some(format!("{recorded_entries:?}")),
8450 Some(format!("{entries:?}")),
8451 "recorded memo entries differ from the current workflow update",
8452 )));
8453 }
8454 state.command_cursor += 1;
8455 Ok(())
8456 }
8457 other => Err(command_mismatch(&other, "memo upsert")),
8458 };
8459 }
8460
8461 let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8462 state.commands.push(json!({
8463 "type": "upsert_memo",
8464 "entries": entries_envelope,
8465 }));
8466 Ok(())
8467 }
8468
8469 pub fn start_child_workflow<T: Serialize>(
8496 &self,
8497 workflow_type: impl Into<String>,
8498 options: ChildWorkflowOptions,
8499 args: T,
8500 ) -> ChildWorkflowCall {
8501 ChildWorkflowCall {
8502 ctx: self.clone(),
8503 workflow_type: workflow_type.into(),
8504 options,
8505 args: Some(AvroValue::from_serialize(&args)),
8506 scheduled: false,
8507 matched_pending: false,
8508 parallel_group_path: Vec::new(),
8509 }
8510 }
8511
8512 pub async fn start_child_workflow_avro_value<T: Serialize>(
8513 &self,
8514 workflow_type: impl Into<String>,
8515 options: ChildWorkflowOptions,
8516 args: T,
8517 ) -> Result<ChildWorkflowAvroResult> {
8518 let mut call = self.start_child_workflow(workflow_type, options, args);
8519 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8520 }
8521
8522 fn take_commands(&self) -> Result<Vec<Value>> {
8523 let mut state = self
8524 .state
8525 .lock()
8526 .map_err(|_| Error::WorkflowStatePoisoned)?;
8527 Ok(std::mem::take(&mut state.commands))
8528 }
8529
8530 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8531 let mut state = self
8532 .state
8533 .lock()
8534 .map_err(|_| Error::WorkflowStatePoisoned)?;
8535
8536 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8537 return Err(command_mismatch(&recorded, "continue as new"));
8538 }
8539 if state.recorded_continue_as_new_sequence.is_some() {
8540 state.continue_as_new_consumed = true;
8541 return Ok(None);
8542 }
8543
8544 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8545 let mut command = serde_json::Map::from_iter([
8546 ("type".to_string(), json!("continue_as_new")),
8547 ("arguments".to_string(), arguments),
8548 ("queue".to_string(), json!(state.task_queue.clone())),
8549 ]);
8550 if let Some(workflow_type) = request.options.workflow_type {
8551 command.insert("workflow_type".to_string(), json!(workflow_type));
8552 }
8553 if let Some(task_queue) = request.options.task_queue {
8554 command.insert("queue".to_string(), json!(task_queue));
8555 }
8556 Ok(Some(Value::Object(command)))
8557 }
8558
8559 fn matched_recorded_pending(&self) -> Result<bool> {
8560 let state = self
8561 .state
8562 .lock()
8563 .map_err(|_| Error::WorkflowStatePoisoned)?;
8564 Ok(state.matched_recorded_pending)
8565 }
8566
8567 fn ensure_history_consumed(&self) -> Result<()> {
8568 let state = self
8569 .state
8570 .lock()
8571 .map_err(|_| Error::WorkflowStatePoisoned)?;
8572 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8573 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8574 "recorded_commands_unconsumed",
8575 Some(command.sequence()),
8576 Some(command.shape().to_string()),
8577 Some("workflow completion".to_string()),
8578 "workflow completed before consuming all recorded durable commands",
8579 )));
8580 }
8581 if let Some(sequence) = state
8582 .recorded_continue_as_new_sequence
8583 .filter(|_| !state.continue_as_new_consumed)
8584 {
8585 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8586 "recorded_continue_as_new_unconsumed",
8587 Some(sequence),
8588 Some("continue as new".to_string()),
8589 Some("workflow completion".to_string()),
8590 "workflow completed without consuming its recorded continue-as-new transition",
8591 )));
8592 }
8593 Ok(())
8594 }
8595}
8596
8597fn contiguous_message_stream_count(
8598 pending: &[MessageStreamMessage],
8599 cursor: u64,
8600 max_items: usize,
8601) -> usize {
8602 pending
8603 .iter()
8604 .take(max_items)
8605 .enumerate()
8606 .take_while(|(offset, message)| {
8607 u64::try_from(*offset)
8608 .ok()
8609 .and_then(|offset| cursor.checked_add(offset + 1))
8610 == Some(message.position)
8611 })
8612 .count()
8613}
8614
8615fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
8616 matches!(
8617 event.event_type.as_str(),
8618 "ActivityScheduled"
8619 | "TimerScheduled"
8620 | "ChildWorkflowScheduled"
8621 | "SignalWaitOpened"
8622 | "ConditionWaitOpened"
8623 | "SearchAttributesUpserted"
8624 | "SideEffectRecorded"
8625 | "VersionMarkerRecorded"
8626 | "MemoUpserted"
8627 | "WorkflowContinuedAsNew"
8628 )
8629}
8630
8631#[derive(Debug)]
8632struct WorkflowState {
8633 workflow_id: Option<String>,
8634 run_id: Option<String>,
8635 task_queue: String,
8636 payload_codec: String,
8637 history_events: Arc<Vec<HistoryEvent>>,
8638 history_budget: WorkflowHistoryBudget,
8639 cancel_requested: bool,
8640 resume_signal: Option<ResumeSignal>,
8641 recorded_commands: Vec<RecordedCommand>,
8642 recorded_continue_as_new_sequence: Option<u64>,
8643 continue_as_new_consumed: bool,
8644 command_cursor: usize,
8645 condition_wait_occurrence_counter: u64,
8646 matched_recorded_pending: bool,
8647 version_markers: HashMap<String, (i32, u64)>,
8648 workflow_command_identity: String,
8649 workflow_stream_command_counter: u64,
8650 commands: Vec<Value>,
8651 message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
8652 message_stream_cursors: HashMap<String, u64>,
8653 message_stream_waits: HashMap<String, u64>,
8654}
8655
8656impl WorkflowState {
8657 #[cfg(test)]
8658 fn new(
8659 history: Vec<HistoryEvent>,
8660 task_queue: String,
8661 payload_codec: String,
8662 resume_signal: Option<ResumeSignal>,
8663 ) -> Result<Self> {
8664 Self::new_with_identity(
8665 history,
8666 None,
8667 None,
8668 task_queue,
8669 payload_codec,
8670 resume_signal,
8671 )
8672 }
8673
8674 fn new_with_identity(
8675 history: Vec<HistoryEvent>,
8676 workflow_id: Option<String>,
8677 run_id: Option<String>,
8678 task_queue: String,
8679 payload_codec: String,
8680 resume_signal: Option<ResumeSignal>,
8681 ) -> Result<Self> {
8682 let recorded_commands = recorded_commands(
8683 &history,
8684 &payload_codec,
8685 WorkflowIdentity {
8686 workflow_id: workflow_id.clone(),
8687 run_id: run_id.clone(),
8688 },
8689 )?;
8690 let recorded_continue_as_new = history
8691 .iter()
8692 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
8693 .collect::<Vec<_>>();
8694 if recorded_continue_as_new.len() > 1 {
8695 return Err(invalid_recorded_history(
8696 "duplicate_continue_as_new_transition",
8697 recorded_continue_as_new
8698 .last()
8699 .and_then(|event| durable_event_sequence(event))
8700 .unwrap_or(0),
8701 "one WorkflowContinuedAsNew event",
8702 &format!(
8703 "{} WorkflowContinuedAsNew events",
8704 recorded_continue_as_new.len()
8705 ),
8706 "workflow history records one continue-as-new transition more than once",
8707 ));
8708 }
8709 let recorded_continue_as_new_sequence = recorded_continue_as_new
8710 .first()
8711 .map(|event| {
8712 durable_event_sequence(event).ok_or_else(|| {
8713 Error::NonDeterministicReplay(ReplayFailure::new(
8714 "continue_as_new_sequence_missing",
8715 None,
8716 Some("recorded transition sequence".to_string()),
8717 Some("missing sequence".to_string()),
8718 "WorkflowContinuedAsNew history is missing its recorded sequence",
8719 ))
8720 })
8721 })
8722 .transpose()?;
8723 let mut message_stream_cursors = HashMap::new();
8724 for event in &history {
8725 if !matches!(
8726 event.event_type.as_str(),
8727 "SignalReceived" | "SignalApplied"
8728 ) || event.payload.get("signal_name").and_then(Value::as_str)
8729 != Some(MESSAGE_STREAM_SIGNAL)
8730 {
8731 continue;
8732 }
8733 let arguments = decode_signal_event_arguments(event, &payload_codec)?;
8734 if arguments.len() != 1 {
8735 continue;
8736 }
8737 let envelope = arguments[0].clone().into_json()?;
8738 let Some(envelope) = envelope.as_object() else {
8739 continue;
8740 };
8741 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
8742 {
8743 continue;
8744 }
8745 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
8746 continue;
8747 };
8748 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
8749 else {
8750 continue;
8751 };
8752 let cursor = message_stream_cursors
8753 .entry(stream_name.to_string())
8754 .or_insert(0);
8755 *cursor = (*cursor).max(through_position);
8756 }
8757 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
8758 let cancel_requested = history.iter().any(|event| {
8759 matches!(
8760 event.event_type.as_str(),
8761 "WorkflowCancellationRequested" | "WorkflowCancelRequested"
8762 )
8763 });
8764 Ok(Self {
8765 workflow_command_identity: String::new(),
8766 workflow_stream_command_counter: 0,
8767 workflow_id,
8768 run_id,
8769 task_queue,
8770 payload_codec,
8771 history_events: Arc::new(history),
8772 history_budget: WorkflowHistoryBudget {
8773 event_count,
8774 ..WorkflowHistoryBudget::default()
8775 },
8776 cancel_requested,
8777 resume_signal,
8778 recorded_commands,
8779 recorded_continue_as_new_sequence,
8780 continue_as_new_consumed: false,
8781 command_cursor: 0,
8782 condition_wait_occurrence_counter: 0,
8783 matched_recorded_pending: false,
8784 version_markers: HashMap::new(),
8785 commands: Vec::new(),
8786 message_stream_messages: HashMap::new(),
8787 message_stream_cursors,
8788 message_stream_waits: HashMap::new(),
8789 })
8790 }
8791}
8792
8793enum MessageStreamDelivery {
8794 Message(MessageStreamMessage),
8795 Cursor {
8796 stream_name: String,
8797 through_position: u64,
8798 },
8799}
8800
8801fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
8802 if arguments.len() != 1 {
8803 return Ok(None);
8804 }
8805 let envelope = arguments
8806 .into_iter()
8807 .next()
8808 .expect("one argument was checked");
8809 let Some(envelope) = envelope.as_object() else {
8810 return Ok(None);
8811 };
8812 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
8813 return Ok(None);
8814 };
8815 if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
8816 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
8817 else {
8818 return Ok(None);
8819 };
8820 return Ok(Some(MessageStreamDelivery::Cursor {
8821 stream_name: stream_name.to_string(),
8822 through_position,
8823 }));
8824 }
8825 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
8826 return Ok(None);
8827 }
8828 let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
8829 return Ok(None);
8830 };
8831 let Some(position) = envelope
8832 .get("position")
8833 .and_then(Value::as_u64)
8834 .filter(|value| *value > 0)
8835 else {
8836 return Ok(None);
8837 };
8838 let Some(payload_envelope) = envelope.get("payload_envelope") else {
8839 return Ok(None);
8840 };
8841 let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
8842 else {
8843 return Ok(None);
8844 };
8845 let decoded = decode_avro_value(&payload_envelope)?;
8846 let AvroValue::Array(values) = decoded else {
8847 return Ok(None);
8848 };
8849 Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
8850 stream_name: stream_name.to_string(),
8851 message_id: message_id.to_string(),
8852 position,
8853 arguments: values,
8854 })))
8855}
8856
8857#[derive(Clone, Debug)]
8858enum RecordedCommand {
8859 Activity {
8860 sequence: u64,
8861 activity_type: Option<String>,
8862 options: Option<RecordedActivityOptions>,
8863 outcome: Option<ActivityOutcome>,
8864 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
8865 },
8866 Timer {
8867 sequence: u64,
8868 delay_seconds: u64,
8869 fired: bool,
8870 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
8871 },
8872 ChildWorkflow {
8873 sequence: u64,
8874 workflow_type: Option<String>,
8875 outcome: Option<ChildWorkflowOutcome>,
8876 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
8877 },
8878 SignalWait {
8879 sequence: u64,
8880 signal_name: String,
8881 value: Option<Vec<AvroValue>>,
8882 },
8883 ConditionWait {
8884 sequence: u64,
8885 occurrence_id: String,
8886 condition_key: Option<String>,
8887 predicate_identity: String,
8888 timeout_seconds: Option<u64>,
8889 result: Option<ConditionWaitResult>,
8890 },
8891 SearchAttributes {
8892 sequence: u64,
8893 attributes: Value,
8894 attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
8895 },
8896 SideEffect {
8897 sequence: u64,
8898 value: AvroValue,
8899 },
8900 VersionMarker {
8901 sequence: u64,
8902 change_id: String,
8903 version: i32,
8904 },
8905 Memo {
8906 sequence: u64,
8907 entries: AvroValue,
8908 },
8909}
8910
8911#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
8912struct RecordedActivityOptions {
8913 task_queue: RecordedSnapshotValue<Option<String>>,
8914 execution_mode: RecordedSnapshotValue<Option<String>>,
8915 retry_policy: ActivityRetrySnapshot,
8916}
8917
8918#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
8919enum RecordedSnapshotValue<T> {
8920 Unknown,
8922 Known(T),
8923}
8924
8925impl<T: PartialEq> RecordedSnapshotValue<T> {
8926 fn matches_current(&self, current: &Self) -> bool {
8927 match self {
8928 Self::Unknown => true,
8929 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
8930 }
8931 }
8932}
8933
8934#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
8935struct ActivityRetrySnapshot {
8936 snapshot_version: RecordedSnapshotValue<Option<u64>>,
8937 max_attempts: RecordedSnapshotValue<Option<u64>>,
8938 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
8939 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
8940 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
8941 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
8942 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
8943 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
8944}
8945
8946impl ActivityRetrySnapshot {
8947 fn matches_current(&self, current: &Self) -> bool {
8948 self.snapshot_version
8949 .matches_current(¤t.snapshot_version)
8950 && self.max_attempts.matches_current(¤t.max_attempts)
8951 && self
8952 .backoff_seconds
8953 .matches_current(¤t.backoff_seconds)
8954 && self
8955 .start_to_close_timeout
8956 .matches_current(¤t.start_to_close_timeout)
8957 && self
8958 .schedule_to_start_timeout
8959 .matches_current(¤t.schedule_to_start_timeout)
8960 && self
8961 .schedule_to_close_timeout
8962 .matches_current(¤t.schedule_to_close_timeout)
8963 && self
8964 .heartbeat_timeout
8965 .matches_current(¤t.heartbeat_timeout)
8966 && self
8967 .non_retryable_error_types
8968 .matches_current(¤t.non_retryable_error_types)
8969 }
8970}
8971
8972fn recorded_optional_u64(
8973 object: Option<&serde_json::Map<String, Value>>,
8974 field: &str,
8975) -> RecordedSnapshotValue<Option<u64>> {
8976 match object.and_then(|object| object.get(field)) {
8977 None => RecordedSnapshotValue::Unknown,
8978 Some(Value::Null) => RecordedSnapshotValue::Known(None),
8979 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
8980 }
8981}
8982
8983fn recorded_optional_string(
8984 object: &serde_json::Map<String, Value>,
8985 field: &str,
8986) -> RecordedSnapshotValue<Option<String>> {
8987 match object.get(field) {
8988 None => RecordedSnapshotValue::Unknown,
8989 Some(Value::Null) => RecordedSnapshotValue::Known(None),
8990 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
8991 }
8992}
8993
8994fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
8995 let policy = policy.and_then(Value::as_object);
8996 let backoff_seconds = policy
8997 .and_then(|policy| policy.get("backoff_seconds"))
8998 .and_then(Value::as_array)
8999 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9000 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9001 let mut non_retryable_error_types = Vec::new();
9002 for error_type in policy
9003 .and_then(|policy| policy.get("non_retryable_error_types"))
9004 .and_then(Value::as_array)
9005 .into_iter()
9006 .flatten()
9007 .filter_map(Value::as_str)
9008 .map(str::trim)
9009 .filter(|error_type| !error_type.is_empty())
9010 {
9011 if !non_retryable_error_types
9012 .iter()
9013 .any(|recorded| recorded == error_type)
9014 {
9015 non_retryable_error_types.push(error_type.to_string());
9016 }
9017 }
9018
9019 ActivityRetrySnapshot {
9020 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9021 max_attempts: recorded_optional_u64(policy, "max_attempts"),
9022 backoff_seconds,
9023 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9024 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9025 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9026 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9027 non_retryable_error_types: if policy
9028 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9029 {
9030 RecordedSnapshotValue::Known(non_retryable_error_types)
9031 } else {
9032 RecordedSnapshotValue::Unknown
9033 },
9034 }
9035}
9036
9037fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9038 let policy = options.retry_policy.as_ref();
9039 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9040 Some(Value::Null) => None,
9041 Some(value) => value_as_u64(value),
9042 None => Some(1),
9043 };
9044 let backoff_seconds = policy
9045 .and_then(|policy| policy.get("backoff_seconds"))
9046 .and_then(Value::as_array)
9047 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9048 .unwrap_or_default();
9049 let non_retryable_error_types = policy
9050 .and_then(|policy| policy.get("non_retryable_error_types"))
9051 .and_then(Value::as_array)
9052 .into_iter()
9053 .flatten()
9054 .filter_map(Value::as_str)
9055 .map(str::to_string)
9056 .collect();
9057
9058 ActivityRetrySnapshot {
9059 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9060 max_attempts: RecordedSnapshotValue::Known(max_attempts),
9061 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9062 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9063 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9064 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9065 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9066 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9067 }
9068}
9069
9070fn activity_options_description(options: &RecordedActivityOptions) -> String {
9071 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9072}
9073
9074impl RecordedCommand {
9075 fn sequence(&self) -> u64 {
9076 match self {
9077 Self::Activity { sequence, .. }
9078 | Self::Timer { sequence, .. }
9079 | Self::ChildWorkflow { sequence, .. }
9080 | Self::SignalWait { sequence, .. }
9081 | Self::ConditionWait { sequence, .. }
9082 | Self::SearchAttributes { sequence, .. }
9083 | Self::SideEffect { sequence, .. }
9084 | Self::VersionMarker { sequence, .. }
9085 | Self::Memo { sequence, .. } => *sequence,
9086 }
9087 }
9088
9089 fn shape(&self) -> &'static str {
9090 match self {
9091 Self::Activity { .. } => "activity",
9092 Self::Timer { .. } => "timer",
9093 Self::ChildWorkflow { .. } => "child workflow",
9094 Self::SignalWait { .. } => "signal wait",
9095 Self::ConditionWait { .. } => "condition wait",
9096 Self::SearchAttributes { .. } => "search-attribute update",
9097 Self::SideEffect { .. } => "side effect",
9098 Self::VersionMarker { .. } => "version marker",
9099 Self::Memo { .. } => "memo upsert",
9100 }
9101 }
9102}
9103
9104fn ensure_version_supported(
9105 change_id: &str,
9106 version: i32,
9107 min_supported: i32,
9108 max_supported: i32,
9109 sequence: u64,
9110) -> Result<()> {
9111 if (min_supported..=max_supported).contains(&version) {
9112 return Ok(());
9113 }
9114 Err(Error::NonDeterministicReplay(ReplayFailure::new(
9115 "version_marker_incompatible_range",
9116 (sequence != 0).then_some(sequence),
9117 Some(format!("{min_supported}..={max_supported}")),
9118 Some(format!("{change_id}:{version}")),
9119 "recorded workflow version is outside the range supported by current code",
9120 )))
9121}
9122
9123#[derive(Clone, Debug)]
9124struct ResumeSignal {
9125 signal_name: String,
9126 arguments: Vec<AvroValue>,
9127}
9128
9129const MAX_PARALLEL_OPERATIONS: usize = 1000;
9130
9131fn parallel_group_prefix(kind: &str) -> &'static str {
9132 match kind {
9133 "activity" => "parallel-activities",
9134 "child" => "parallel-children",
9135 "timer" => "parallel-timers",
9136 _ => "parallel-calls",
9137 }
9138}
9139
9140fn parallel_group_entry(
9141 base_sequence: u64,
9142 size: usize,
9143 index: usize,
9144 kind: &str,
9145) -> ParallelGroupMetadata {
9146 ParallelGroupMetadata {
9147 parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9148 parallel_group_kind: kind.to_string(),
9149 parallel_group_base_sequence: base_sequence,
9150 parallel_group_size: size,
9151 parallel_group_index: index,
9152 }
9153}
9154
9155fn apply_parallel_group_path(
9156 command: &mut serde_json::Map<String, Value>,
9157 path: &[ParallelGroupMetadata],
9158) {
9159 let Some(inner) = path.last() else {
9160 return;
9161 };
9162 command.insert(
9163 "parallel_group_id".to_string(),
9164 json!(inner.parallel_group_id),
9165 );
9166 command.insert(
9167 "parallel_group_kind".to_string(),
9168 json!(inner.parallel_group_kind),
9169 );
9170 command.insert(
9171 "parallel_group_base_sequence".to_string(),
9172 json!(inner.parallel_group_base_sequence),
9173 );
9174 command.insert(
9175 "parallel_group_size".to_string(),
9176 json!(inner.parallel_group_size),
9177 );
9178 command.insert(
9179 "parallel_group_index".to_string(),
9180 json!(inner.parallel_group_index),
9181 );
9182 command.insert("parallel_group_path".to_string(), json!(path));
9183}
9184
9185fn ensure_parallel_path_matches(
9186 sequence: u64,
9187 recorded: Option<&[ParallelGroupMetadata]>,
9188 expected: &[ParallelGroupMetadata],
9189) -> Result<()> {
9190 match (recorded, expected.is_empty()) {
9191 (None, true) => Ok(()),
9192 (Some(recorded), false) if recorded == expected => Ok(()),
9193 (None, false) => Err(invalid_recorded_history(
9194 "parallel_group_metadata_missing",
9195 sequence,
9196 &serde_json::to_string(expected).unwrap_or_default(),
9197 "<missing>",
9198 "recorded parallel member is missing its durable group path",
9199 )),
9200 (Some(recorded), true) => Err(invalid_recorded_history(
9201 "parallel_group_shape_mismatch",
9202 sequence,
9203 "sequential command",
9204 &serde_json::to_string(recorded).unwrap_or_default(),
9205 "recorded command belonged to a parallel group but current code schedules it sequentially",
9206 )),
9207 (Some(recorded), false) => Err(invalid_recorded_history(
9208 "parallel_group_shape_mismatch",
9209 sequence,
9210 &serde_json::to_string(recorded).unwrap_or_default(),
9211 &serde_json::to_string(expected).unwrap_or_default(),
9212 "recorded parallel-group identity or path changed during replay",
9213 )),
9214 }
9215}
9216
9217#[derive(Clone, Debug)]
9218enum ParallelShape {
9219 Leaf,
9220 Group(Vec<ParallelShape>),
9221}
9222
9223struct ParallelDescriptor {
9224 operation: ParallelOperation,
9225 offset: usize,
9226 member_path: Vec<usize>,
9227 group_path: Vec<ParallelGroupMetadata>,
9228}
9229
9230fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
9231 operations
9232 .iter()
9233 .map(|operation| match operation {
9234 ParallelOperation::Group(children) => parallel_leaf_count(children),
9235 _ => 1,
9236 })
9237 .sum()
9238}
9239
9240fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
9241 match operation {
9242 ParallelOperation::Activity { .. } => Some("activity"),
9243 ParallelOperation::ChildWorkflow { .. } => Some("child"),
9244 ParallelOperation::Timer(_) => Some("timer"),
9245 ParallelOperation::Group(children) => parallel_group_kind(children),
9246 }
9247}
9248
9249fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
9250 let mut kind = None;
9251 for operation in operations {
9252 let Some(operation_kind) = parallel_operation_kind(operation) else {
9253 continue;
9254 };
9255 match kind {
9256 None => kind = Some(operation_kind),
9257 Some(current) if current == operation_kind => {}
9258 Some(_) => return Some("mixed"),
9259 }
9260 }
9261 kind
9262}
9263
9264fn validate_parallel_operations(
9265 operations: &[ParallelOperation],
9266 member_path: &mut Vec<usize>,
9267 root: bool,
9268) -> Result<()> {
9269 let leaves = parallel_leaf_count(operations);
9270 if leaves > MAX_PARALLEL_OPERATIONS {
9271 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9272 reason: "fan_out_limit_exceeded",
9273 member_path: member_path.clone(),
9274 message: format!(
9275 "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
9276 ),
9277 }));
9278 }
9279 if !root && operations.is_empty() {
9280 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9281 reason: "nested_group_empty",
9282 member_path: member_path.clone(),
9283 message: "a nested group must contain at least one durable leaf".to_string(),
9284 }));
9285 }
9286
9287 for (index, operation) in operations.iter().enumerate() {
9288 member_path.push(index);
9289 match operation {
9290 ParallelOperation::Activity {
9291 options, arguments, ..
9292 } => {
9293 options
9294 .validate()
9295 .map_err(|error| Error::InvalidActivityOptions(error))?;
9296 if let Err(error) = arguments {
9297 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9298 reason: "arguments_invalid",
9299 member_path: member_path.clone(),
9300 message: error.to_string(),
9301 }));
9302 }
9303 }
9304 ParallelOperation::ChildWorkflow {
9305 options, arguments, ..
9306 } => {
9307 validate_parallel_child_options(options)?;
9308 if let Err(error) = arguments {
9309 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9310 reason: "arguments_invalid",
9311 member_path: member_path.clone(),
9312 message: error.to_string(),
9313 }));
9314 }
9315 }
9316 ParallelOperation::Timer(duration)
9317 if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
9318 {
9319 return Err(Error::TimerDurationOverflow);
9320 }
9321 ParallelOperation::Timer(_) => {}
9322 ParallelOperation::Group(children) => {
9323 validate_parallel_operations(children, member_path, false)?;
9324 }
9325 }
9326 member_path.pop();
9327 }
9328 Ok(())
9329}
9330
9331fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
9332 if options.task_queue.trim().is_empty() {
9333 return Err(Error::InvalidChildWorkflowOptions(
9334 "task_queue must not be empty".to_string(),
9335 ));
9336 }
9337 for (name, value) in [
9338 (
9339 "execution_timeout_seconds",
9340 options.execution_timeout_seconds,
9341 ),
9342 ("run_timeout_seconds", options.run_timeout_seconds),
9343 ] {
9344 if value == Some(0) {
9345 return Err(Error::InvalidChildWorkflowOptions(format!(
9346 "{name} must be at least 1"
9347 )));
9348 }
9349 }
9350 if options
9351 .retry_policy
9352 .as_ref()
9353 .is_some_and(|policy| policy.max_attempts == Some(0))
9354 {
9355 return Err(Error::InvalidChildWorkflowOptions(
9356 "retry_policy.max_attempts must be at least 1".to_string(),
9357 ));
9358 }
9359 Ok(())
9360}
9361
9362fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
9363 ParallelShape::Group(
9364 operations
9365 .iter()
9366 .map(|operation| match operation {
9367 ParallelOperation::Group(children) => parallel_shape(children),
9368 _ => ParallelShape::Leaf,
9369 })
9370 .collect(),
9371 )
9372}
9373
9374fn parallel_descriptors(
9375 operations: Vec<ParallelOperation>,
9376 base_sequence: u64,
9377) -> Result<Vec<ParallelDescriptor>> {
9378 let size = parallel_leaf_count(&operations);
9379 let kind = parallel_group_kind(&operations).unwrap_or("activity");
9380 let mut descriptors = Vec::with_capacity(size);
9381 let mut cursor = 0;
9382
9383 for (index, operation) in operations.into_iter().enumerate() {
9384 match operation {
9385 ParallelOperation::Group(children) => {
9386 let child_base = base_sequence
9387 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
9388 .ok_or(Error::TimerDurationOverflow)?;
9389 for mut descriptor in parallel_descriptors(children, child_base)? {
9390 let outer_index = cursor + descriptor.offset;
9391 descriptor.group_path.insert(
9392 0,
9393 parallel_group_entry(base_sequence, size, outer_index, kind),
9394 );
9395 descriptor.member_path.insert(0, index);
9396 descriptor.offset = outer_index;
9397 descriptors.push(descriptor);
9398 }
9399 cursor = descriptors.len();
9400 }
9401 operation => {
9402 descriptors.push(ParallelDescriptor {
9403 operation,
9404 offset: cursor,
9405 member_path: vec![index],
9406 group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
9407 });
9408 cursor += 1;
9409 }
9410 }
9411 }
9412 Ok(descriptors)
9413}
9414
9415enum ParallelLeafCall {
9416 Activity(ActivityCall),
9417 ChildWorkflow(ChildWorkflowCall),
9418 Timer(TimerCall),
9419}
9420
9421impl ParallelLeafCall {
9422 fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
9423 match self {
9424 Self::Activity(call) => Pin::new(call)
9425 .poll_avro_value(cx)
9426 .map_ok(ParallelAvroResult::Activity),
9427 Self::ChildWorkflow(call) => Pin::new(call)
9428 .poll_avro_value(cx)
9429 .map_ok(ParallelAvroResult::ChildWorkflow),
9430 Self::Timer(call) => Pin::new(call)
9431 .poll(cx)
9432 .map_ok(|()| ParallelAvroResult::Timer),
9433 }
9434 }
9435}
9436
9437struct ParallelLeaf {
9438 call: ParallelLeafCall,
9439 member_path: Vec<usize>,
9440 group_path: Vec<ParallelGroupMetadata>,
9441 result: Option<ParallelAvroResult>,
9442}
9443
9444pub struct ParallelCall {
9446 ctx: WorkflowContext,
9447 operations: Option<Vec<ParallelOperation>>,
9448 shape: Option<ParallelShape>,
9449 leaves: Vec<ParallelLeaf>,
9450}
9451
9452impl ParallelCall {
9453 fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
9454 Self {
9455 ctx,
9456 operations: Some(operations),
9457 shape: None,
9458 leaves: Vec::new(),
9459 }
9460 }
9461
9462 fn initialize(&mut self) -> Result<()> {
9463 let operations = self.operations.take().unwrap_or_default();
9464 validate_parallel_operations(&operations, &mut Vec::new(), true)?;
9465 self.shape = Some(parallel_shape(&operations));
9466 if operations.is_empty() {
9467 return Ok(());
9468 }
9469
9470 let base_sequence = {
9471 let state = self
9472 .ctx
9473 .state
9474 .lock()
9475 .map_err(|_| Error::WorkflowStatePoisoned)?;
9476 if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
9477 recorded.sequence()
9478 } else {
9479 let last = state
9480 .recorded_commands
9481 .last()
9482 .map(RecordedCommand::sequence)
9483 .unwrap_or(0);
9484 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
9485 .and_then(|sequence| sequence.checked_add(1))
9486 .ok_or_else(|| {
9487 Error::InvalidParallelGroup(ParallelGroupError {
9488 reason: "sequence_overflow",
9489 member_path: Vec::new(),
9490 message: "parallel group sequence identity overflowed u64".to_string(),
9491 })
9492 })?
9493 }
9494 };
9495
9496 self.leaves = parallel_descriptors(operations, base_sequence)?
9497 .into_iter()
9498 .map(|descriptor| {
9499 let path = descriptor.group_path.clone();
9500 let call = match descriptor.operation {
9501 ParallelOperation::Activity {
9502 activity_type,
9503 options,
9504 arguments,
9505 } => ParallelLeafCall::Activity(ActivityCall {
9506 ctx: self.ctx.clone(),
9507 activity_type,
9508 options,
9509 args: Some(arguments),
9510 scheduled: false,
9511 parallel_group_path: path,
9512 }),
9513 ParallelOperation::ChildWorkflow {
9514 workflow_type,
9515 options,
9516 arguments,
9517 } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
9518 ctx: self.ctx.clone(),
9519 workflow_type,
9520 options,
9521 args: Some(arguments),
9522 scheduled: false,
9523 matched_pending: false,
9524 parallel_group_path: path,
9525 }),
9526 ParallelOperation::Timer(duration) => {
9527 let delay_seconds = duration
9528 .as_secs()
9529 .checked_add(u64::from(duration.subsec_nanos() > 0));
9530 ParallelLeafCall::Timer(TimerCall {
9531 ctx: self.ctx.clone(),
9532 delay_seconds,
9533 scheduled: false,
9534 matched_pending: false,
9535 parallel_group_path: path,
9536 })
9537 }
9538 ParallelOperation::Group(_) => {
9539 unreachable!("parallel descriptors contain only durable leaves")
9540 }
9541 };
9542 ParallelLeaf {
9543 call,
9544 member_path: descriptor.member_path,
9545 group_path: descriptor.group_path,
9546 result: None,
9547 }
9548 })
9549 .collect();
9550 Ok(())
9551 }
9552
9553 fn poll_avro_value(
9554 mut self: Pin<&mut Self>,
9555 cx: &mut TaskContext<'_>,
9556 ) -> Poll<Result<Vec<ParallelAvroResult>>> {
9557 if self.operations.is_some() {
9558 if let Err(error) = self.initialize() {
9559 return Poll::Ready(Err(error));
9560 }
9561 }
9562 if self.leaves.is_empty() {
9563 return Poll::Ready(Ok(Vec::new()));
9564 }
9565
9566 let mut failures = Vec::new();
9567 let mut pending = false;
9568 for (index, leaf) in self.leaves.iter_mut().enumerate() {
9569 if leaf.result.is_some() {
9570 continue;
9571 }
9572 match leaf.call.poll_avro_value(cx) {
9573 Poll::Ready(Ok(result)) => leaf.result = Some(result),
9574 Poll::Ready(Err(error)) => failures.push((index, error)),
9575 Poll::Pending => pending = true,
9576 }
9577 }
9578
9579 if !failures.is_empty() {
9580 if let Some(position) = failures
9581 .iter()
9582 .position(|(_, error)| workflow_task_integrity_error(error))
9583 {
9584 return Poll::Ready(Err(failures.remove(position).1));
9585 }
9586 failures.sort_by_key(|(index, _)| *index);
9587 let (failed_index, cause) = failures.remove(0);
9588 let failed = &self.leaves[failed_index];
9589 let completed = self
9590 .leaves
9591 .iter()
9592 .filter_map(|leaf| {
9593 leaf.result
9594 .clone()
9595 .and_then(|result| result.into_json_result().ok())
9596 .map(|result| ParallelCompletion {
9597 member_path: leaf.member_path.clone(),
9598 result,
9599 })
9600 })
9601 .collect();
9602 let group_id = failed
9603 .group_path
9604 .first()
9605 .map(|entry| entry.parallel_group_id.clone())
9606 .unwrap_or_default();
9607 return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
9608 group_id,
9609 member_path: failed.member_path.clone(),
9610 group_path: failed.group_path.clone(),
9611 completed,
9612 cause: Box::new(cause),
9613 })));
9614 }
9615 if pending {
9616 return Poll::Pending;
9617 }
9618
9619 let mut flat_results = self
9620 .leaves
9621 .iter_mut()
9622 .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
9623 .collect::<Vec<_>>()
9624 .into_iter();
9625 let results = parallel_results_for_shape(
9626 self.shape.as_ref().expect("initialized parallel shape"),
9627 &mut flat_results,
9628 );
9629 Poll::Ready(Ok(match results {
9630 ParallelAvroResult::Group(results) => results,
9631 ParallelAvroResult::Activity(_)
9632 | ParallelAvroResult::ChildWorkflow(_)
9633 | ParallelAvroResult::Timer => unreachable!("root parallel shape is a group"),
9634 }))
9635 }
9636}
9637
9638fn parallel_results_for_shape(
9639 shape: &ParallelShape,
9640 flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
9641) -> ParallelAvroResult {
9642 match shape {
9643 ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
9644 ParallelShape::Group(children) => ParallelAvroResult::Group(
9645 children
9646 .iter()
9647 .map(|child| parallel_results_for_shape(child, flat_results))
9648 .collect(),
9649 ),
9650 }
9651}
9652
9653impl Future for ParallelCall {
9654 type Output = Result<Vec<ParallelResult>>;
9655
9656 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9657 self.poll_avro_value(cx)
9658 .map_ok(|results| {
9659 results
9660 .into_iter()
9661 .map(ParallelAvroResult::into_json_result)
9662 .collect::<Result<Vec<_>>>()
9663 })
9664 .map_ok(|result| result)
9665 .flatten_result()
9666 }
9667}
9668
9669trait PollNestedResultExt<T> {
9670 fn flatten_result(self) -> Poll<Result<T>>;
9671}
9672
9673impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
9674 fn flatten_result(self) -> Poll<Result<T>> {
9675 match self {
9676 Poll::Ready(Ok(result)) => Poll::Ready(result),
9677 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
9678 Poll::Pending => Poll::Pending,
9679 }
9680 }
9681}
9682
9683struct SagaCompensation {
9684 activity_type: String,
9685 options: ActivityOptions,
9686 arguments: AvroValue,
9687 registration_order: usize,
9688}
9689
9690pub struct Saga {
9698 ctx: WorkflowContext,
9699 compensations: Vec<SagaCompensation>,
9700}
9701
9702impl Saga {
9703 fn new(ctx: WorkflowContext) -> Self {
9704 Self {
9705 ctx,
9706 compensations: Vec::new(),
9707 }
9708 }
9709
9710 pub fn add_compensation<T: Serialize>(
9711 &mut self,
9712 activity_type: impl Into<String>,
9713 args: T,
9714 ) -> Result<&mut Self> {
9715 self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
9716 }
9717
9718 pub fn add_compensation_with_options<T: Serialize>(
9719 &mut self,
9720 activity_type: impl Into<String>,
9721 options: ActivityOptions,
9722 args: T,
9723 ) -> Result<&mut Self> {
9724 let activity_type = activity_type.into();
9725 if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
9726 return Err(Error::Codec(
9727 "saga compensation activity type must be non-empty without surrounding whitespace"
9728 .to_string(),
9729 ));
9730 }
9731 options.validate().map_err(Error::InvalidActivityOptions)?;
9732 let arguments = AvroValue::from_serialize(&args)?;
9733 let registration_order = self.compensations.len() + 1;
9734 self.compensations.push(SagaCompensation {
9735 activity_type,
9736 options,
9737 arguments,
9738 registration_order,
9739 });
9740 Ok(self)
9741 }
9742
9743 pub async fn compensate(mut self, initiating_failure: Error) -> Error {
9745 while let Some(compensation) = self.compensations.pop() {
9746 if let Err(compensation_failure) = self
9747 .ctx
9748 .activity_with_options(
9749 compensation.activity_type.clone(),
9750 compensation.options,
9751 compensation.arguments,
9752 )
9753 .await
9754 {
9755 if workflow_task_integrity_error(&compensation_failure) {
9756 return compensation_failure;
9757 }
9758 return Error::SagaCompensationFailed(SagaCompensationFailure {
9759 initiating_failure: Box::new(initiating_failure),
9760 compensation_failure: Box::new(compensation_failure),
9761 compensation_activity_type: compensation.activity_type,
9762 compensation_registration_order: compensation.registration_order,
9763 });
9764 }
9765 }
9766 initiating_failure
9767 }
9768
9769 pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
9771 match outcome {
9772 Ok(value) => Ok(value),
9773 Err(error) => Err(self.compensate(error).await),
9774 }
9775 }
9776}
9777
9778pub struct ActivityCall {
9779 ctx: WorkflowContext,
9780 activity_type: String,
9781 options: ActivityOptions,
9782 args: Option<Result<AvroValue>>,
9783 scheduled: bool,
9784 parallel_group_path: Vec<ParallelGroupMetadata>,
9785}
9786
9787impl ActivityCall {
9788 fn poll_avro_value(
9789 mut self: Pin<&mut Self>,
9790 _cx: &mut TaskContext<'_>,
9791 ) -> Poll<Result<AvroValue>> {
9792 let ctx = self.ctx.clone();
9793 let mut state = match ctx.state.lock() {
9794 Ok(state) => state,
9795 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9796 };
9797
9798 if self.scheduled {
9799 return Poll::Pending;
9800 }
9801
9802 let options = match self.options.validate() {
9803 Ok(options) => options,
9804 Err(error) => {
9805 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
9806 }
9807 };
9808 let task_queue = options
9809 .task_queue
9810 .clone()
9811 .unwrap_or_else(|| state.task_queue.clone());
9812 let current_recorded_options = RecordedActivityOptions {
9813 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
9814 execution_mode: RecordedSnapshotValue::Known(None),
9817 retry_policy: current_activity_retry_snapshot(&options),
9818 };
9819
9820 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9821 let sequence = recorded.sequence();
9822 match recorded {
9823 RecordedCommand::Activity {
9824 activity_type,
9825 options: recorded_options,
9826 outcome,
9827 parallel_group_path,
9828 ..
9829 } => {
9830 if let Err(error) = ensure_parallel_path_matches(
9831 sequence,
9832 parallel_group_path.as_deref(),
9833 &self.parallel_group_path,
9834 ) {
9835 return Poll::Ready(Err(error));
9836 }
9837 if let Some(recorded_type) = activity_type {
9838 if recorded_type != self.activity_type {
9839 return Poll::Ready(Err(Error::NonDeterministicReplay(
9840 ReplayFailure::new(
9841 "recorded_command_detail_mismatch",
9842 Some(sequence),
9843 Some(format!("activity:{recorded_type}")),
9844 Some(format!("activity:{}", self.activity_type)),
9845 "recorded activity type differs from the current workflow command",
9846 ),
9847 )));
9848 }
9849 }
9850 if let Some(recorded_options) = recorded_options {
9851 if !recorded_options
9852 .task_queue
9853 .matches_current(¤t_recorded_options.task_queue)
9854 {
9855 return Poll::Ready(Err(Error::NonDeterministicReplay(
9856 ReplayFailure::new(
9857 "activity_task_queue_mismatch",
9858 Some(sequence),
9859 Some(activity_options_description(&recorded_options)),
9860 Some(activity_options_description(¤t_recorded_options)),
9861 "recorded activity task queue differs from the current workflow command",
9862 ),
9863 )));
9864 }
9865 if !recorded_options
9866 .execution_mode
9867 .matches_current(¤t_recorded_options.execution_mode)
9868 {
9869 return Poll::Ready(Err(Error::NonDeterministicReplay(
9870 ReplayFailure::new(
9871 "activity_execution_mode_mismatch",
9872 Some(sequence),
9873 Some(activity_options_description(&recorded_options)),
9874 Some(activity_options_description(¤t_recorded_options)),
9875 "recorded activity execution mode differs from the current workflow command",
9876 ),
9877 )));
9878 }
9879 if !recorded_options
9880 .retry_policy
9881 .matches_current(¤t_recorded_options.retry_policy)
9882 {
9883 return Poll::Ready(Err(Error::NonDeterministicReplay(
9884 ReplayFailure::new(
9885 "activity_retry_policy_mismatch",
9886 Some(sequence),
9887 Some(activity_options_description(&recorded_options)),
9888 Some(activity_options_description(¤t_recorded_options)),
9889 "recorded activity retry policy differs from the current workflow command",
9890 ),
9891 )));
9892 }
9893 }
9894 state.command_cursor += 1;
9895 if let Some(outcome) = outcome {
9896 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
9897 }
9898 state.matched_recorded_pending = true;
9899 self.scheduled = true;
9900 return Poll::Pending;
9901 }
9902 other => {
9903 return Poll::Ready(Err(command_mismatch(
9904 &other,
9905 format!("activity:{}", self.activity_type),
9906 )));
9907 }
9908 }
9909 }
9910
9911 if !self.scheduled {
9912 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
9913 Ok(args) => args,
9914 Err(error) => return Poll::Ready(Err(error)),
9915 };
9916 let arguments = normalize_avro_arguments(args);
9917 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
9918 Ok(envelope) => envelope,
9919 Err(error) => return Poll::Ready(Err(error)),
9920 };
9921
9922 let mut command = serde_json::Map::from_iter([
9923 ("type".to_string(), json!("schedule_activity")),
9924 (
9925 "activity_type".to_string(),
9926 json!(self.activity_type.clone()),
9927 ),
9928 ("queue".to_string(), json!(task_queue)),
9929 ("arguments".to_string(), envelope),
9930 ]);
9931 for (field, value) in [
9932 ("start_to_close_timeout", options.start_to_close_timeout),
9933 (
9934 "schedule_to_start_timeout",
9935 options.schedule_to_start_timeout,
9936 ),
9937 (
9938 "schedule_to_close_timeout",
9939 options.schedule_to_close_timeout,
9940 ),
9941 ("heartbeat_timeout", options.heartbeat_timeout),
9942 ] {
9943 if let Some(value) = value {
9944 command.insert(field.to_string(), json!(value));
9945 }
9946 }
9947 if let Some(retry_policy) = options.retry_policy {
9948 command.insert("retry_policy".to_string(), retry_policy);
9949 }
9950 apply_parallel_group_path(&mut command, &self.parallel_group_path);
9951 state.commands.push(Value::Object(command));
9952 self.scheduled = true;
9953 }
9954
9955 Poll::Pending
9956 }
9957}
9958
9959impl Future for ActivityCall {
9960 type Output = Result<Value>;
9961
9962 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9963 match self.poll_avro_value(cx) {
9964 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
9965 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
9966 Poll::Pending => Poll::Pending,
9967 }
9968 }
9969}
9970
9971pub struct TimerCall {
9973 ctx: WorkflowContext,
9974 delay_seconds: Option<u64>,
9975 scheduled: bool,
9976 matched_pending: bool,
9977 parallel_group_path: Vec<ParallelGroupMetadata>,
9978}
9979
9980impl Future for TimerCall {
9981 type Output = Result<()>;
9982
9983 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9984 if self.matched_pending {
9985 return Poll::Pending;
9986 }
9987
9988 let ctx = self.ctx.clone();
9989 let Some(requested_delay) = self.delay_seconds else {
9990 return Poll::Ready(Err(Error::TimerDurationOverflow));
9991 };
9992 let mut state = match ctx.state.lock() {
9993 Ok(state) => state,
9994 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9995 };
9996
9997 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9998 match recorded {
9999 RecordedCommand::Timer {
10000 sequence,
10001 delay_seconds,
10002 fired,
10003 parallel_group_path,
10004 ..
10005 } => {
10006 if let Err(error) = ensure_parallel_path_matches(
10007 sequence,
10008 parallel_group_path.as_deref(),
10009 &self.parallel_group_path,
10010 ) {
10011 return Poll::Ready(Err(error));
10012 }
10013 if delay_seconds != requested_delay {
10014 return Poll::Ready(Err(Error::NonDeterministicReplay(
10015 ReplayFailure::new(
10016 "timer_delay_mismatch",
10017 Some(sequence),
10018 Some(format!("timer:{delay_seconds}s")),
10019 Some(format!("timer:{requested_delay}s")),
10020 "recorded timer delay differs from the current workflow command",
10021 ),
10022 )));
10023 }
10024 state.command_cursor += 1;
10025 if fired {
10026 return Poll::Ready(Ok(()));
10027 }
10028 state.matched_recorded_pending = true;
10029 self.scheduled = true;
10030 self.matched_pending = true;
10031 return Poll::Pending;
10032 }
10033 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
10034 }
10035 }
10036
10037 if !self.scheduled {
10038 let mut command = serde_json::Map::from_iter([
10039 ("type".to_string(), json!("start_timer")),
10040 ("delay_seconds".to_string(), json!(requested_delay)),
10041 ]);
10042 apply_parallel_group_path(&mut command, &self.parallel_group_path);
10043 state.commands.push(Value::Object(command));
10044 self.scheduled = true;
10045 }
10046
10047 Poll::Pending
10048 }
10049}
10050
10051pub struct ConditionWaitCall {
10053 ctx: WorkflowContext,
10054 options: ConditionWaitOptions,
10055 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
10056 occurrence_id: Option<String>,
10057 opened_wait: bool,
10058}
10059
10060impl Future for ConditionWaitCall {
10061 type Output = Result<ConditionWaitResult>;
10062
10063 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10064 if self.opened_wait {
10065 return Poll::Pending;
10066 }
10067
10068 let options = match self.options.validate() {
10069 Ok(options) => options,
10070 Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
10071 };
10072 let ctx = self.ctx.clone();
10073 let occurrence_id = match self.occurrence_id.as_ref() {
10074 Some(occurrence_id) => occurrence_id.clone(),
10075 None => {
10076 let mut state = match ctx.state.lock() {
10077 Ok(state) => state,
10078 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10079 };
10080 let ordinal = state.condition_wait_occurrence_counter;
10081 state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
10082 Some(next) => next,
10083 None => {
10084 return Poll::Ready(Err(Error::WorkerLoop(
10085 "condition wait occurrence counter overflowed".to_string(),
10086 )))
10087 }
10088 };
10089 let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
10090 drop(state);
10091 self.occurrence_id = Some(occurrence_id.clone());
10092 occurrence_id
10093 }
10094 };
10095
10096 let recorded_result = {
10097 let mut state = match ctx.state.lock() {
10098 Ok(state) => state,
10099 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10100 };
10101 let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
10102 drop(state);
10103 return self.poll_new_condition(options);
10104 };
10105 if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
10106 return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
10107 }
10108
10109 let mut cursor = state.command_cursor;
10110 let mut result = None;
10111 loop {
10112 let Some(RecordedCommand::ConditionWait {
10113 sequence,
10114 occurrence_id: recorded_occurrence_id,
10115 condition_key,
10116 predicate_identity,
10117 timeout_seconds,
10118 result: recorded_result,
10119 ..
10120 }) = state.recorded_commands.get(cursor)
10121 else {
10122 break;
10123 };
10124
10125 if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
10126 break;
10127 }
10128 if let Err(error) = validate_recorded_condition_wait(
10129 *sequence,
10130 recorded_occurrence_id,
10131 condition_key.as_deref(),
10132 predicate_identity,
10133 *timeout_seconds,
10134 &occurrence_id,
10135 &options,
10136 ) {
10137 return Poll::Ready(Err(error));
10138 }
10139 if result == Some(ConditionWaitResult::TimedOut) {
10140 return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
10141 "condition_wait_reopened_after_timeout",
10142 Some(*sequence),
10143 Some("timed-out condition is terminal".to_string()),
10144 Some("another physical wait-open".to_string()),
10145 "condition history reopened one logical wait after its durable timeout",
10146 ))));
10147 }
10148 result = *recorded_result;
10149 cursor += 1;
10150 }
10151 state.command_cursor = cursor;
10152 result
10153 };
10154
10155 if let Some(result) = recorded_result {
10156 return Poll::Ready(Ok(result));
10157 }
10158
10159 self.poll_open_condition(options)
10160 }
10161}
10162
10163impl ConditionWaitCall {
10164 fn poll_new_condition(
10165 self: Pin<&mut Self>,
10166 options: ValidatedConditionWaitOptions,
10167 ) -> Poll<Result<ConditionWaitResult>> {
10168 self.poll_open_condition(options)
10169 }
10170
10171 fn poll_open_condition(
10172 mut self: Pin<&mut Self>,
10173 options: ValidatedConditionWaitOptions,
10174 ) -> Poll<Result<ConditionWaitResult>> {
10175 match (self.predicate)() {
10176 Ok(true) => return Poll::Ready(Ok(ConditionWaitResult::Satisfied)),
10177 Ok(false) => {}
10178 Err(error) => return Poll::Ready(Err(error)),
10179 }
10180 if options.timeout_seconds == Some(0) {
10181 return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
10182 }
10183
10184 let ctx = self.ctx.clone();
10185 let mut state = match ctx.state.lock() {
10186 Ok(state) => state,
10187 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10188 };
10189 let mut command = serde_json::Map::from_iter([
10190 ("type".to_string(), json!("open_condition_wait")),
10191 (
10192 "condition_wait_occurrence_id".to_string(),
10193 json!(self.occurrence_id.as_deref().unwrap_or_default()),
10194 ),
10195 ("condition_key".to_string(), json!(options.condition_key)),
10196 (
10197 "condition_definition_fingerprint".to_string(),
10198 json!(options.predicate_identity),
10199 ),
10200 ]);
10201 if let Some(timeout_seconds) = options.timeout_seconds {
10202 command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
10203 }
10204 state.commands.push(Value::Object(command));
10205 drop(state);
10206 self.opened_wait = true;
10207 Poll::Pending
10208 }
10209}
10210
10211fn validate_recorded_condition_wait(
10212 sequence: u64,
10213 recorded_occurrence_id: &str,
10214 recorded_key: Option<&str>,
10215 recorded_predicate_identity: &str,
10216 recorded_timeout_seconds: Option<u64>,
10217 current_occurrence_id: &str,
10218 current: &ValidatedConditionWaitOptions,
10219) -> Result<()> {
10220 if recorded_occurrence_id != current_occurrence_id {
10221 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10222 "condition_wait_occurrence_mismatch",
10223 Some(sequence),
10224 Some(recorded_occurrence_id.to_string()),
10225 Some(current_occurrence_id.to_string()),
10226 "recorded condition occurrence differs from the current authored wait position",
10227 )));
10228 }
10229 if recorded_key != Some(current.condition_key.as_str()) {
10230 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10231 "condition_wait_key_mismatch",
10232 Some(sequence),
10233 recorded_key.map(str::to_string),
10234 Some(current.condition_key.clone()),
10235 "recorded condition identity differs from the current workflow wait",
10236 )));
10237 }
10238 if recorded_predicate_identity != current.predicate_identity {
10239 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10240 "condition_wait_predicate_mismatch",
10241 Some(sequence),
10242 Some(recorded_predicate_identity.to_string()),
10243 Some(current.predicate_identity.clone()),
10244 "recorded condition predicate behavior differs from current workflow code",
10245 )));
10246 }
10247 if recorded_timeout_seconds != current.timeout_seconds {
10248 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10249 "condition_wait_timeout_mismatch",
10250 Some(sequence),
10251 recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
10252 current.timeout_seconds.map(|seconds| format!("{seconds}s")),
10253 "recorded condition timeout differs from the current workflow wait",
10254 )));
10255 }
10256 Ok(())
10257}
10258
10259pub struct ChildWorkflowCall {
10261 ctx: WorkflowContext,
10262 workflow_type: String,
10263 options: ChildWorkflowOptions,
10264 args: Option<Result<AvroValue>>,
10265 scheduled: bool,
10266 matched_pending: bool,
10267 parallel_group_path: Vec<ParallelGroupMetadata>,
10268}
10269
10270impl ChildWorkflowCall {
10271 fn poll_avro_value(
10272 mut self: Pin<&mut Self>,
10273 _cx: &mut TaskContext<'_>,
10274 ) -> Poll<Result<ChildWorkflowAvroResult>> {
10275 if self.matched_pending {
10276 return Poll::Pending;
10277 }
10278
10279 let ctx = self.ctx.clone();
10280 let mut state = match ctx.state.lock() {
10281 Ok(state) => state,
10282 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10283 };
10284
10285 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10286 let sequence = recorded.sequence();
10287 match recorded {
10288 RecordedCommand::ChildWorkflow {
10289 workflow_type,
10290 outcome,
10291 parallel_group_path,
10292 ..
10293 } => {
10294 if let Err(error) = ensure_parallel_path_matches(
10295 sequence,
10296 parallel_group_path.as_deref(),
10297 &self.parallel_group_path,
10298 ) {
10299 return Poll::Ready(Err(error));
10300 }
10301 if let Some(recorded_type) = workflow_type {
10302 if recorded_type != self.workflow_type {
10303 return Poll::Ready(Err(Error::NonDeterministicReplay(
10304 ReplayFailure::new(
10305 "recorded_command_detail_mismatch",
10306 Some(sequence),
10307 Some(format!("child workflow:{recorded_type}")),
10308 Some(format!("child workflow:{}", self.workflow_type)),
10309 "recorded child workflow type differs from the current workflow command",
10310 ),
10311 )));
10312 }
10313 }
10314 state.command_cursor += 1;
10315 if let Some(outcome) = outcome {
10316 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
10317 }
10318 state.matched_recorded_pending = true;
10319 self.scheduled = true;
10320 self.matched_pending = true;
10321 return Poll::Pending;
10322 }
10323 other => {
10324 return Poll::Ready(Err(command_mismatch(
10325 &other,
10326 format!("child workflow:{}", self.workflow_type),
10327 )));
10328 }
10329 }
10330 }
10331
10332 if !self.scheduled {
10333 if self.options.task_queue.trim().is_empty() {
10334 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
10335 "task_queue must not be empty".to_string(),
10336 )));
10337 }
10338 for (name, value) in [
10339 (
10340 "execution_timeout_seconds",
10341 self.options.execution_timeout_seconds,
10342 ),
10343 ("run_timeout_seconds", self.options.run_timeout_seconds),
10344 ] {
10345 if value == Some(0) {
10346 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
10347 "{name} must be at least 1"
10348 ))));
10349 }
10350 }
10351
10352 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
10353 Ok(args) => args,
10354 Err(error) => return Poll::Ready(Err(error)),
10355 };
10356 let arguments = match encode_typed_envelope(
10357 &normalize_avro_arguments(args),
10358 &state.payload_codec,
10359 ) {
10360 Ok(arguments) => arguments,
10361 Err(error) => return Poll::Ready(Err(error)),
10362 };
10363 let mut command = json!({
10364 "type": "start_child_workflow",
10365 "workflow_type": self.workflow_type,
10366 "queue": self.options.task_queue,
10367 "parent_close_policy": self.options.parent_close_policy.as_str(),
10368 "arguments": arguments,
10369 });
10370 let object = command
10371 .as_object_mut()
10372 .expect("child workflow command is always an object");
10373 if let Some(policy) = &self.options.retry_policy {
10374 let mut retry_policy = serde_json::Map::new();
10375 if let Some(max_attempts) = policy.max_attempts {
10376 if max_attempts == 0 {
10377 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
10378 "retry_policy.max_attempts must be at least 1".to_string(),
10379 )));
10380 }
10381 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
10382 }
10383 if !policy.backoff_seconds.is_empty() {
10384 retry_policy
10385 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
10386 }
10387 if !policy.non_retryable_error_types.is_empty() {
10388 retry_policy.insert(
10389 "non_retryable_error_types".to_string(),
10390 json!(policy.non_retryable_error_types),
10391 );
10392 }
10393 if retry_policy.is_empty() {
10394 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
10395 "retry_policy must configure at least one field".to_string(),
10396 )));
10397 }
10398 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
10399 }
10400 if let Some(seconds) = self.options.execution_timeout_seconds {
10401 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
10402 }
10403 if let Some(seconds) = self.options.run_timeout_seconds {
10404 object.insert("run_timeout_seconds".to_string(), json!(seconds));
10405 }
10406 apply_parallel_group_path(object, &self.parallel_group_path);
10407 state.commands.push(command);
10408 self.scheduled = true;
10409 }
10410
10411 Poll::Pending
10412 }
10413}
10414
10415impl Future for ChildWorkflowCall {
10416 type Output = Result<ChildWorkflowResult>;
10417
10418 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10419 match self.poll_avro_value(cx) {
10420 Poll::Ready(Ok(result)) => match result.result.into_json() {
10421 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
10422 parent: result.parent,
10423 child: result.child,
10424 child_workflow_type: result.child_workflow_type,
10425 result: projected,
10426 })),
10427 Err(error) => Poll::Ready(Err(error)),
10428 },
10429 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
10430 Poll::Pending => Poll::Pending,
10431 }
10432 }
10433}
10434
10435fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
10436 Error::NonDeterministicReplay(ReplayFailure::new(
10437 "recorded_command_mismatch",
10438 Some(recorded.sequence()),
10439 Some(recorded.shape().to_string()),
10440 Some(actual.into()),
10441 "current workflow command does not match the recorded durable command sequence",
10442 ))
10443}
10444
10445pub struct SignalCall {
10446 ctx: WorkflowContext,
10447 signal_name: String,
10448 runtime_reserved_allowed: bool,
10449 opened_wait: bool,
10450 matched_pending: bool,
10451}
10452
10453impl SignalCall {
10454 fn poll_avro_value(
10455 mut self: Pin<&mut Self>,
10456 _cx: &mut TaskContext<'_>,
10457 ) -> Poll<Result<Vec<AvroValue>>> {
10458 if self.matched_pending {
10459 return Poll::Pending;
10460 }
10461 if !self.runtime_reserved_allowed {
10462 if let Err(error) = validate_user_signal_name(&self.signal_name) {
10463 return Poll::Ready(Err(error));
10464 }
10465 }
10466
10467 let ctx = self.ctx.clone();
10468 let mut state = match ctx.state.lock() {
10469 Ok(state) => state,
10470 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10471 };
10472
10473 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
10474 match recorded {
10475 RecordedCommand::SignalWait {
10476 sequence,
10477 signal_name,
10478 value,
10479 } => {
10480 if signal_name != self.signal_name {
10481 return Poll::Ready(Err(Error::NonDeterministicReplay(
10482 ReplayFailure::new(
10483 "recorded_command_detail_mismatch",
10484 Some(sequence),
10485 Some(format!("signal wait:{signal_name}")),
10486 Some(format!("signal wait:{}", self.signal_name)),
10487 "recorded signal name differs from the current workflow command",
10488 ),
10489 )));
10490 }
10491
10492 state.command_cursor += 1;
10493 if let Some(value) = value {
10494 return Poll::Ready(Ok(value));
10495 }
10496 if state
10497 .resume_signal
10498 .as_ref()
10499 .is_some_and(|signal| signal.signal_name == self.signal_name)
10500 {
10501 let signal = state
10502 .resume_signal
10503 .take()
10504 .expect("matching resume signal is present");
10505 return Poll::Ready(Ok(signal.arguments));
10506 }
10507
10508 state.matched_recorded_pending = true;
10509 self.opened_wait = true;
10510 self.matched_pending = true;
10511 return Poll::Pending;
10512 }
10513 other => {
10514 return Poll::Ready(Err(command_mismatch(
10515 &other,
10516 format!("signal wait:{}", self.signal_name),
10517 )));
10518 }
10519 }
10520 }
10521
10522 if state
10523 .resume_signal
10524 .as_ref()
10525 .is_some_and(|signal| signal.signal_name == self.signal_name)
10526 {
10527 let signal = state
10528 .resume_signal
10529 .take()
10530 .expect("matching resume signal is present");
10531 return Poll::Ready(Ok(signal.arguments));
10532 }
10533
10534 if !self.opened_wait {
10535 state.commands.push(json!({
10536 "type": "open_signal_wait",
10537 "signal_name": self.signal_name
10538 }));
10539 self.opened_wait = true;
10540 }
10541
10542 Poll::Pending
10543 }
10544}
10545
10546impl Future for SignalCall {
10547 type Output = Result<Vec<Value>>;
10548
10549 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10550 match self.poll_avro_value(cx) {
10551 Poll::Ready(Ok(values)) => Poll::Ready(
10552 values
10553 .into_iter()
10554 .map(AvroValue::into_json)
10555 .collect::<Result<Vec<_>>>(),
10556 ),
10557 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
10558 Poll::Pending => Poll::Pending,
10559 }
10560 }
10561}
10562
10563#[derive(Clone, Debug)]
10564pub struct ActivityContext {
10565 client: Client,
10566 pub task_id: String,
10567 pub activity_attempt_id: String,
10568 pub lease_owner: String,
10569 pub activity_type: String,
10570 pub attempt_number: u64,
10571 pub task_queue: String,
10572 pub worker_id: String,
10573}
10574
10575impl ActivityContext {
10576 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
10577 self.client
10578 .heartbeat_activity_task(
10579 &self.task_id,
10580 &self.activity_attempt_id,
10581 &self.lease_owner,
10582 details,
10583 )
10584 .await
10585 }
10586}
10587
10588fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
10589 validate_payload_codec(codec)?;
10590 match value {
10591 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
10592 value, codec,
10593 )?)),
10594 None => Ok(AvroValue::Array(Vec::new())),
10595 }
10596}
10597
10598fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
10599 let Some(signal_name) = task
10600 .signal_name
10601 .as_deref()
10602 .filter(|value| !value.is_empty())
10603 else {
10604 return Ok(None);
10605 };
10606 let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
10607 let AvroValue::Array(arguments) = decoded else {
10608 unreachable!("normalize_avro_arguments always returns an array");
10609 };
10610
10611 Ok(Some(ResumeSignal {
10612 signal_name: signal_name.to_string(),
10613 arguments,
10614 }))
10615}
10616
10617fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
10618 validate_payload_codec(&task.payload_codec)?;
10619 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
10620 validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
10621 for event in &task.history_events {
10622 validate_history_event_payloads(event, &task.payload_codec)?;
10623 }
10624 Ok(())
10625}
10626
10627fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
10628 validate_payload_codec(&task.payload_codec)?;
10629 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
10630}
10631
10632fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
10633 validate_payload_codec(&task.payload_codec)?;
10634 validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
10635 validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
10636 for event in &task.history_events {
10637 validate_history_event_payloads(event, &task.payload_codec)?;
10638 }
10639
10640 let Some(export) = task.history_export.as_ref() else {
10641 return Ok(());
10642 };
10643 let export_codec = match export.get("payloads") {
10644 Some(payloads) => declared_payload_codec(payloads, "codec")?,
10645 None => None,
10646 }
10647 .unwrap_or(&task.payload_codec);
10648 validate_payload_codec(export_codec)?;
10649
10650 if let Some(events) = export.get("history_events").and_then(Value::as_array) {
10651 for event in events {
10652 let event_type = event
10653 .get("event_type")
10654 .or_else(|| event.get("type"))
10655 .and_then(Value::as_str)
10656 .unwrap_or_default();
10657 if let Some(payload) = event.get("payload") {
10658 validate_history_payloads(event_type, payload, export_codec)?;
10659 }
10660 }
10661 }
10662 for signal in export
10663 .get("signals")
10664 .and_then(Value::as_array)
10665 .into_iter()
10666 .flatten()
10667 {
10668 let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
10669 validate_payload_codec(codec)?;
10670 validate_optional_inbound_payload(signal.get("arguments"), codec)?;
10671 }
10672 for activity in export
10673 .get("activities")
10674 .and_then(Value::as_array)
10675 .into_iter()
10676 .flatten()
10677 {
10678 let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
10679 validate_payload_codec(codec)?;
10680 validate_optional_inbound_payload(activity.get("arguments"), codec)?;
10681 validate_optional_inbound_payload(activity.get("result"), codec)?;
10682 }
10683 Ok(())
10684}
10685
10686fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
10687 validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
10688}
10689
10690fn validate_history_payloads(
10691 event_type: &str,
10692 payload: &Value,
10693 fallback_codec: &str,
10694) -> Result<()> {
10695 let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
10696 validate_payload_codec(codec)?;
10697 for field in history_payload_fields(event_type) {
10698 validate_optional_inbound_payload(payload.get(*field), codec)?;
10699 }
10700 Ok(())
10701}
10702
10703const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
10704
10705fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
10706 match event_type {
10707 "ActivityCompleted" => &["result"],
10708 "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
10709 "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
10710 "UpdateCompleted" | "SideEffectRecorded" => &["result"],
10711 "ChildRunCompleted" => &["result", "output"],
10712 "WorkflowCompleted" => &["output"],
10713 "ServiceCallStarted"
10714 | "ServiceCallCompleted"
10715 | "ServiceCallFailed"
10716 | "ServiceCallCancelled" => &["request_payload", "response_payload"],
10717 _ => &[],
10718 }
10719}
10720
10721fn signal_history_payload(payload: &Value) -> Option<&Value> {
10722 SIGNAL_HISTORY_PAYLOAD_FIELDS
10723 .iter()
10724 .find_map(|field| payload.get(*field))
10725}
10726
10727fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
10728 match value.get(field) {
10729 None => Ok(None),
10730 Some(Value::String(codec)) => Ok(Some(codec)),
10731 Some(_) => Err(invalid_payload_envelope()),
10732 }
10733}
10734
10735fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
10736 validate_payload_codec(codec)?;
10737 if let Some(value) = value.filter(|value| !value.is_null()) {
10738 decode_wire_avro_value(value, codec)?;
10739 }
10740 Ok(())
10741}
10742
10743fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
10744 let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
10745 invalid_recorded_history(
10746 "parallel_group_metadata_invalid",
10747 sequence,
10748 "non-empty parallel_group_id",
10749 &payload.to_string(),
10750 "parallel-group history is missing its stable identity",
10751 )
10752 })?;
10753 let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
10754 invalid_recorded_history(
10755 "parallel_group_metadata_invalid",
10756 sequence,
10757 "activity, child, timer, or mixed group kind",
10758 &payload.to_string(),
10759 "parallel-group history is missing its group kind",
10760 )
10761 })?;
10762 if !matches!(kind.as_str(), "activity" | "child" | "timer" | "mixed") {
10763 return Err(invalid_recorded_history(
10764 "parallel_group_metadata_invalid",
10765 sequence,
10766 "activity, child, timer, or mixed group kind",
10767 &kind,
10768 "parallel-group history contains an unsupported group kind",
10769 ));
10770 }
10771 let base_sequence = payload
10772 .get("parallel_group_base_sequence")
10773 .and_then(value_as_u64)
10774 .filter(|value| *value > 0)
10775 .ok_or_else(|| {
10776 invalid_recorded_history(
10777 "parallel_group_metadata_invalid",
10778 sequence,
10779 "positive parallel_group_base_sequence",
10780 &payload.to_string(),
10781 "parallel-group history contains an invalid base sequence",
10782 )
10783 })?;
10784 let size = payload
10785 .get("parallel_group_size")
10786 .and_then(value_as_u64)
10787 .and_then(|value| usize::try_from(value).ok())
10788 .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
10789 .ok_or_else(|| {
10790 invalid_recorded_history(
10791 "parallel_group_metadata_invalid",
10792 sequence,
10793 "bounded positive parallel_group_size",
10794 &payload.to_string(),
10795 "parallel-group history contains an invalid group size",
10796 )
10797 })?;
10798 let index = payload
10799 .get("parallel_group_index")
10800 .and_then(value_as_u64)
10801 .and_then(|value| usize::try_from(value).ok())
10802 .filter(|value| *value < size)
10803 .ok_or_else(|| {
10804 invalid_recorded_history(
10805 "parallel_group_metadata_invalid",
10806 sequence,
10807 "parallel_group_index within group bounds",
10808 &payload.to_string(),
10809 "parallel-group history contains an invalid member index",
10810 )
10811 })?;
10812 if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
10813 return Err(invalid_recorded_history(
10814 "parallel_group_metadata_invalid",
10815 sequence,
10816 "base sequence plus member index equals workflow sequence",
10817 &payload.to_string(),
10818 "parallel-group path does not preserve durable workflow position",
10819 ));
10820 }
10821 let expected_id = format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind));
10822 if group_id != expected_id {
10823 return Err(invalid_recorded_history(
10824 "parallel_group_metadata_invalid",
10825 sequence,
10826 &expected_id,
10827 &group_id,
10828 "parallel-group history contains an incompatible stable group ID",
10829 ));
10830 }
10831 Ok(ParallelGroupMetadata {
10832 parallel_group_id: group_id,
10833 parallel_group_kind: kind,
10834 parallel_group_base_sequence: base_sequence,
10835 parallel_group_size: size,
10836 parallel_group_index: index,
10837 })
10838}
10839
10840fn recorded_parallel_group_path(
10841 events: &[&HistoryEvent],
10842 sequence: u64,
10843) -> Result<Option<Vec<ParallelGroupMetadata>>> {
10844 let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
10845 for event in events {
10846 let payload = &event.payload;
10847 let has_metadata = payload.get("parallel_group_path").is_some()
10848 || payload.get("parallel_group_id").is_some()
10849 || payload.get("parallel_group_kind").is_some()
10850 || payload.get("parallel_group_base_sequence").is_some()
10851 || payload.get("parallel_group_size").is_some()
10852 || payload.get("parallel_group_index").is_some();
10853 if !has_metadata {
10854 continue;
10855 }
10856
10857 let top_level = recorded_parallel_group_entry(payload, sequence)?;
10858 let path = match payload.get("parallel_group_path") {
10859 None => vec![top_level.clone()],
10860 Some(Value::Array(entries)) if !entries.is_empty() => entries
10861 .iter()
10862 .map(|entry| recorded_parallel_group_entry(entry, sequence))
10863 .collect::<Result<Vec<_>>>()?,
10864 Some(value) => {
10865 return Err(invalid_recorded_history(
10866 "parallel_group_metadata_invalid",
10867 sequence,
10868 "non-empty parallel_group_path list",
10869 &value.to_string(),
10870 "parallel-group history contains an invalid group path",
10871 ));
10872 }
10873 };
10874 if path.last() != Some(&top_level) {
10875 return Err(invalid_recorded_history(
10876 "parallel_group_metadata_invalid",
10877 sequence,
10878 &serde_json::to_string(&path.last()).unwrap_or_default(),
10879 &serde_json::to_string(&top_level).unwrap_or_default(),
10880 "parallel-group top-level fields do not match the innermost path entry",
10881 ));
10882 }
10883 if recorded.as_ref().is_some_and(|existing| existing != &path) {
10884 return Err(invalid_recorded_history(
10885 "parallel_group_history_conflict",
10886 sequence,
10887 &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
10888 &serde_json::to_string(&path).unwrap_or_default(),
10889 "parallel-group metadata changed between scheduling and resolution history",
10890 ));
10891 }
10892 recorded = Some(path);
10893 }
10894 Ok(recorded)
10895}
10896
10897fn recorded_commands(
10898 events: &[HistoryEvent],
10899 fallback_codec: &str,
10900 parent: WorkflowIdentity,
10901) -> Result<Vec<RecordedCommand>> {
10902 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
10903 let mut last_new_sequence = None;
10904
10905 for event in events {
10906 let is_activity = matches!(
10907 event.event_type.as_str(),
10908 "ActivityScheduled"
10909 | "ActivityStarted"
10910 | "ActivityHeartbeatRecorded"
10911 | "ActivityRetryScheduled"
10912 | "ActivityCompleted"
10913 | "ActivityFailed"
10914 | "ActivityCancelled"
10915 | "ActivityTimedOut"
10916 );
10917 let is_workflow_timer = matches!(
10918 event.event_type.as_str(),
10919 "TimerScheduled" | "TimerCancelled" | "TimerFired"
10920 ) && !is_internal_timer_event(event);
10921 let is_child_workflow = matches!(
10922 event.event_type.as_str(),
10923 "ChildWorkflowScheduled"
10924 | "ChildRunCompleted"
10925 | "ChildRunFailed"
10926 | "ChildRunCancelled"
10927 | "ChildRunTerminated"
10928 );
10929 let is_signal_wait = is_recorded_signal_wait_event(event);
10930 let is_condition_wait = is_recorded_condition_wait_event(event);
10931 let is_search_attributes = event.event_type == "SearchAttributesUpserted";
10932 let is_side_effect = event.event_type == "SideEffectRecorded";
10933 let is_version_marker = event.event_type == "VersionMarkerRecorded";
10934 let is_memo = event.event_type == "MemoUpserted";
10935 if !is_activity
10936 && !is_workflow_timer
10937 && !is_child_workflow
10938 && !is_signal_wait
10939 && !is_condition_wait
10940 && !is_search_attributes
10941 && !is_side_effect
10942 && !is_version_marker
10943 && !is_memo
10944 {
10945 continue;
10946 }
10947
10948 let sequence = durable_event_sequence(event).ok_or_else(|| {
10949 Error::NonDeterministicReplay(ReplayFailure::new(
10950 "durable_command_sequence_missing",
10951 None,
10952 Some("positive workflow sequence".to_string()),
10953 Some(event.event_type.clone()),
10954 "durable command history event has no workflow sequence",
10955 ))
10956 })?;
10957 if sequence == 0 {
10958 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10959 "durable_command_sequence_invalid",
10960 Some(sequence),
10961 Some("positive workflow sequence".to_string()),
10962 Some(sequence.to_string()),
10963 "durable command history uses an invalid workflow sequence",
10964 )));
10965 }
10966 if !events_by_sequence.contains_key(&sequence) {
10967 if let Some(previous) = last_new_sequence {
10968 if sequence < previous {
10969 return Err(invalid_recorded_history(
10970 "durable_command_sequence_mismatch",
10971 sequence,
10972 &format!("workflow sequence greater than {previous}"),
10973 &sequence.to_string(),
10974 "durable commands are not strictly ordered by their recorded workflow sequence",
10975 ));
10976 }
10977 }
10978 last_new_sequence = Some(sequence);
10979 }
10980 events_by_sequence.entry(sequence).or_default().push(event);
10981 }
10982
10983 let commands: Vec<RecordedCommand> = events_by_sequence
10984 .into_iter()
10985 .map(|(sequence, sequence_events)| {
10986 let activity_events: Vec<_> = sequence_events
10987 .iter()
10988 .copied()
10989 .filter(|event| event.event_type.starts_with("Activity"))
10990 .collect();
10991 let timer_events: Vec<_> = sequence_events
10992 .iter()
10993 .copied()
10994 .filter(|event| event.event_type.starts_with("Timer"))
10995 .collect();
10996 let child_events: Vec<_> = sequence_events
10997 .iter()
10998 .copied()
10999 .filter(|event| {
11000 event.event_type == "ChildWorkflowScheduled"
11001 || event.event_type.starts_with("ChildRun")
11002 })
11003 .collect();
11004 let signal_wait_events: Vec<_> = sequence_events
11005 .iter()
11006 .copied()
11007 .filter(|event| is_recorded_signal_wait_event(event))
11008 .collect();
11009 let condition_wait_events: Vec<_> = sequence_events
11010 .iter()
11011 .copied()
11012 .filter(|event| is_recorded_condition_wait_event(event))
11013 .collect();
11014 let search_attribute_events: Vec<_> = sequence_events
11015 .iter()
11016 .copied()
11017 .filter(|event| event.event_type == "SearchAttributesUpserted")
11018 .collect();
11019 let side_effect_events: Vec<_> = sequence_events
11020 .iter()
11021 .copied()
11022 .filter(|event| event.event_type == "SideEffectRecorded")
11023 .collect();
11024 let version_marker_events: Vec<_> = sequence_events
11025 .iter()
11026 .copied()
11027 .filter(|event| event.event_type == "VersionMarkerRecorded")
11028 .collect();
11029 let memo_events: Vec<_> = sequence_events
11030 .iter()
11031 .copied()
11032 .filter(|event| event.event_type == "MemoUpserted")
11033 .collect();
11034
11035 let command_kind_count = usize::from(!activity_events.is_empty())
11036 + usize::from(!timer_events.is_empty())
11037 + usize::from(!child_events.is_empty())
11038 + usize::from(!signal_wait_events.is_empty())
11039 + usize::from(!condition_wait_events.is_empty())
11040 + usize::from(!search_attribute_events.is_empty())
11041 + usize::from(!side_effect_events.is_empty())
11042 + usize::from(!version_marker_events.is_empty())
11043 + usize::from(!memo_events.is_empty());
11044 if command_kind_count > 1 {
11045 let actual = [
11046 (!activity_events.is_empty()).then_some("activity"),
11047 (!timer_events.is_empty()).then_some("timer"),
11048 (!child_events.is_empty()).then_some("child workflow"),
11049 (!signal_wait_events.is_empty()).then_some("signal wait"),
11050 (!condition_wait_events.is_empty()).then_some("condition wait"),
11051 (!search_attribute_events.is_empty()).then_some("search-attribute update"),
11052 (!side_effect_events.is_empty()).then_some("side effect"),
11053 (!version_marker_events.is_empty()).then_some("version marker"),
11054 (!memo_events.is_empty()).then_some("memo upsert"),
11055 ]
11056 .into_iter()
11057 .flatten()
11058 .collect::<Vec<_>>()
11059 .join(" and ");
11060 return Err(invalid_recorded_history(
11061 "durable_command_sequence_collision",
11062 sequence,
11063 "one durable command kind",
11064 &actual,
11065 "one workflow sequence records more than one durable command kind",
11066 ));
11067 }
11068
11069 if !activity_events.is_empty() {
11070 let parallel_group_path =
11071 recorded_parallel_group_path(&activity_events, sequence)?;
11072 let scheduled_count = activity_events
11073 .iter()
11074 .filter(|event| event.event_type == "ActivityScheduled")
11075 .count();
11076 if scheduled_count > 1 {
11077 return Err(invalid_recorded_history(
11078 "duplicate_activity_schedule",
11079 sequence,
11080 "at most one ActivityScheduled event",
11081 "multiple ActivityScheduled events",
11082 "activity history schedules more than one command at one workflow sequence",
11083 ));
11084 }
11085 let activity_type = activity_events.iter().find_map(|event| {
11086 event
11087 .payload
11088 .get("activity_type")
11089 .or_else(|| event.payload.get("activity_name"))
11090 .and_then(Value::as_str)
11091 .map(str::to_string)
11092 });
11093 if activity_events.iter().filter_map(|event| {
11094 event
11095 .payload
11096 .get("activity_type")
11097 .or_else(|| event.payload.get("activity_name"))
11098 .and_then(Value::as_str)
11099 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
11100 return Err(invalid_recorded_history(
11101 "activity_identity_mismatch",
11102 sequence,
11103 activity_type.as_deref().unwrap_or("one activity identity"),
11104 "conflicting activity identities",
11105 "activity lifecycle events at one workflow sequence disagree on identity",
11106 ));
11107 }
11108 let terminal: Vec<_> = activity_events
11109 .iter()
11110 .copied()
11111 .filter(|event| {
11112 matches!(
11113 event.event_type.as_str(),
11114 "ActivityCompleted"
11115 | "ActivityFailed"
11116 | "ActivityCancelled"
11117 | "ActivityTimedOut"
11118 )
11119 })
11120 .collect();
11121 let duplicate_delivery = terminal.first().is_some_and(|first| {
11122 terminal.iter().all(|event| {
11123 event.event_type == first.event_type && event.payload == first.payload
11124 })
11125 });
11126 if terminal.len() > 1 && !duplicate_delivery {
11127 return Err(invalid_recorded_history(
11128 "duplicate_activity_terminal_event",
11129 sequence,
11130 "at most one terminal activity event",
11131 "multiple terminal activity events",
11132 "activity history settles one command more than once",
11133 ));
11134 }
11135 let outcome = terminal
11136 .first()
11137 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
11138 .transpose()?;
11139 let options = activity_events
11140 .iter()
11141 .find(|event| event.event_type == "ActivityScheduled")
11142 .and_then(|event| event.payload.get("activity"))
11143 .and_then(Value::as_object)
11144 .map(|activity| RecordedActivityOptions {
11145 task_queue: recorded_optional_string(activity, "queue"),
11146 execution_mode: recorded_optional_string(activity, "execution_mode"),
11147 retry_policy: recorded_activity_retry_snapshot(
11148 activity.get("retry_policy"),
11149 ),
11150 });
11151 return Ok(RecordedCommand::Activity {
11152 sequence,
11153 activity_type,
11154 options,
11155 outcome,
11156 parallel_group_path,
11157 });
11158 }
11159
11160 if !child_events.is_empty() {
11161 let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
11162 let scheduled: Vec<_> = child_events
11163 .iter()
11164 .copied()
11165 .filter(|event| event.event_type == "ChildWorkflowScheduled")
11166 .collect();
11167 if scheduled.len() != 1 {
11168 return Err(invalid_recorded_history(
11169 "child_workflow_schedule_missing_or_duplicate",
11170 sequence,
11171 "one ChildWorkflowScheduled event",
11172 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
11173 "child workflow replay requires exactly one recorded schedule event",
11174 ));
11175 }
11176 let workflow_type = child_events.iter().find_map(|event| {
11177 event
11178 .payload
11179 .get("child_workflow_type")
11180 .or_else(|| event.payload.get("workflow_type"))
11181 .and_then(Value::as_str)
11182 .filter(|value| !value.is_empty())
11183 .map(str::to_string)
11184 });
11185 if child_events
11186 .iter()
11187 .filter_map(|event| {
11188 event
11189 .payload
11190 .get("child_workflow_type")
11191 .or_else(|| event.payload.get("workflow_type"))
11192 .and_then(Value::as_str)
11193 })
11194 .any(|candidate| Some(candidate) != workflow_type.as_deref())
11195 {
11196 return Err(invalid_recorded_history(
11197 "child_workflow_identity_mismatch",
11198 sequence,
11199 workflow_type
11200 .as_deref()
11201 .unwrap_or("one child workflow type"),
11202 "conflicting child workflow types",
11203 "child workflow lifecycle events at one sequence disagree on type",
11204 ));
11205 }
11206 let mut outcomes = child_workflow_outcomes(
11207 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
11208 fallback_codec,
11209 parent.clone(),
11210 )?;
11211 let terminal_events = child_events
11212 .iter()
11213 .copied()
11214 .filter(|event| event.event_type.starts_with("ChildRun"))
11215 .collect::<Vec<_>>();
11216 let duplicate_delivery = terminal_events.first().is_some_and(|first| {
11217 terminal_events.iter().all(|event| {
11218 event.event_type == first.event_type && event.payload == first.payload
11219 })
11220 });
11221 if outcomes.len() > 1 && !duplicate_delivery {
11222 return Err(invalid_recorded_history(
11223 "duplicate_child_workflow_terminal_event",
11224 sequence,
11225 "at most one terminal child event",
11226 "multiple terminal child events",
11227 "child workflow history settles one command more than once",
11228 ));
11229 }
11230 return Ok(RecordedCommand::ChildWorkflow {
11231 sequence,
11232 workflow_type,
11233 outcome: outcomes.pop(),
11234 parallel_group_path,
11235 });
11236 }
11237
11238 if !signal_wait_events.is_empty() {
11239 let opened: Vec<_> = signal_wait_events
11240 .iter()
11241 .copied()
11242 .filter(|event| event.event_type == "SignalWaitOpened")
11243 .collect();
11244 if opened.len() != 1 {
11245 return Err(invalid_recorded_history(
11246 "signal_wait_open_missing_or_duplicate",
11247 sequence,
11248 "one SignalWaitOpened event",
11249 &format!("{} SignalWaitOpened events", opened.len()),
11250 "signal replay requires exactly one canonical wait-open event",
11251 ));
11252 }
11253
11254 let applied: Vec<_> = signal_wait_events
11255 .iter()
11256 .copied()
11257 .filter(|event| event.event_type == "SignalApplied")
11258 .collect();
11259 if applied.len() > 1 {
11260 return Err(invalid_recorded_history(
11261 "duplicate_signal_wait_apply",
11262 sequence,
11263 "at most one SignalApplied event",
11264 "multiple SignalApplied events",
11265 "signal history applies one durable wait more than once",
11266 ));
11267 }
11268
11269 let signal_names = signal_wait_events
11270 .iter()
11271 .map(|event| required_signal_wait_name(event, sequence))
11272 .collect::<Result<Vec<_>>>()?;
11273 let signal_name = signal_names
11274 .first()
11275 .expect("signal wait events are not empty")
11276 .clone();
11277 if signal_names.iter().any(|candidate| candidate != &signal_name) {
11278 return Err(invalid_recorded_history(
11279 "signal_wait_identity_mismatch",
11280 sequence,
11281 &signal_name,
11282 "conflicting signal names",
11283 "signal wait lifecycle events at one workflow sequence disagree on identity",
11284 ));
11285 }
11286 let value = applied
11287 .first()
11288 .map(|event| decode_signal_event_arguments(event, fallback_codec))
11289 .transpose()?;
11290 return Ok(RecordedCommand::SignalWait {
11291 sequence,
11292 signal_name,
11293 value,
11294 });
11295 }
11296
11297 if !condition_wait_events.is_empty() {
11298 return recorded_condition_wait(
11299 sequence,
11300 &condition_wait_events,
11301 events,
11302 );
11303 }
11304
11305 if !search_attribute_events.is_empty() {
11306 if search_attribute_events.len() != 1 {
11307 return Err(invalid_recorded_history(
11308 "duplicate_search_attribute_update",
11309 sequence,
11310 "one SearchAttributesUpserted event",
11311 &format!(
11312 "{} SearchAttributesUpserted events",
11313 search_attribute_events.len()
11314 ),
11315 "search-attribute history records one workflow command more than once",
11316 ));
11317 }
11318 let payload = &search_attribute_events[0].payload;
11319 let attributes = payload
11320 .get("attributes")
11321 .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
11322 .cloned()
11323 .ok_or_else(|| {
11324 invalid_recorded_history(
11325 "search_attribute_update_missing",
11326 sequence,
11327 "non-empty attributes object",
11328 "missing or invalid attributes",
11329 "search-attribute history is missing its recorded mutation",
11330 )
11331 })?;
11332 let attribute_types =
11333 recorded_search_attribute_types(payload, &attributes, sequence)?;
11334 return Ok(RecordedCommand::SearchAttributes {
11335 sequence,
11336 attributes,
11337 attribute_types,
11338 });
11339 }
11340
11341 if !side_effect_events.is_empty() {
11342 if side_effect_events.len() != 1 {
11343 return Err(invalid_recorded_history(
11344 "duplicate_side_effect_record",
11345 sequence,
11346 "one SideEffectRecorded event",
11347 &format!("{} SideEffectRecorded events", side_effect_events.len()),
11348 "side-effect history records one workflow command more than once",
11349 ));
11350 }
11351 let event = side_effect_events[0];
11352 let result = event.payload.get("result").ok_or_else(|| {
11353 invalid_recorded_history(
11354 "side_effect_result_missing",
11355 sequence,
11356 "recorded result payload",
11357 "missing result",
11358 "side-effect history is missing its recorded value",
11359 )
11360 })?;
11361 let has_published_envelope = result.as_str().is_some()
11362 || result.as_object().is_some_and(|envelope| {
11363 envelope.get("codec").and_then(Value::as_str).is_some()
11364 && envelope.get("blob").and_then(Value::as_str).is_some()
11365 });
11366 if !has_published_envelope {
11367 return Err(invalid_recorded_history(
11368 "side_effect_payload_malformed",
11369 sequence,
11370 "payload blob or {codec, blob} envelope",
11371 &result.to_string(),
11372 "side-effect history result does not use a published payload envelope",
11373 ));
11374 }
11375 let codec = event
11376 .payload
11377 .get("payload_codec")
11378 .and_then(Value::as_str)
11379 .unwrap_or(fallback_codec);
11380 let value = decode_wire_avro_value(result, codec).map_err(|error| {
11381 if error.to_string().contains("unsupported_payload_codec") {
11382 return error;
11383 }
11384
11385 invalid_recorded_history(
11386 "side_effect_payload_incompatible",
11387 sequence,
11388 &format!("valid {codec} payload envelope"),
11389 &error.to_string(),
11390 "side-effect history payload cannot be decoded with its recorded codec",
11391 )
11392 })?;
11393 return Ok(RecordedCommand::SideEffect { sequence, value });
11394 }
11395
11396 if !version_marker_events.is_empty() {
11397 if version_marker_events.len() != 1 {
11398 return Err(invalid_recorded_history(
11399 "duplicate_version_marker_record",
11400 sequence,
11401 "one VersionMarkerRecorded event",
11402 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
11403 "version-marker history records one workflow command more than once",
11404 ));
11405 }
11406 let payload = &version_marker_events[0].payload;
11407 let change_id = payload
11408 .get("change_id")
11409 .and_then(Value::as_str)
11410 .filter(|value| !value.is_empty())
11411 .map(str::to_string)
11412 .ok_or_else(|| {
11413 invalid_recorded_history(
11414 "version_marker_field_missing",
11415 sequence,
11416 "non-empty change_id",
11417 "missing or invalid change_id",
11418 "version-marker history is missing its stable change ID",
11419 )
11420 })?;
11421 let version = required_version_i32(payload, "version", sequence)?;
11422 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
11423 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
11424 if min_supported > max_supported || version < min_supported || version > max_supported {
11425 return Err(invalid_recorded_history(
11426 "version_marker_history_range_invalid",
11427 sequence,
11428 "min_supported <= version <= max_supported",
11429 &format!("{min_supported} <= {version} <= {max_supported}"),
11430 "recorded version marker contains an internally incompatible range",
11431 ));
11432 }
11433 return Ok(RecordedCommand::VersionMarker {
11434 sequence,
11435 change_id,
11436 version,
11437 });
11438 }
11439
11440 if !memo_events.is_empty() {
11441 if memo_events.len() != 1 {
11442 return Err(invalid_recorded_history(
11443 "duplicate_memo_upsert_record",
11444 sequence,
11445 "one MemoUpserted event",
11446 &format!("{} MemoUpserted events", memo_events.len()),
11447 "memo history records one workflow update more than once",
11448 ));
11449 }
11450 let payload = &memo_events[0].payload;
11451 let entries = payload.get("entries").cloned().ok_or_else(|| {
11452 invalid_recorded_history(
11453 "memo_entries_missing",
11454 sequence,
11455 "memo entries object",
11456 "missing entries",
11457 "MemoUpserted history is missing replay identity entries",
11458 )
11459 })?;
11460 let entries = decode_memo_history_map(&entries, true).map_err(|error| {
11461 invalid_recorded_history(
11462 "memo_entries_invalid",
11463 sequence,
11464 "valid canonical memo entries",
11465 &error.to_string(),
11466 "MemoUpserted history contains invalid replay identity entries",
11467 )
11468 })?;
11469 let merged = payload.get("merged").cloned().ok_or_else(|| {
11470 invalid_recorded_history(
11471 "memo_merged_projection_missing",
11472 sequence,
11473 "merged memo projection",
11474 "missing merged",
11475 "MemoUpserted history is missing its merged projection",
11476 )
11477 })?;
11478 decode_memo_history_map(&merged, false).map_err(|error| {
11479 invalid_recorded_history(
11480 "memo_merged_projection_invalid",
11481 sequence,
11482 "valid merged memo projection",
11483 &error.to_string(),
11484 "MemoUpserted history contains an invalid merged projection",
11485 )
11486 })?;
11487
11488 return Ok(RecordedCommand::Memo { sequence, entries });
11489 }
11490 let scheduled: Vec<_> = timer_events
11491 .iter()
11492 .copied()
11493 .filter(|event| event.event_type == "TimerScheduled")
11494 .collect();
11495 let fired: Vec<_> = timer_events
11496 .iter()
11497 .copied()
11498 .filter(|event| event.event_type == "TimerFired")
11499 .collect();
11500 if scheduled.len() != 1 {
11501 return Err(invalid_recorded_history(
11502 "timer_schedule_missing_or_duplicate",
11503 sequence,
11504 "one TimerScheduled event",
11505 &format!("{} TimerScheduled events", scheduled.len()),
11506 "timer replay requires exactly one recorded schedule event",
11507 ));
11508 }
11509 if fired.len() > 1 {
11510 return Err(invalid_recorded_history(
11511 "duplicate_timer_fire",
11512 sequence,
11513 "at most one TimerFired event",
11514 "multiple TimerFired events",
11515 "timer history contains more than one fire event for a workflow sequence",
11516 ));
11517 }
11518
11519 let scheduled = scheduled[0];
11520 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
11521 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
11522 if let Some(fired) = fired.first() {
11523 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
11524 if fired_timer_id != timer_id {
11525 return Err(invalid_recorded_history(
11526 "timer_identity_mismatch",
11527 sequence,
11528 &timer_id,
11529 &fired_timer_id,
11530 "TimerFired does not correspond to the recorded TimerScheduled event",
11531 ));
11532 }
11533 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
11534 if fired_delay != delay_seconds {
11535 return Err(invalid_recorded_history(
11536 "timer_history_delay_mismatch",
11537 sequence,
11538 &delay_seconds.to_string(),
11539 &fired_delay.to_string(),
11540 "TimerScheduled and TimerFired record different delays",
11541 ));
11542 }
11543 }
11544
11545 Ok(RecordedCommand::Timer {
11546 sequence,
11547 delay_seconds,
11548 fired: !fired.is_empty(),
11549 parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
11550 })
11551 })
11552 .collect::<Result<_>>()?;
11553
11554 let mut marker_sequences = HashMap::new();
11555 for command in &commands {
11556 if let RecordedCommand::VersionMarker {
11557 sequence,
11558 change_id,
11559 ..
11560 } = command
11561 {
11562 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
11563 return Err(invalid_recorded_history(
11564 "duplicate_version_marker",
11565 *sequence,
11566 &format!("one marker for change ID {change_id:?}"),
11567 &format!("markers at sequences {first_sequence} and {sequence}"),
11568 "workflow history contains duplicate markers for one stable change ID",
11569 ));
11570 }
11571 }
11572 }
11573
11574 Ok(commands)
11575}
11576
11577fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
11578 payload
11579 .get(field)
11580 .and_then(Value::as_i64)
11581 .and_then(|value| i32::try_from(value).ok())
11582 .ok_or_else(|| {
11583 invalid_recorded_history(
11584 "version_marker_field_missing",
11585 sequence,
11586 &format!("integer {field}"),
11587 "missing or out-of-range integer",
11588 "version-marker history is missing a required integer field",
11589 )
11590 })
11591}
11592
11593fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
11594 event
11595 .payload
11596 .get("sequence")
11597 .or_else(|| event.payload.get("workflow_sequence"))
11598 .or_else(|| event.raw.get("sequence"))
11599 .or_else(|| event.raw.get("workflow_sequence"))
11600 .and_then(value_as_u64)
11601}
11602
11603fn is_internal_timer_event(event: &HistoryEvent) -> bool {
11604 matches!(
11605 event
11606 .payload
11607 .get("timer_kind")
11608 .or_else(|| event.raw.get("timer_kind"))
11609 .and_then(Value::as_str),
11610 Some("condition_timeout" | "signal_timeout")
11611 )
11612}
11613
11614fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
11615 matches!(
11616 event.event_type.as_str(),
11617 "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
11618 )
11619}
11620
11621fn recorded_condition_wait(
11622 sequence: u64,
11623 condition_events: &[&HistoryEvent],
11624 all_events: &[HistoryEvent],
11625) -> Result<RecordedCommand> {
11626 let opened = condition_events
11627 .iter()
11628 .copied()
11629 .filter(|event| event.event_type == "ConditionWaitOpened")
11630 .collect::<Vec<_>>();
11631 if opened.len() != 1 {
11632 return Err(invalid_recorded_history(
11633 "condition_wait_open_missing_or_duplicate",
11634 sequence,
11635 "one ConditionWaitOpened event",
11636 &format!("{} ConditionWaitOpened events", opened.len()),
11637 "condition replay requires exactly one canonical wait-open event",
11638 ));
11639 }
11640 let terminal = condition_events
11641 .iter()
11642 .copied()
11643 .filter(|event| {
11644 matches!(
11645 event.event_type.as_str(),
11646 "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
11647 )
11648 })
11649 .collect::<Vec<_>>();
11650 if terminal.len() > 1 {
11651 return Err(invalid_recorded_history(
11652 "duplicate_condition_wait_terminal_event",
11653 sequence,
11654 "at most one condition terminal event",
11655 "multiple condition terminal events",
11656 "condition history settles one durable wait more than once",
11657 ));
11658 }
11659
11660 let opened = opened[0];
11661 let condition_wait_id = required_condition_wait_id(opened, sequence)?;
11662 let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
11663 for event in condition_events
11664 .iter()
11665 .copied()
11666 .filter(|event| !std::ptr::eq(*event, opened))
11667 {
11668 let event_wait_id = required_condition_wait_id(event, sequence)?;
11669 if event_wait_id != condition_wait_id {
11670 return Err(invalid_recorded_history(
11671 "condition_wait_id_mismatch",
11672 sequence,
11673 &condition_wait_id,
11674 &event_wait_id,
11675 "condition lifecycle events at one sequence disagree on wait identity",
11676 ));
11677 }
11678 let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
11679 if event_occurrence_id != occurrence_id {
11680 return Err(invalid_recorded_history(
11681 "condition_wait_occurrence_history_mismatch",
11682 sequence,
11683 &occurrence_id,
11684 &event_occurrence_id,
11685 "condition lifecycle events at one sequence disagree on authored occurrence identity",
11686 ));
11687 }
11688 }
11689
11690 let condition_key = optional_non_empty_history_string(opened, "condition_key");
11691 let predicate_identity = opened
11692 .payload
11693 .get("condition_definition_fingerprint")
11694 .and_then(Value::as_str)
11695 .filter(|value| !value.is_empty())
11696 .map(str::to_string)
11697 .ok_or_else(|| {
11698 invalid_recorded_history(
11699 "condition_wait_predicate_fingerprint_missing",
11700 sequence,
11701 "non-empty condition_definition_fingerprint",
11702 &opened.event_type,
11703 "canonical condition history is missing its predicate identity",
11704 )
11705 })?;
11706 let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
11707 for event in condition_events
11708 .iter()
11709 .copied()
11710 .filter(|event| !std::ptr::eq(*event, opened))
11711 {
11712 for (field, opened_value) in [
11713 ("condition_key", condition_key.as_deref()),
11714 (
11715 "condition_definition_fingerprint",
11716 Some(predicate_identity.as_str()),
11717 ),
11718 ] {
11719 if let Some(value) = optional_non_empty_history_string(event, field) {
11720 if opened_value.is_some_and(|opened_value| opened_value != value) {
11721 return Err(invalid_recorded_history(
11722 "condition_wait_definition_history_mismatch",
11723 sequence,
11724 opened_value.unwrap_or_default(),
11725 &value,
11726 "condition lifecycle events disagree on the recorded definition",
11727 ));
11728 }
11729 }
11730 }
11731 if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
11732 if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
11733 return Err(invalid_recorded_history(
11734 "condition_wait_definition_history_mismatch",
11735 sequence,
11736 &format!("{}s", timeout_seconds.unwrap_or_default()),
11737 &format!("{event_timeout}s"),
11738 "condition lifecycle events disagree on the recorded timeout",
11739 ));
11740 }
11741 }
11742 }
11743
11744 let timeout_timer_events = all_events
11745 .iter()
11746 .filter(|event| {
11747 matches!(
11748 event.event_type.as_str(),
11749 "TimerScheduled" | "TimerCancelled" | "TimerFired"
11750 ) && event.payload.get("timer_kind").and_then(Value::as_str)
11751 == Some("condition_timeout")
11752 && event
11753 .payload
11754 .get("condition_wait_id")
11755 .and_then(Value::as_str)
11756 == Some(condition_wait_id.as_str())
11757 })
11758 .collect::<Vec<_>>();
11759 let scheduled = timeout_timer_events
11760 .iter()
11761 .copied()
11762 .filter(|event| event.event_type == "TimerScheduled")
11763 .collect::<Vec<_>>();
11764 let fired = timeout_timer_events
11765 .iter()
11766 .copied()
11767 .filter(|event| event.event_type == "TimerFired")
11768 .collect::<Vec<_>>();
11769 if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
11770 return Err(invalid_recorded_history(
11771 "condition_wait_timeout_history_invalid",
11772 sequence,
11773 "one timeout schedule and at most one fire",
11774 &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
11775 "condition timeout history has a missing or duplicate lifecycle event",
11776 ));
11777 }
11778 if let Some(scheduled) = scheduled.first() {
11779 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
11780 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
11781 if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
11782 return Err(invalid_recorded_history(
11783 "condition_wait_timeout_delay_mismatch",
11784 sequence,
11785 &format!("{}s", timeout_seconds.unwrap_or_default()),
11786 &format!("{delay_seconds}s"),
11787 "condition timeout timer differs from the wait definition",
11788 ));
11789 }
11790 if let Some(fired) = fired.first() {
11791 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
11792 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
11793 if fired_timer_id != timer_id || fired_delay != delay_seconds {
11794 return Err(invalid_recorded_history(
11795 "condition_wait_timeout_identity_mismatch",
11796 sequence,
11797 &format!("{timer_id}:{delay_seconds}s"),
11798 &format!("{fired_timer_id}:{fired_delay}s"),
11799 "condition timeout fire does not match its durable schedule",
11800 ));
11801 }
11802 }
11803 }
11804
11805 let result = terminal.first().map(|event| {
11806 if event.event_type == "ConditionWaitTimedOut" {
11807 ConditionWaitResult::TimedOut
11808 } else {
11809 ConditionWaitResult::Satisfied
11810 }
11811 });
11812 let result = if !fired.is_empty() {
11813 if result == Some(ConditionWaitResult::Satisfied) {
11814 return Err(invalid_recorded_history(
11815 "condition_wait_terminal_conflict",
11816 sequence,
11817 "one satisfied or timed-out outcome",
11818 "satisfied event and fired timeout",
11819 "condition history records conflicting terminal outcomes",
11820 ));
11821 }
11822 Some(ConditionWaitResult::TimedOut)
11823 } else {
11824 result
11825 };
11826
11827 Ok(RecordedCommand::ConditionWait {
11828 sequence,
11829 occurrence_id,
11830 condition_key,
11831 predicate_identity,
11832 timeout_seconds,
11833 result,
11834 })
11835}
11836
11837fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
11838 event
11839 .payload
11840 .get("condition_wait_occurrence_id")
11841 .and_then(Value::as_str)
11842 .filter(|value| !value.is_empty())
11843 .map(str::to_string)
11844 .ok_or_else(|| {
11845 invalid_recorded_history(
11846 "condition_wait_occurrence_id_missing",
11847 sequence,
11848 "non-empty condition_wait_occurrence_id",
11849 &event.event_type,
11850 "condition history is missing authored occurrence identity",
11851 )
11852 })
11853}
11854
11855fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
11856 event
11857 .payload
11858 .get("condition_wait_id")
11859 .and_then(Value::as_str)
11860 .filter(|value| !value.is_empty())
11861 .map(str::to_string)
11862 .ok_or_else(|| {
11863 invalid_recorded_history(
11864 "condition_wait_id_missing",
11865 sequence,
11866 "non-empty condition_wait_id",
11867 &event.event_type,
11868 "canonical condition history is missing its durable wait identity",
11869 )
11870 })
11871}
11872
11873fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
11874 event
11875 .payload
11876 .get(field)
11877 .and_then(Value::as_str)
11878 .filter(|value| !value.is_empty())
11879 .map(str::to_string)
11880}
11881
11882fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
11883 match event.payload.get(field) {
11884 None | Some(Value::Null) => Ok(None),
11885 Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
11886 invalid_recorded_history(
11887 "condition_wait_definition_invalid",
11888 sequence,
11889 &format!("non-negative integer {field}"),
11890 &value.to_string(),
11891 "condition history contains an invalid numeric definition field",
11892 )
11893 }),
11894 }
11895}
11896
11897fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
11898 event
11899 .payload
11900 .get("signal_name")
11901 .or_else(|| event.raw.get("signal_name"))
11902 .and_then(Value::as_str)
11903 .filter(|value| !value.is_empty())
11904 .map(str::to_string)
11905 .ok_or_else(|| {
11906 invalid_recorded_history(
11907 "signal_wait_name_missing",
11908 sequence,
11909 "non-empty signal_name",
11910 &event.event_type,
11911 "canonical signal-wait history is missing its signal identity",
11912 )
11913 })
11914}
11915
11916fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
11917 matches!(
11918 event.event_type.as_str(),
11919 "SignalWaitOpened" | "SignalApplied"
11920 )
11921}
11922
11923fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
11924 event
11925 .payload
11926 .get(field)
11927 .and_then(Value::as_str)
11928 .filter(|value| !value.is_empty())
11929 .map(str::to_string)
11930 .ok_or_else(|| {
11931 invalid_recorded_history(
11932 "timer_history_field_missing",
11933 sequence,
11934 field,
11935 &event.event_type,
11936 "timer history is missing a required identity field",
11937 )
11938 })
11939}
11940
11941fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
11942 event
11943 .payload
11944 .get(field)
11945 .and_then(value_as_u64)
11946 .ok_or_else(|| {
11947 invalid_recorded_history(
11948 "timer_history_field_missing",
11949 sequence,
11950 field,
11951 &event.event_type,
11952 "timer history is missing a required numeric field",
11953 )
11954 })
11955}
11956
11957fn recorded_search_attribute_types(
11958 payload: &Value,
11959 attributes: &Value,
11960 sequence: u64,
11961) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
11962 let Some(raw_types) = payload.get("attribute_types") else {
11963 return Ok(RecordedSnapshotValue::Unknown);
11967 };
11968 let Some(raw_types) = raw_types.as_object() else {
11969 return Err(invalid_recorded_history(
11970 "search_attribute_types_malformed",
11971 sequence,
11972 "canonical attribute type map",
11973 &raw_types.to_string(),
11974 "search-attribute history contains malformed type identity",
11975 ));
11976 };
11977 let attribute_keys = attributes
11978 .as_object()
11979 .expect("recorded search attributes were validated as an object");
11980 let mut types = BTreeMap::new();
11981 for (key, value) in raw_types {
11982 let Some(attribute_type) = value.as_str() else {
11983 return Err(invalid_recorded_history(
11984 "search_attribute_types_malformed",
11985 sequence,
11986 "canonical string type name",
11987 &value.to_string(),
11988 "search-attribute history contains a non-string type identity",
11989 ));
11990 };
11991 if !attribute_keys.contains_key(key)
11992 || !matches!(
11993 attribute_type,
11994 "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
11995 )
11996 {
11997 return Err(invalid_recorded_history(
11998 "search_attribute_types_malformed",
11999 sequence,
12000 "canonical types for keys present in attributes",
12001 &format!("{key}:{attribute_type}"),
12002 "search-attribute history contains unsupported or orphaned type identity",
12003 ));
12004 }
12005 types.insert(key.clone(), attribute_type.to_string());
12006 }
12007 Ok(RecordedSnapshotValue::Known(types))
12008}
12009
12010fn invalid_recorded_history(
12011 reason: &str,
12012 sequence: u64,
12013 expected: &str,
12014 actual: &str,
12015 message: &str,
12016) -> Error {
12017 Error::NonDeterministicReplay(ReplayFailure::new(
12018 reason,
12019 Some(sequence),
12020 Some(expected.to_string()),
12021 Some(actual.to_string()),
12022 message,
12023 ))
12024}
12025
12026type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
12027
12028fn activity_outcome(
12029 event: &HistoryEvent,
12030 fallback_codec: &str,
12031 recorded_activity_type: Option<String>,
12032) -> Result<ActivityOutcome> {
12033 if event.event_type == "ActivityCompleted" {
12034 let codec = event
12035 .payload
12036 .get("payload_codec")
12037 .and_then(Value::as_str)
12038 .unwrap_or(fallback_codec);
12039 return Ok(Ok(decode_wire_avro_value(
12040 event.payload.get("result").unwrap_or(&Value::Null),
12041 codec,
12042 )?));
12043 }
12044
12045 let payload = &event.payload;
12046 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
12047 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
12048 "ActivityCancelled" => (
12049 ActivityFailureKind::Cancelled,
12050 "cancelled",
12051 "activity was cancelled",
12052 ),
12053 "ActivityTimedOut" => (
12054 ActivityFailureKind::TimedOut,
12055 "timeout",
12056 "activity timed out",
12057 ),
12058 _ => unreachable!("activity_outcome is called only for terminal activity events"),
12059 };
12060 let exception = payload
12061 .get("exception")
12062 .filter(|value| !value.is_null())
12063 .cloned();
12064 let failure_category = payload_string(payload, "failure_category");
12065 let timeout_kind = payload_string(payload, "timeout_kind");
12066 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
12067 ActivityFailureKind::Failed => failure_category
12068 .clone()
12069 .unwrap_or_else(|| fallback_reason.to_string()),
12070 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
12071 ActivityFailureKind::TimedOut => timeout_kind
12072 .clone()
12073 .unwrap_or_else(|| fallback_reason.to_string()),
12074 });
12075 let message = payload_string(payload, "message")
12076 .or_else(|| {
12077 exception
12078 .as_ref()
12079 .and_then(|value| payload_string(value, "message"))
12080 })
12081 .unwrap_or_else(|| fallback_message.to_string());
12082
12083 Ok(Err(ActivityFailure {
12084 kind,
12085 reason,
12086 message,
12087 activity_execution_id: payload_string(payload, "activity_execution_id"),
12088 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
12089 activity_type: payload_string(payload, "activity_type")
12090 .or_else(|| payload_string(payload, "activity_name"))
12091 .or(recorded_activity_type),
12092 activity_class: payload_string(payload, "activity_class"),
12093 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
12094 failure_id: payload_string(payload, "failure_id"),
12095 failure_category,
12096 timeout_kind,
12097 non_retryable: payload
12098 .get("non_retryable")
12099 .and_then(Value::as_bool)
12100 .unwrap_or(false),
12101 exception_type: payload_string(payload, "exception_type").or_else(|| {
12102 exception
12103 .as_ref()
12104 .and_then(|value| payload_string(value, "type"))
12105 }),
12106 exception_class: payload_string(payload, "exception_class").or_else(|| {
12107 exception
12108 .as_ref()
12109 .and_then(|value| payload_string(value, "class"))
12110 }),
12111 code: payload
12112 .get("code")
12113 .filter(|value| !value.is_null())
12114 .cloned(),
12115 exception,
12116 }))
12117}
12118
12119type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
12120
12121fn child_workflow_outcomes(
12122 events: &[HistoryEvent],
12123 fallback_codec: &str,
12124 parent: WorkflowIdentity,
12125) -> Result<Vec<ChildWorkflowOutcome>> {
12126 let mut outcomes = Vec::new();
12127
12128 for event in events {
12129 let kind = match event.event_type.as_str() {
12130 "ChildRunCompleted" => None,
12131 "ChildRunFailed" => Some((
12132 ChildWorkflowFailureKind::Failed,
12133 "child_workflow",
12134 "child workflow failed",
12135 )),
12136 "ChildRunCancelled" => Some((
12137 ChildWorkflowFailureKind::Cancelled,
12138 "cancelled",
12139 "child workflow was cancelled",
12140 )),
12141 "ChildRunTerminated" => Some((
12142 ChildWorkflowFailureKind::Terminated,
12143 "terminated",
12144 "child workflow was terminated",
12145 )),
12146 _ => continue,
12147 };
12148 let payload = &event.payload;
12149 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
12150 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
12151 let child_workflow_type = payload_string(payload, "child_workflow_type");
12152
12153 if let Some((kind, reason, fallback_message)) = kind {
12154 let exception = payload
12155 .get("exception")
12156 .filter(|value| !value.is_null())
12157 .cloned();
12158 let message = payload_string(payload, "message")
12159 .or_else(|| {
12160 exception
12161 .as_ref()
12162 .and_then(|value| payload_string(value, "message"))
12163 })
12164 .unwrap_or_else(|| fallback_message.to_string());
12165 let exception_type = payload_string(payload, "exception_type").or_else(|| {
12166 exception
12167 .as_ref()
12168 .and_then(|value| payload_string(value, "type"))
12169 });
12170 let exception_class = payload_string(payload, "exception_class").or_else(|| {
12171 exception
12172 .as_ref()
12173 .and_then(|value| payload_string(value, "class"))
12174 });
12175 outcomes.push(Err(ChildWorkflowFailure {
12176 kind,
12177 reason: reason.to_string(),
12178 message,
12179 parent_workflow_id: parent.workflow_id.clone(),
12180 parent_workflow_run_id: parent.run_id.clone(),
12181 child_workflow_id,
12182 child_workflow_run_id,
12183 child_workflow_type,
12184 failure_id: payload_string(payload, "failure_id"),
12185 failure_category: payload_string(payload, "failure_category"),
12186 exception_type,
12187 exception_class,
12188 non_retryable: payload
12189 .get("non_retryable")
12190 .and_then(Value::as_bool)
12191 .unwrap_or(false),
12192 code: payload
12193 .get("code")
12194 .filter(|value| !value.is_null())
12195 .cloned(),
12196 exception,
12197 }));
12198 continue;
12199 }
12200
12201 let codec = payload
12202 .get("payload_codec")
12203 .and_then(Value::as_str)
12204 .unwrap_or(fallback_codec);
12205 let result = payload
12206 .get("result")
12207 .or_else(|| payload.get("output"))
12208 .unwrap_or(&Value::Null);
12209 outcomes.push(Ok(ChildWorkflowAvroResult {
12210 parent: parent.clone(),
12211 child: WorkflowIdentity {
12212 workflow_id: child_workflow_id,
12213 run_id: child_workflow_run_id,
12214 },
12215 child_workflow_type,
12216 result: decode_wire_avro_value(result, codec)?,
12217 }));
12218 }
12219
12220 Ok(outcomes)
12221}
12222
12223fn payload_string(payload: &Value, key: &str) -> Option<String> {
12224 payload
12225 .get(key)
12226 .and_then(Value::as_str)
12227 .filter(|value| !value.is_empty())
12228 .map(str::to_string)
12229}
12230
12231fn workflow_failure_command(error: &Error) -> Value {
12232 let (exception_type, exception_class, properties) = match error {
12233 Error::ActivityFailed(failure) => (
12234 match failure.kind {
12235 ActivityFailureKind::Failed => "ActivityFailed",
12236 ActivityFailureKind::Cancelled => "ActivityCancelled",
12237 ActivityFailureKind::TimedOut => "ActivityTimedOut",
12238 },
12239 "durable_workflow::ActivityFailure",
12240 json!({
12241 "reason": failure.reason,
12242 "activity_execution_id": failure.activity_execution_id,
12243 "activity_attempt_id": failure.activity_attempt_id,
12244 "activity_type": failure.activity_type,
12245 "activity_class": failure.activity_class,
12246 "attempt_number": failure.attempt_number,
12247 "failure_id": failure.failure_id,
12248 "failure_category": failure.failure_category,
12249 "timeout_kind": failure.timeout_kind,
12250 "activity_non_retryable": failure.non_retryable,
12251 "activity_exception_type": failure.exception_type,
12252 "activity_exception_class": failure.exception_class,
12253 "activity_code": failure.code,
12254 "activity_exception": failure.exception,
12255 }),
12256 ),
12257 Error::ChildWorkflowFailed(failure) => (
12258 match failure.kind {
12259 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
12260 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
12261 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
12262 },
12263 "durable_workflow::ChildWorkflowFailure",
12264 json!({
12265 "reason": failure.reason,
12266 "parent_workflow_id": failure.parent_workflow_id,
12267 "parent_workflow_run_id": failure.parent_workflow_run_id,
12268 "child_workflow_id": failure.child_workflow_id,
12269 "child_workflow_run_id": failure.child_workflow_run_id,
12270 "child_workflow_type": failure.child_workflow_type,
12271 "failure_id": failure.failure_id,
12272 "failure_category": failure.failure_category,
12273 "child_exception_type": failure.exception_type,
12274 "child_exception_class": failure.exception_class,
12275 "child_non_retryable": failure.non_retryable,
12276 "child_code": failure.code,
12277 "child_exception": failure.exception,
12278 }),
12279 ),
12280 Error::ParallelFailed(failure) => (
12281 "ParallelFailed",
12282 "durable_workflow::ParallelFailure",
12283 json!({
12284 "parallel_group_id": failure.group_id,
12285 "parallel_member_path": failure.member_path,
12286 "parallel_group_path": failure.group_path,
12287 "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
12288 "cause_type": workflow_error_type(&failure.cause),
12289 "cause_message": failure.cause.to_string(),
12290 }),
12291 ),
12292 Error::SagaCompensationFailed(failure) => (
12293 "SagaCompensationFailed",
12294 "durable_workflow::SagaCompensationFailure",
12295 json!({
12296 "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
12297 "initiating_failure_message": failure.initiating_failure.to_string(),
12298 "compensation_activity_type": failure.compensation_activity_type,
12299 "compensation_registration_order": failure.compensation_registration_order,
12300 "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
12301 "compensation_failure_message": failure.compensation_failure.to_string(),
12302 }),
12303 ),
12304 Error::WorkflowCancellationRequested(_) => (
12305 "WorkflowCancellationRequested",
12306 "durable_workflow::WorkflowCancellationRequested",
12307 json!({"reason": "cancelled"}),
12308 ),
12309 Error::NonDeterministicReplay(_) => (
12310 "NonDeterministicReplay",
12311 "durable_workflow::Error",
12312 Value::Null,
12313 ),
12314 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
12315 };
12316 let non_retryable = match error {
12317 Error::ActivityFailed(failure) => failure.non_retryable,
12318 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
12319 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
12320 Error::SagaCompensationFailed(failure) => {
12321 workflow_error_non_retryable(&failure.compensation_failure)
12322 }
12323 Error::WorkflowCancellationRequested(_) => true,
12324 Error::NonDeterministicReplay(_) => true,
12325 _ => false,
12326 };
12327
12328 json!({
12329 "type": "fail_workflow",
12330 "message": error.to_string(),
12331 "exception_type": exception_type,
12332 "exception_class": exception_class,
12333 "non_retryable": non_retryable,
12334 "exception": {
12335 "type": exception_type,
12336 "class": exception_class,
12337 "message": error.to_string(),
12338 "properties": properties,
12339 }
12340 })
12341}
12342
12343fn workflow_error_type(error: &Error) -> &'static str {
12344 match error {
12345 Error::ActivityFailed(failure) => match failure.kind {
12346 ActivityFailureKind::Failed => "ActivityFailed",
12347 ActivityFailureKind::Cancelled => "ActivityCancelled",
12348 ActivityFailureKind::TimedOut => "ActivityTimedOut",
12349 },
12350 Error::ChildWorkflowFailed(failure) => match failure.kind {
12351 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
12352 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
12353 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
12354 },
12355 Error::ParallelFailed(_) => "ParallelFailed",
12356 Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
12357 Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
12358 Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
12359 _ => "RustWorkflowError",
12360 }
12361}
12362
12363fn workflow_error_non_retryable(error: &Error) -> bool {
12364 match error {
12365 Error::ActivityFailed(failure) => failure.non_retryable,
12366 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
12367 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
12368 Error::SagaCompensationFailed(failure) => {
12369 workflow_error_non_retryable(&failure.compensation_failure)
12370 }
12371 Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
12372 _ => false,
12373 }
12374}
12375
12376fn workflow_task_integrity_error(error: &Error) -> bool {
12377 matches!(
12378 error,
12379 Error::NonDeterministicReplay(_)
12380 | Error::Protocol(_)
12381 | Error::MissingWorkflowCommandIdentity
12382 | Error::WorkflowStatePoisoned
12383 )
12384}
12385
12386fn decode_signal_event_arguments(
12387 event: &HistoryEvent,
12388 fallback_codec: &str,
12389) -> Result<Vec<AvroValue>> {
12390 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
12391 validate_payload_codec(codec)?;
12392 let raw = signal_history_payload(&event.payload);
12393 let decoded = match raw.filter(|value| !value.is_null()) {
12394 Some(value) => decode_wire_avro_value(value, codec)?,
12395 None => AvroValue::Array(Vec::new()),
12396 };
12397 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
12398 unreachable!("normalize_avro_arguments always returns an array");
12399 };
12400 Ok(arguments)
12401}
12402
12403fn decode_update_event_arguments(
12404 event: &HistoryEvent,
12405 fallback_codec: &str,
12406) -> Result<Vec<AvroValue>> {
12407 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
12408 validate_payload_codec(codec)?;
12409 let decoded = match event
12410 .payload
12411 .get("arguments")
12412 .filter(|value| !value.is_null())
12413 {
12414 Some(value) => decode_wire_avro_value(value, codec)?,
12415 None => AvroValue::Array(Vec::new()),
12416 };
12417 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
12418 unreachable!("normalize_avro_arguments always returns an array");
12419 };
12420 Ok(arguments)
12421}
12422
12423fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
12424 let Some(export_events) = task
12425 .history_export
12426 .as_ref()
12427 .and_then(|export| export.get("history_events"))
12428 .and_then(Value::as_array)
12429 else {
12430 return Ok(());
12431 };
12432
12433 if export_events.len() > task.history_events.len() {
12434 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
12435 }
12436
12437 Ok(())
12438}
12439
12440fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
12441 let Some(export) = task.history_export.as_ref() else {
12442 return Ok(());
12443 };
12444 let signals = export
12445 .get("signals")
12446 .and_then(Value::as_array)
12447 .cloned()
12448 .unwrap_or_default();
12449 let activities = export
12450 .get("activities")
12451 .and_then(Value::as_array)
12452 .cloned()
12453 .unwrap_or_default();
12454 let export_codec = export
12455 .get("payloads")
12456 .and_then(|payloads| payloads.get("codec"))
12457 .and_then(Value::as_str)
12458 .unwrap_or(&task.payload_codec)
12459 .to_string();
12460 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
12461
12462 for event in &mut task.history_events {
12463 if event.event_type == "ActivityCompleted" {
12464 let sequence = event
12465 .payload
12466 .get("sequence")
12467 .or_else(|| event.payload.get("workflow_sequence"))
12468 .and_then(value_as_u64);
12469 let Some(activity) = sequence.and_then(|sequence| {
12470 activities.iter().find(|activity| {
12471 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
12472 })
12473 }) else {
12474 continue;
12475 };
12476 let Some(payload) = event.payload.as_object_mut() else {
12477 continue;
12478 };
12479 if missing_payload(payload.get("result")) {
12480 if let Some(result) = activity
12481 .get("result")
12482 .filter(|value| !missing_payload(Some(value)))
12483 {
12484 payload.insert("result".to_string(), result.clone());
12485 }
12486 }
12487 for field in ["payload_codec", "activity_type"] {
12488 if payload
12489 .get(field)
12490 .and_then(Value::as_str)
12491 .unwrap_or_default()
12492 .is_empty()
12493 {
12494 if let Some(value) = activity.get(field) {
12495 payload.insert(field.to_string(), value.clone());
12496 }
12497 }
12498 }
12499 continue;
12500 }
12501
12502 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
12503 continue;
12504 }
12505 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
12506 let command_id = event
12507 .payload
12508 .get("workflow_command_id")
12509 .or_else(|| event.raw.get("workflow_command_id"))
12510 .and_then(Value::as_str);
12511 let signal_name = event
12512 .payload
12513 .get("signal_name")
12514 .and_then(Value::as_str)
12515 .unwrap_or_default()
12516 .to_string();
12517 let matched = signals
12518 .iter()
12519 .find(|signal| {
12520 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
12521 })
12522 .or_else(|| {
12523 signals.iter().find(|signal| {
12524 command_id.is_some()
12525 && signal.get("command_id").and_then(Value::as_str) == command_id
12526 })
12527 })
12528 .or_else(|| {
12529 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
12530 let signal = signals
12531 .iter()
12532 .filter(|signal| {
12533 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
12534 })
12535 .nth(*offset);
12536 if signal.is_some() {
12537 *offset += 1;
12538 }
12539 signal
12540 });
12541 let Some(signal) = matched else {
12542 continue;
12543 };
12544 let signal_codec = signal
12545 .get("payload_codec")
12546 .and_then(Value::as_str)
12547 .unwrap_or(&export_codec);
12548 let Some(payload) = event.payload.as_object_mut() else {
12549 continue;
12550 };
12551 if missing_payload(payload.get("arguments")) {
12552 if let Some(arguments) = signal
12553 .get("arguments")
12554 .filter(|value| !missing_payload(Some(value)))
12555 {
12556 let envelope = match arguments {
12557 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
12558 other => other.clone(),
12559 };
12560 payload.insert("arguments".to_string(), envelope);
12561 }
12562 }
12563 if payload
12564 .get("payload_codec")
12565 .and_then(Value::as_str)
12566 .unwrap_or_default()
12567 .is_empty()
12568 {
12569 payload.insert("payload_codec".to_string(), json!(signal_codec));
12570 }
12571 }
12572
12573 Ok(())
12574}
12575
12576fn missing_payload(value: Option<&Value>) -> bool {
12577 match value {
12578 None | Some(Value::Null) => true,
12579 Some(Value::String(value)) => value.is_empty(),
12580 Some(_) => false,
12581 }
12582}
12583
12584fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
12585 let export_signals = task
12586 .history_export
12587 .as_ref()
12588 .and_then(|export| export.get("signals"))
12589 .and_then(Value::as_array)
12590 .cloned()
12591 .unwrap_or_default();
12592 let export_codec = task
12593 .history_export
12594 .as_ref()
12595 .and_then(|export| export.get("payloads"))
12596 .and_then(|payloads| payloads.get("codec"))
12597 .and_then(Value::as_str)
12598 .unwrap_or(&task.payload_codec);
12599 let mut name_offsets: HashMap<String, usize> = HashMap::new();
12600 let mut signals = Vec::new();
12601
12602 for event in &task.history_events {
12603 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
12604 continue;
12605 }
12606
12607 let name = event
12608 .payload
12609 .get("signal_name")
12610 .and_then(Value::as_str)
12611 .unwrap_or_default();
12612 if name.is_empty() {
12613 continue;
12614 }
12615 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
12616 let command_id = event
12617 .payload
12618 .get("workflow_command_id")
12619 .or_else(|| event.raw.get("workflow_command_id"))
12620 .and_then(Value::as_str);
12621 let matched_export = export_signals
12622 .iter()
12623 .find(|candidate| {
12624 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
12625 })
12626 .or_else(|| {
12627 export_signals.iter().find(|candidate| {
12628 command_id.is_some()
12629 && candidate.get("command_id").and_then(Value::as_str) == command_id
12630 })
12631 })
12632 .or_else(|| {
12633 let offset = name_offsets.entry(name.to_string()).or_default();
12634 let candidate = export_signals
12635 .iter()
12636 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
12637 .nth(*offset);
12638 if candidate.is_some() {
12639 *offset += 1;
12640 }
12641 candidate
12642 });
12643 let codec = event
12644 .payload
12645 .get("payload_codec")
12646 .and_then(Value::as_str)
12647 .or_else(|| {
12648 matched_export
12649 .and_then(|signal| signal.get("payload_codec"))
12650 .and_then(Value::as_str)
12651 })
12652 .unwrap_or(export_codec);
12653 let raw_arguments = signal_history_payload(&event.payload)
12654 .filter(|value| !value.is_null())
12655 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
12656 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
12657 let workflow_sequence = event
12658 .payload
12659 .get("workflow_sequence")
12660 .and_then(value_as_u64)
12661 .or_else(|| {
12662 matched_export
12663 .and_then(|signal| signal.get("workflow_sequence"))
12664 .and_then(value_as_u64)
12665 });
12666
12667 signals.push(QuerySignal {
12668 id: signal_id.map(str::to_string).or_else(|| {
12669 matched_export
12670 .and_then(|signal| signal.get("id"))
12671 .and_then(Value::as_str)
12672 .map(str::to_string)
12673 }),
12674 name: name.to_string(),
12675 arguments,
12676 avro_arguments,
12677 workflow_sequence,
12678 });
12679 }
12680
12681 if signals.is_empty() {
12682 for signal in export_signals {
12683 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
12684 continue;
12685 }
12686 let Some(name) = signal.get("name").and_then(Value::as_str) else {
12687 continue;
12688 };
12689 let codec = signal
12690 .get("payload_codec")
12691 .and_then(Value::as_str)
12692 .unwrap_or(export_codec);
12693 let (arguments, avro_arguments) =
12694 decode_query_signal_arguments(signal.get("arguments"), codec)?;
12695 signals.push(QuerySignal {
12696 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
12697 name: name.to_string(),
12698 arguments,
12699 avro_arguments,
12700 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
12701 });
12702 }
12703 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
12704 }
12705
12706 Ok(signals)
12707}
12708
12709fn decode_query_signal_arguments(
12710 raw: Option<&Value>,
12711 codec: &str,
12712) -> Result<(Vec<Value>, Vec<AvroValue>)> {
12713 validate_payload_codec(codec)?;
12714 let decoded = match raw.filter(|value| !value.is_null()) {
12715 Some(value) => decode_wire_avro_value(value, codec)?,
12716 None => AvroValue::Array(Vec::new()),
12717 };
12718 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
12719 unreachable!("normalize_avro_arguments always returns an array");
12720 };
12721 let arguments = avro_arguments
12722 .iter()
12723 .cloned()
12724 .map(AvroValue::into_json)
12725 .collect::<Result<Vec<_>>>()?;
12726 Ok((arguments, avro_arguments))
12727}
12728
12729fn value_as_u64(value: &Value) -> Option<u64> {
12730 value
12731 .as_u64()
12732 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
12733}
12734
12735#[cfg(test)]
12736mod tests {
12737 use super::*;
12738 use std::{
12739 io::{Read, Write},
12740 net::{SocketAddr, TcpListener, TcpStream},
12741 sync::atomic::AtomicUsize,
12742 thread,
12743 };
12744
12745 #[derive(Clone, Copy, Debug)]
12746 enum InvalidTaskPayloadCodec {
12747 Missing,
12748 Null,
12749 NonString,
12750 }
12751
12752 impl InvalidTaskPayloadCodec {
12753 fn label(self) -> &'static str {
12754 match self {
12755 Self::Missing => "missing",
12756 Self::Null => "null",
12757 Self::NonString => "non-string",
12758 }
12759 }
12760
12761 fn apply(self, task: &mut Value) {
12762 let task = task.as_object_mut().expect("task fixture object");
12763 match self {
12764 Self::Missing => {
12765 task.remove("payload_codec");
12766 }
12767 Self::Null => {
12768 task.insert("payload_codec".to_string(), Value::Null);
12769 }
12770 Self::NonString => {
12771 task.insert("payload_codec".to_string(), json!(42));
12772 }
12773 }
12774 }
12775 }
12776
12777 fn fixture_envelope(value: Value) -> Value {
12778 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
12779 }
12780
12781 fn fixture_blob(value: Value) -> String {
12782 encode_payload(&value, DEFAULT_CODEC)
12783 .expect("encode Avro test fixture")
12784 .blob
12785 }
12786
12787 #[test]
12788 fn client_builder_rejects_the_sdk_owned_api_suffix() {
12789 for base_url in [
12790 "http://127.0.0.1:8080/api",
12791 "http://localhost:8080/api/",
12792 "https://runtime.example.test/namespaces/orders/api",
12793 ] {
12794 let error = Client::builder(base_url)
12795 .build()
12796 .expect_err("SDK-owned /api suffix must be rejected during build");
12797
12798 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
12799 assert!(
12800 error.to_string().contains("SDK appends /api automatically"),
12801 "the validation error must explain how to fix the endpoint"
12802 );
12803 }
12804 }
12805
12806 #[test]
12807 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
12808 for (base_url, expected) in [
12809 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
12810 (
12811 "http://localhost:8080/durable-workflow/",
12812 "http://localhost:8080/durable-workflow",
12813 ),
12814 (
12815 "https://runtime.example.test/namespaces/orders",
12816 "https://runtime.example.test/namespaces/orders",
12817 ),
12818 (
12819 "https://runtime.example.test/gateway/api/namespaces/orders",
12820 "https://runtime.example.test/gateway/api/namespaces/orders",
12821 ),
12822 (
12823 "https://api.example.test/runtime/orders/",
12824 "https://api.example.test/runtime/orders",
12825 ),
12826 ] {
12827 let client = Client::builder(base_url)
12828 .build()
12829 .expect("Server and Cloud runtime base URL must remain valid");
12830
12831 assert_eq!(client.base_url, expected);
12832 }
12833 }
12834
12835 #[test]
12836 fn workflow_completion_uses_the_additive_command_protocol_floor() {
12837 assert_eq!(
12838 workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
12839 WORKER_PROTOCOL_VERSION
12840 );
12841 assert_eq!(
12842 workflow_completion_protocol_version(&[json!({
12843 "type": "upsert_search_attributes",
12844 "attributes": {"OrderStatus": "waiting"},
12845 })]),
12846 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
12847 );
12848 assert_eq!(
12849 workflow_completion_protocol_version(&[json!({
12850 "type": "upsert_search_attributes",
12851 "attributes": {"OrderStatus": "waiting"},
12852 "attribute_types": {"OrderStatus": "keyword"},
12853 })]),
12854 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
12855 );
12856 assert_eq!(
12857 workflow_completion_protocol_version(&[
12858 json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
12859 json!({"type": "open_condition_wait", "condition_key": "ready"}),
12860 ]),
12861 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
12862 );
12863 assert_eq!(
12864 workflow_completion_protocol_version(&[
12865 json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
12866 json!({"type": "open_condition_wait", "condition_key": "ready"}),
12867 ]),
12868 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
12869 );
12870 assert_eq!(
12871 workflow_completion_protocol_version(&[json!({
12872 "type": "open_condition_wait",
12873 "condition_wait_occurrence_id": "rust:condition-wait:0",
12874 "condition_key": "ready",
12875 })]),
12876 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
12877 );
12878 assert_eq!(
12879 workflow_completion_protocol_version_with_message_streams(
12880 &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
12881 true,
12882 ),
12883 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
12884 );
12885 assert_eq!(
12886 workflow_completion_protocol_version_with_message_streams(
12887 &[json!({
12888 "type": "open_condition_wait",
12889 "condition_wait_occurrence_id": "rust:condition-wait:0",
12890 "condition_key": "ready",
12891 })],
12892 true,
12893 ),
12894 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
12895 );
12896 }
12897
12898 fn typed_fidelity_probe() -> AvroValue {
12899 AvroValue::Map(BTreeMap::from([
12900 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
12901 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
12902 (
12903 "numeric".to_string(),
12904 AvroValue::Map(BTreeMap::from([
12905 ("0".to_string(), AvroValue::String("zero".to_string())),
12906 ("1".to_string(), AvroValue::String("one".to_string())),
12907 ])),
12908 ),
12909 (
12910 "nested".to_string(),
12911 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
12912 "enabled".to_string(),
12913 AvroValue::Boolean(true),
12914 )]))]),
12915 ),
12916 (
12917 "projection_collisions".to_string(),
12918 AvroValue::Array(projection_collision_probe()),
12919 ),
12920 ]))
12921 }
12922
12923 fn projection_collision_probe() -> Vec<AvroValue> {
12924 vec![
12925 AvroValue::Map(BTreeMap::from([
12926 ("$type".to_string(), AvroValue::String("bytes".to_string())),
12927 (
12928 "base64".to_string(),
12929 AvroValue::String("ordinary user text".to_string()),
12930 ),
12931 ])),
12932 AvroValue::Map(BTreeMap::from([
12933 ("$type".to_string(), AvroValue::String("map".to_string())),
12934 (
12935 "entries".to_string(),
12936 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
12937 ("key".to_string(), AvroValue::String("ordinary".to_string())),
12938 (
12939 "value".to_string(),
12940 AvroValue::String("user map".to_string()),
12941 ),
12942 ]))]),
12943 ),
12944 ])),
12945 ]
12946 }
12947
12948 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12949 struct TypedContract {
12950 nested: TypedNested,
12951 mode: TypedMode,
12952 optional: Option<String>,
12953 absent: Option<String>,
12954 items: Vec<i64>,
12955 labels: BTreeMap<String, String>,
12956 bytes: serde_bytes::ByteBuf,
12957 signed: i64,
12958 finite: f64,
12959 }
12960
12961 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12962 struct TypedNested {
12963 enabled: bool,
12964 }
12965
12966 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12967 enum TypedMode {
12968 Detailed { label: String },
12969 }
12970
12971 fn typed_contract() -> TypedContract {
12972 TypedContract {
12973 nested: TypedNested { enabled: true },
12974 mode: TypedMode::Detailed {
12975 label: "compiler-checked".to_string(),
12976 },
12977 optional: Some("present".to_string()),
12978 absent: None,
12979 items: vec![i64::MIN, 0, i64::MAX],
12980 labels: BTreeMap::from([
12981 ("language".to_string(), "rust".to_string()),
12982 ("wire".to_string(), "avro".to_string()),
12983 ]),
12984 bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
12985 signed: -9_223_372_036_854_775_000,
12986 finite: 12.5,
12987 }
12988 }
12989
12990 #[derive(Clone, Debug, Default, PartialEq)]
12991 struct ReplayCounterState {
12992 loaded: Option<String>,
12993 count: i64,
12994 finished: bool,
12995 }
12996
12997 fn replay_counter_worker() -> Worker {
12998 let client = Client::new("http://127.0.0.1:8080").expect("client");
12999 let mut worker = Worker::new(client, "rust-workers");
13000 worker.register_replayed_workflow(
13001 "replay-counter",
13002 ReplayCounterState::default,
13003 |ctx, _input, state| async move {
13004 let loaded = ctx.activity("load-counter", json!([])).await?;
13005 state.update(|current| {
13006 current.loaded = loaded.as_str().map(str::to_string);
13007 })?;
13008 for _ in 0..2 {
13009 let signal = ctx.wait_signal("increment").await?;
13010 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
13011 state.update(|current| current.count += amount)?;
13012 }
13013 state.update(|current| current.finished = true)?;
13014 state.read(|current| Ok(json!(current.count)))?
13015 },
13016 );
13017 worker.register_replayed_query::<ReplayCounterState, _, _>(
13018 "replay-counter",
13019 "current",
13020 |_ctx, state, _args| async move {
13021 Ok(json!({
13022 "loaded": state.loaded,
13023 "count": state.count,
13024 "finished": state.finished,
13025 }))
13026 },
13027 );
13028 worker.register_replayed_query::<ReplayCounterState, _, _>(
13029 "replay-counter",
13030 "detached-mutation",
13031 |_ctx, state, _args| async move {
13032 let mut detached = (*state).clone();
13033 detached.count = 999;
13034 Ok(json!(detached.count))
13035 },
13036 );
13037 worker.register_replayed_query::<ReplayCounterState, _, _>(
13038 "replay-counter",
13039 "failed-mutation",
13040 |_ctx, state, _args| async move {
13041 let mut detached = (*state).clone();
13042 detached.count = 999;
13043 Err(Error::WorkerLoop("query refused".to_string()))
13044 },
13045 );
13046 worker
13047 }
13048
13049 fn replay_counter_query(
13050 query_name: &str,
13051 history_events: Value,
13052 run_status: &str,
13053 ) -> QueryTask {
13054 let arguments = fixture_envelope(json!([]));
13055 serde_json::from_value(json!({
13056 "query_task_id": format!("query-{query_name}"),
13057 "workflow_type": "replay-counter",
13058 "query_name": query_name,
13059 "payload_codec": DEFAULT_CODEC,
13060 "workflow_arguments": arguments.clone(),
13061 "query_arguments": arguments,
13062 "history_events": history_events,
13063 "run_status": run_status,
13064 }))
13065 .expect("query task")
13066 }
13067
13068 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
13069 workflow_context_with_codec(history, DEFAULT_CODEC)
13070 }
13071
13072 fn workflow_context_with_codec(
13073 history: Vec<HistoryEvent>,
13074 payload_codec: &str,
13075 ) -> WorkflowContext {
13076 WorkflowContext {
13077 state: Arc::new(Mutex::new(
13078 WorkflowState::new_with_identity(
13079 history,
13080 None,
13081 None,
13082 "rust-workers".to_string(),
13083 payload_codec.to_string(),
13084 None,
13085 )
13086 .expect("valid workflow history"),
13087 )),
13088 }
13089 }
13090
13091 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
13092 HistoryEvent {
13093 event_type: event_type.to_string(),
13094 payload,
13095 raw: HashMap::new(),
13096 }
13097 }
13098
13099 fn parallel_path_entry(
13100 kind: &str,
13101 base: u64,
13102 size: usize,
13103 index: usize,
13104 ) -> ParallelGroupMetadata {
13105 parallel_group_entry(base, size, index, kind)
13106 }
13107
13108 fn parallel_history_event(
13109 event_type: &str,
13110 sequence: u64,
13111 identity_field: &str,
13112 identity: &str,
13113 path: Vec<ParallelGroupMetadata>,
13114 result: Option<Value>,
13115 ) -> HistoryEvent {
13116 let mut payload = serde_json::Map::from_iter([
13117 ("sequence".to_string(), json!(sequence)),
13118 (identity_field.to_string(), json!(identity)),
13119 ]);
13120 let inner = path.last().expect("parallel history path");
13121 apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
13122 payload.insert("parallel_group_path".to_string(), json!(path));
13123 if let Some(result) = result {
13124 let field = if event_type == "ChildRunCompleted" {
13125 "result"
13126 } else {
13127 "result"
13128 };
13129 payload.insert(field.to_string(), fixture_envelope(result));
13130 payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
13131 }
13132 history_event(event_type, Value::Object(payload))
13133 }
13134
13135 fn nested_parallel_operations() -> Vec<ParallelOperation> {
13136 vec![
13137 ParallelOperation::activity("first", json!([])),
13138 ParallelOperation::group(vec![
13139 ParallelOperation::child_workflow(
13140 "second",
13141 ChildWorkflowOptions::new("child-workers"),
13142 json!([]),
13143 ),
13144 ParallelOperation::activity("third", json!([])),
13145 ]),
13146 ]
13147 }
13148
13149 fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
13150 let outer = [
13151 parallel_path_entry("mixed", 1, 3, 0),
13152 parallel_path_entry("mixed", 1, 3, 1),
13153 parallel_path_entry("mixed", 1, 3, 2),
13154 ];
13155 [
13156 vec![outer[0].clone()],
13157 vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
13158 vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
13159 ]
13160 }
13161
13162 #[test]
13163 fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
13164 let ctx = workflow_context(Vec::new());
13165 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
13166 let mut task_context = TaskContext::from_waker(noop_waker_ref());
13167
13168 assert!(matches!(
13169 call.as_mut().poll(&mut task_context),
13170 Poll::Pending
13171 ));
13172 let commands = ctx.take_commands().expect("parallel commands");
13173 assert_eq!(
13174 commands
13175 .iter()
13176 .map(|command| command["type"].as_str().unwrap_or_default())
13177 .collect::<Vec<_>>(),
13178 [
13179 "schedule_activity",
13180 "start_child_workflow",
13181 "schedule_activity"
13182 ]
13183 );
13184 let paths = nested_parallel_paths();
13185 for (command, path) in commands.iter().zip(paths) {
13186 assert_eq!(command["parallel_group_path"], json!(path));
13187 assert_eq!(
13188 command["parallel_group_id"],
13189 json!(path.last().expect("inner group").parallel_group_id)
13190 );
13191 }
13192 }
13193
13194 fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
13195 let paths = nested_parallel_paths();
13196 let third = parallel_history_event(
13197 "ActivityCompleted",
13198 3,
13199 "activity_type",
13200 "third",
13201 paths[2].clone(),
13202 Some(json!("three")),
13203 );
13204 vec![
13205 parallel_history_event(
13206 "ActivityCompleted",
13207 1,
13208 "activity_type",
13209 "first",
13210 paths[0].clone(),
13211 Some(json!("one")),
13212 ),
13213 parallel_history_event(
13214 "ChildWorkflowScheduled",
13215 2,
13216 "child_workflow_type",
13217 "second",
13218 paths[1].clone(),
13219 None,
13220 ),
13221 parallel_history_event(
13222 "ChildRunCompleted",
13223 2,
13224 "child_workflow_type",
13225 "second",
13226 paths[1].clone(),
13227 Some(json!("two")),
13228 ),
13229 third.clone(),
13230 third,
13231 ]
13232 }
13233
13234 #[test]
13235 fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
13236 for _restart_or_completed_replay in 0..2 {
13237 let ctx = workflow_context(completed_nested_parallel_history());
13238 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
13239 let mut task_context = TaskContext::from_waker(noop_waker_ref());
13240 let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
13241 panic!("completed nested parallel history must replay");
13242 };
13243 assert_eq!(
13244 results,
13245 vec![
13246 ParallelResult::Activity(json!("one")),
13247 ParallelResult::Group(vec![
13248 ParallelResult::ChildWorkflow(ChildWorkflowResult {
13249 parent: WorkflowIdentity {
13250 workflow_id: None,
13251 run_id: None,
13252 },
13253 child: WorkflowIdentity {
13254 workflow_id: None,
13255 run_id: None,
13256 },
13257 child_workflow_type: Some("second".to_string()),
13258 result: json!("two"),
13259 }),
13260 ParallelResult::Activity(json!("three")),
13261 ]),
13262 ]
13263 );
13264 assert!(ctx.take_commands().expect("commands").is_empty());
13265 ctx.ensure_history_consumed().expect("history consumed");
13266 }
13267 }
13268
13269 #[test]
13270 fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
13271 let paths = nested_parallel_paths();
13272 let history = vec![
13273 parallel_history_event(
13274 "ActivityCompleted",
13275 1,
13276 "activity_type",
13277 "first",
13278 paths[0].clone(),
13279 Some(json!("one")),
13280 ),
13281 parallel_history_event(
13282 "ChildWorkflowScheduled",
13283 2,
13284 "child_workflow_type",
13285 "second",
13286 paths[1].clone(),
13287 None,
13288 ),
13289 parallel_history_event(
13290 "ChildRunFailed",
13291 2,
13292 "child_workflow_type",
13293 "second",
13294 paths[1].clone(),
13295 None,
13296 ),
13297 parallel_history_event(
13298 "ActivityCompleted",
13299 3,
13300 "activity_type",
13301 "third",
13302 paths[2].clone(),
13303 Some(json!("late")),
13304 ),
13305 ];
13306 let ctx = workflow_context(history);
13307 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
13308 let mut task_context = TaskContext::from_waker(noop_waker_ref());
13309 let outcome = call.as_mut().poll(&mut task_context);
13310 let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
13311 panic!("one failed child must return a typed partial failure: {outcome:?}");
13312 };
13313 assert_eq!(failure.member_path, [1, 0]);
13314 assert_eq!(failure.group_id, "parallel-calls:1:3");
13315 assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
13316 assert_eq!(
13317 failure
13318 .completed
13319 .iter()
13320 .map(|completion| completion.member_path.clone())
13321 .collect::<Vec<_>>(),
13322 [vec![0], vec![1, 1]]
13323 );
13324 }
13325
13326 #[test]
13327 fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
13328 let paths = nested_parallel_paths();
13329 let history = vec![
13330 parallel_history_event(
13331 "ActivityScheduled",
13332 1,
13333 "activity_type",
13334 "first",
13335 paths[0].clone(),
13336 None,
13337 ),
13338 parallel_history_event(
13339 "ChildWorkflowScheduled",
13340 2,
13341 "child_workflow_type",
13342 "second",
13343 paths[1].clone(),
13344 None,
13345 ),
13346 parallel_history_event(
13347 "ActivityScheduled",
13348 3,
13349 "activity_type",
13350 "third",
13351 paths[2].clone(),
13352 None,
13353 ),
13354 ];
13355 for _restart in 0..2 {
13356 let ctx = workflow_context(history.clone());
13357 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
13358 let mut task_context = TaskContext::from_waker(noop_waker_ref());
13359 let outcome = call.as_mut().poll(&mut task_context);
13360 assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
13361 assert!(ctx.take_commands().expect("commands").is_empty());
13362 }
13363 }
13364
13365 async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
13366 let mut saga = ctx.saga();
13367 let outcome = async {
13368 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
13369 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
13370 let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
13371 saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
13372 ctx.activity("trip.charge", json!([])).await?;
13373 Ok(json!({"status": "booked"}))
13374 }
13375 .await;
13376 saga.finish(outcome).await
13377 }
13378
13379 fn saga_activity(
13380 event_type: &str,
13381 sequence: u64,
13382 activity_type: &str,
13383 result: Option<Value>,
13384 ) -> HistoryEvent {
13385 let mut payload = json!({
13386 "sequence": sequence,
13387 "activity_type": activity_type,
13388 "message": format!("{activity_type} failed"),
13389 "exception_type": "PlannedFailure",
13390 "non_retryable": true,
13391 });
13392 if let Some(result) = result {
13393 payload["result"] = fixture_envelope(result);
13394 }
13395 history_event(event_type, payload)
13396 }
13397
13398 #[test]
13399 fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
13400 let completed_hotel_compensation = saga_activity(
13401 "ActivityCompleted",
13402 4,
13403 "trip.cancel-hotel",
13404 Some(Value::Null),
13405 );
13406 let history = vec![
13407 saga_activity(
13408 "ActivityCompleted",
13409 1,
13410 "trip.reserve-flight",
13411 Some(json!("flight-1")),
13412 ),
13413 saga_activity(
13414 "ActivityCompleted",
13415 2,
13416 "trip.reserve-hotel",
13417 Some(json!("hotel-1")),
13418 ),
13419 saga_activity("ActivityFailed", 3, "trip.charge", None),
13420 completed_hotel_compensation.clone(),
13421 completed_hotel_compensation,
13422 ];
13423
13424 for _restart in 0..2 {
13425 let ctx = workflow_context(history.clone());
13426 let mut future = Box::pin(trip_saga(ctx.clone()));
13427 let mut task_context = TaskContext::from_waker(noop_waker_ref());
13428 assert!(matches!(
13429 future.as_mut().poll(&mut task_context),
13430 Poll::Pending
13431 ));
13432 let commands = ctx.take_commands().expect("compensation command");
13433 assert_eq!(commands.len(), 1);
13434 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
13435 }
13436 }
13437
13438 #[test]
13439 fn saga_compensation_failure_preserves_both_typed_failures() {
13440 let history = vec![
13441 saga_activity(
13442 "ActivityCompleted",
13443 1,
13444 "trip.reserve-flight",
13445 Some(json!("flight-1")),
13446 ),
13447 saga_activity(
13448 "ActivityCompleted",
13449 2,
13450 "trip.reserve-hotel",
13451 Some(json!("hotel-1")),
13452 ),
13453 saga_activity("ActivityFailed", 3, "trip.charge", None),
13454 saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
13455 ];
13456 let ctx = workflow_context(history);
13457 let mut future = Box::pin(trip_saga(ctx));
13458 let mut task_context = TaskContext::from_waker(noop_waker_ref());
13459 let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
13460 future.as_mut().poll(&mut task_context)
13461 else {
13462 panic!("compensation failure must remain structured");
13463 };
13464 assert!(matches!(
13465 *failure.initiating_failure,
13466 Error::ActivityFailed(_)
13467 ));
13468 assert!(matches!(
13469 *failure.compensation_failure,
13470 Error::ActivityFailed(_)
13471 ));
13472 assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
13473 assert_eq!(failure.compensation_registration_order, 2);
13474 }
13475
13476 #[test]
13477 fn saga_compensates_cooperative_cancellation() {
13478 let ctx = workflow_context(vec![saga_activity(
13479 "ActivityCompleted",
13480 1,
13481 "trip.reserve-flight",
13482 Some(json!("flight-1")),
13483 )]);
13484 ctx.state.lock().expect("state").cancel_requested = true;
13485 let run = {
13486 let ctx = ctx.clone();
13487 async move {
13488 let mut saga = ctx.saga();
13489 let outcome = async {
13490 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
13491 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
13492 ctx.throw_if_cancellation_requested()?;
13493 Ok(json!("unexpected"))
13494 }
13495 .await;
13496 saga.finish(outcome).await
13497 }
13498 };
13499 let mut future = Box::pin(run);
13500 let mut task_context = TaskContext::from_waker(noop_waker_ref());
13501 assert!(matches!(
13502 future.as_mut().poll(&mut task_context),
13503 Poll::Pending
13504 ));
13505 let commands = ctx.take_commands().expect("cancellation compensation");
13506 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
13507 }
13508
13509 fn workflow_task(
13510 workflow_type: &str,
13511 history_events: Vec<HistoryEvent>,
13512 payload_codec: &str,
13513 ) -> WorkflowTask {
13514 WorkflowTask {
13515 task_id: format!("wft-{workflow_type}"),
13516 workflow_command_id: None,
13517 workflow_id: Some(format!("wf-{workflow_type}")),
13518 run_id: Some(format!("run-{workflow_type}")),
13519 workflow_type: workflow_type.to_string(),
13520 cancel_requested: false,
13521 payload_codec: payload_codec.to_string(),
13522 arguments: Some(
13523 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
13524 ),
13525 total_history_events: Some(history_events.len() as u64),
13526 history_size_bytes: None,
13527 continue_as_new_recommended: None,
13528 history_budget_pressure: None,
13529 history_events,
13530 next_history_page_token: None,
13531 workflow_task_attempt: 1,
13532 workflow_signal_id: None,
13533 signal_name: None,
13534 signal_arguments: None,
13535 workflow_update_id: None,
13536 update_name: None,
13537 lease_owner: Some("rust-worker".to_string()),
13538 }
13539 }
13540
13541 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
13542 struct SideEffectProbe {
13543 request_id: String,
13544 attempt: u32,
13545 }
13546
13547 #[test]
13548 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
13549 let calls = AtomicUsize::new(0);
13550 let ctx = workflow_context(Vec::new());
13551 let value = ctx
13552 .side_effect(|| {
13553 calls.fetch_add(1, Ordering::SeqCst);
13554 SideEffectProbe {
13555 request_id: "request-42".to_string(),
13556 attempt: 3,
13557 }
13558 })
13559 .expect("first side effect");
13560 assert_eq!(value.attempt, 3);
13561 assert_eq!(calls.load(Ordering::SeqCst), 1);
13562 let commands = ctx.take_commands().expect("commands");
13563 assert_eq!(commands.len(), 1);
13564 assert_eq!(commands[0]["type"], "record_side_effect");
13565 assert_eq!(
13566 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
13567 serde_json::to_value(&value).expect("value")
13568 );
13569
13570 let replay = workflow_context(vec![history_event(
13571 "SideEffectRecorded",
13572 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
13573 )]);
13574 let replayed: SideEffectProbe = replay
13575 .side_effect(|| {
13576 calls.fetch_add(1, Ordering::SeqCst);
13577 panic!("committed side-effect callbacks must not run during replay")
13578 })
13579 .expect("replayed side effect");
13580 assert_eq!(replayed, value);
13581 assert_eq!(calls.load(Ordering::SeqCst), 1);
13582 assert!(replay.take_commands().expect("commands").is_empty());
13583 replay.ensure_history_consumed().expect("history consumed");
13584 }
13585
13586 #[test]
13587 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
13588 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
13589 let value = ctx
13590 .side_effect(|| SideEffectProbe {
13591 request_id: "avro-request".to_string(),
13592 attempt: 1,
13593 })
13594 .expect("Avro side effect");
13595 let uuid = ctx.uuid_v4().expect("deterministic UUID");
13596 let commands = ctx.take_commands().expect("commands");
13597 assert_eq!(commands.len(), 2);
13598 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
13599 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
13600 assert_eq!(
13601 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
13602 serde_json::to_value(&value).expect("value")
13603 );
13604
13605 let replay = workflow_context_with_codec(
13606 vec![
13607 history_event(
13608 "SideEffectRecorded",
13609 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
13610 ),
13611 history_event(
13612 "SideEffectRecorded",
13613 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
13614 ),
13615 ],
13616 DEFAULT_CODEC,
13617 );
13618 let replayed: SideEffectProbe = replay
13619 .side_effect(|| panic!("Avro callback must not run"))
13620 .expect("replayed Avro value");
13621 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
13622 assert_eq!(replayed, value);
13623 assert_eq!(replayed_uuid, uuid);
13624 assert!(replay.take_commands().expect("commands").is_empty());
13625 }
13626
13627 #[test]
13628 fn typed_side_effect_replay_preserves_bytes_and_maps() {
13629 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
13630 let value = ctx
13631 .side_effect_avro_value(typed_fidelity_probe)
13632 .expect("typed side effect");
13633 let commands = ctx.take_commands().expect("side-effect command");
13634 assert_eq!(
13635 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
13636 .expect("recorded side effect"),
13637 value
13638 );
13639
13640 let replay = workflow_context_with_codec(
13641 vec![history_event(
13642 "SideEffectRecorded",
13643 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
13644 )],
13645 DEFAULT_CODEC,
13646 );
13647 assert_eq!(
13648 replay
13649 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
13650 .expect("replayed typed side effect"),
13651 value
13652 );
13653 }
13654
13655 #[test]
13656 fn ordered_side_effects_share_the_durable_command_stream() {
13657 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
13658 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
13659 let ctx = workflow_context(vec![
13660 history_event(
13661 "SideEffectRecorded",
13662 json!({"sequence": 1, "result": first}),
13663 ),
13664 history_event(
13665 "SideEffectRecorded",
13666 json!({"sequence": 2, "result": second}),
13667 ),
13668 ]);
13669 let first: String = ctx
13670 .side_effect(|| panic!("first callback must not run"))
13671 .expect("first replay");
13672 let second: i32 = ctx
13673 .side_effect(|| panic!("second callback must not run"))
13674 .expect("second replay");
13675 assert_eq!(first, "first");
13676 assert_eq!(second, 29);
13677 ctx.ensure_history_consumed().expect("ordered history");
13678
13679 let reordered = workflow_context(vec![history_event(
13680 "VersionMarkerRecorded",
13681 json!({
13682 "sequence": 1,
13683 "change_id": "before-side-effect",
13684 "version": 1,
13685 "min_supported": 1,
13686 "max_supported": 1,
13687 }),
13688 )]);
13689 let error = reordered
13690 .side_effect(|| "new".to_string())
13691 .expect_err("command reordering must fail");
13692 assert!(matches!(
13693 error,
13694 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13695 if reason == "recorded_command_mismatch"
13696 ));
13697 }
13698
13699 #[test]
13700 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
13701 let ctx = workflow_context(Vec::new());
13702 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
13703 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
13704 assert!(ctx.patched("new-search").expect("patch"));
13705 ctx.deprecate_patch("new-search").expect("deprecate patch");
13706 let commands = ctx.take_commands().expect("commands");
13707 assert_eq!(commands.len(), 2);
13708 assert_eq!(commands[0]["type"], "record_version_marker");
13709 assert_eq!(commands[0]["version"], 2);
13710 assert_eq!(commands[1]["change_id"], "new-search");
13711
13712 let replay = workflow_context(vec![history_event(
13713 "VersionMarkerRecorded",
13714 json!({
13715 "sequence": 1,
13716 "change_id": "checkout-v2",
13717 "version": 2,
13718 "min_supported": 1,
13719 "max_supported": 2,
13720 }),
13721 )]);
13722 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
13723 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
13724 assert!(replay.take_commands().expect("commands").is_empty());
13725 replay.ensure_history_consumed().expect("history consumed");
13726 }
13727
13728 #[test]
13729 fn version_markers_reject_incompatible_or_malformed_history() {
13730 let incompatible = workflow_context(vec![history_event(
13731 "VersionMarkerRecorded",
13732 json!({
13733 "sequence": 1,
13734 "change_id": "checkout-v2",
13735 "version": 1,
13736 "min_supported": 1,
13737 "max_supported": 2,
13738 }),
13739 )]);
13740 let error = incompatible
13741 .get_version("checkout-v2", 2, 3)
13742 .expect_err("old version is unsupported");
13743 assert!(matches!(
13744 error,
13745 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13746 if reason == "version_marker_incompatible_range"
13747 ));
13748
13749 for (history, reason) in [
13750 (
13751 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
13752 "side_effect_result_missing",
13753 ),
13754 (
13755 vec![history_event(
13756 "SideEffectRecorded",
13757 json!({
13758 "sequence": 1,
13759 "result": {"codec": "avro", "blob": "not-base64"},
13760 }),
13761 )],
13762 "side_effect_payload_incompatible",
13763 ),
13764 (
13765 vec![history_event(
13766 "SideEffectRecorded",
13767 json!({"sequence": 1, "result": {"unwrapped": true}}),
13768 )],
13769 "side_effect_payload_malformed",
13770 ),
13771 (
13772 vec![history_event(
13773 "VersionMarkerRecorded",
13774 json!({
13775 "sequence": 1,
13776 "change_id": "change",
13777 "version": 1,
13778 "min_supported": 2,
13779 "max_supported": 1,
13780 }),
13781 )],
13782 "version_marker_history_range_invalid",
13783 ),
13784 ] {
13785 let error = WorkflowState::new(
13786 history,
13787 "rust-workers".to_string(),
13788 DEFAULT_CODEC.to_string(),
13789 None,
13790 )
13791 .expect_err("malformed history must fail");
13792 assert!(matches!(
13793 error,
13794 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
13795 if actual == reason
13796 ));
13797 }
13798 }
13799
13800 #[test]
13801 fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
13802 let history = vec![history_event(
13803 "SearchAttributesUpserted",
13804 json!({
13805 "sequence": 1,
13806 "attributes": {"customer_tier": "gold"},
13807 "attribute_types": {"customer_tier": "keyword"},
13808 "merged": {"customer_tier": "gold"}
13809 }),
13810 )];
13811
13812 let matching = workflow_context(history.clone());
13813 matching
13814 .upsert_search_attributes(
13815 SearchAttributeUpdate::new()
13816 .keyword("customer_tier", "gold")
13817 .expect("keyword update"),
13818 )
13819 .expect("matching typed update must replay");
13820 matching
13821 .ensure_history_consumed()
13822 .expect("history consumed");
13823
13824 let changed_type = workflow_context(history.clone());
13825 let error = changed_type
13826 .upsert_search_attributes(
13827 SearchAttributeUpdate::new()
13828 .string("customer_tier", "gold")
13829 .expect("string update"),
13830 )
13831 .expect_err("same JSON value with a different declaration must be nondeterministic");
13832 let Error::NonDeterministicReplay(failure) = error else {
13833 panic!("typed identity drift must be a replay failure");
13834 };
13835 assert_eq!(failure.reason, "search_attribute_type_mismatch");
13836 assert_eq!(failure.sequence, Some(1));
13837
13838 let changed_value = workflow_context(history);
13839 let error = changed_value
13840 .upsert_search_attributes(
13841 SearchAttributeUpdate::new()
13842 .keyword("customer_tier", "platinum")
13843 .expect("keyword update"),
13844 )
13845 .expect_err("changed values must be nondeterministic");
13846 let Error::NonDeterministicReplay(failure) = error else {
13847 panic!("value drift must be a replay failure");
13848 };
13849 assert_eq!(failure.reason, "search_attribute_value_mismatch");
13850 }
13851
13852 #[test]
13853 fn legacy_search_attribute_history_keeps_type_identity_unknown() {
13854 let history = vec![history_event(
13855 "SearchAttributesUpserted",
13856 json!({
13857 "sequence": 1,
13858 "attributes": {"customer_tier": "gold"},
13859 "merged": {"customer_tier": "gold"}
13860 }),
13861 )];
13862
13863 for update in [
13864 SearchAttributeUpdate::new()
13865 .keyword("customer_tier", "gold")
13866 .expect("keyword update"),
13867 SearchAttributeUpdate::new()
13868 .string("customer_tier", "gold")
13869 .expect("string update"),
13870 ] {
13871 let restarted = workflow_context(history.clone());
13872 restarted
13873 .upsert_search_attributes(update)
13874 .expect("legacy history constrains values but has unknown type identity");
13875 restarted
13876 .ensure_history_consumed()
13877 .expect("history consumed");
13878 }
13879 }
13880
13881 #[test]
13882 fn search_attribute_command_emits_canonical_types() {
13883 let ctx = workflow_context(Vec::new());
13884 ctx.upsert_search_attributes(
13885 SearchAttributeUpdate::new()
13886 .keyword("customer_tier", "gold")
13887 .expect("keyword update")
13888 .int("attempts", 3)
13889 .expect("int update")
13890 .delete("obsolete")
13891 .expect("delete update"),
13892 )
13893 .expect("valid search attributes");
13894
13895 assert_eq!(
13896 ctx.take_commands().expect("commands"),
13897 vec![json!({
13898 "type": "upsert_search_attributes",
13899 "attributes": {
13900 "attempts": 3,
13901 "customer_tier": "gold",
13902 "obsolete": null
13903 },
13904 "attribute_types": {
13905 "attempts": "int",
13906 "customer_tier": "keyword"
13907 }
13908 })]
13909 );
13910 }
13911
13912 #[test]
13913 fn duplicate_side_effects_and_version_markers_are_rejected() {
13914 let duplicate_side_effect = WorkflowState::new(
13915 vec![
13916 history_event(
13917 "SideEffectRecorded",
13918 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
13919 ),
13920 history_event(
13921 "SideEffectRecorded",
13922 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
13923 ),
13924 ],
13925 "rust-workers".to_string(),
13926 DEFAULT_CODEC.to_string(),
13927 None,
13928 )
13929 .expect_err("duplicate side effect");
13930 assert!(matches!(
13931 duplicate_side_effect,
13932 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13933 if reason == "duplicate_side_effect_record"
13934 ));
13935
13936 let marker = |sequence| {
13937 history_event(
13938 "VersionMarkerRecorded",
13939 json!({
13940 "sequence": sequence,
13941 "change_id": "same-change",
13942 "version": 1,
13943 "min_supported": 1,
13944 "max_supported": 1,
13945 }),
13946 )
13947 };
13948 let duplicate_marker = WorkflowState::new(
13949 vec![marker(1), marker(3)],
13950 "rust-workers".to_string(),
13951 DEFAULT_CODEC.to_string(),
13952 None,
13953 )
13954 .expect_err("duplicate marker");
13955 assert!(matches!(
13956 duplicate_marker,
13957 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13958 if reason == "duplicate_version_marker"
13959 ));
13960 }
13961
13962 #[test]
13963 fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
13964 let mut state = WorkflowState::new(
13965 Vec::new(),
13966 "rust-workers".to_string(),
13967 DEFAULT_CODEC.to_string(),
13968 None,
13969 )
13970 .expect("workflow state");
13971 state.workflow_command_identity = "command-7".to_string();
13972 let context = WorkflowContext {
13973 state: Arc::new(Mutex::new(state)),
13974 };
13975 let item =
13976 WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
13977
13978 context
13979 .append_workflow_stream("output", &[item], Some(10))
13980 .expect("append command");
13981 context
13982 .error_workflow_stream("output", "producer failed", None)
13983 .expect("error command");
13984 let commands = context.take_commands().expect("commands");
13985
13986 assert_eq!(commands[0]["type"], "record_side_effect");
13987 assert_eq!(
13988 commands[0]["workflow_stream"]["command_identity"],
13989 "command-7"
13990 );
13991 assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
13992 assert_eq!(
13993 commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
13994 "dw-stream:command-7:0:0"
13995 );
13996 assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
13997
13998 let recorded = history_event(
13999 "SideEffectRecorded",
14000 json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
14001 );
14002 let mut replay_state = WorkflowState::new(
14003 vec![recorded],
14004 "rust-workers".to_string(),
14005 DEFAULT_CODEC.to_string(),
14006 None,
14007 )
14008 .expect("replay state");
14009 replay_state.workflow_command_identity = "command-7".to_string();
14010 let replay_context = WorkflowContext {
14011 state: Arc::new(Mutex::new(replay_state)),
14012 };
14013 replay_context
14014 .append_workflow_stream(
14015 "output",
14016 &[WorkflowStreamAppendItem::from_reference(
14017 "s3://bucket/item.avro",
14018 )],
14019 Some(10),
14020 )
14021 .expect("replayed append");
14022 assert!(replay_context
14023 .take_commands()
14024 .expect("replayed commands")
14025 .is_empty());
14026 }
14027
14028 #[test]
14029 fn workflow_stream_authoring_requires_server_durable_command_identity() {
14030 let context = workflow_context(Vec::new());
14031 let error = context
14032 .append_workflow_stream(
14033 "output",
14034 &[WorkflowStreamAppendItem::from_reference(
14035 "s3://bucket/item.avro",
14036 )],
14037 None,
14038 )
14039 .expect_err("stream append without durable command identity must fail closed");
14040
14041 assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
14042 assert!(context.take_commands().expect("commands").is_empty());
14043 }
14044
14045 #[test]
14046 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
14047 fn worker(calls: Arc<AtomicUsize>) -> Worker {
14048 let client = Client::new("http://127.0.0.1:8080").expect("client");
14049 let mut worker = Worker::new(client, "rust-workers");
14050 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
14051 let calls = Arc::clone(&calls);
14052 async move {
14053 let captured = ctx.side_effect(|| {
14054 calls.fetch_add(1, Ordering::SeqCst);
14055 "captured-once".to_string()
14056 })?;
14057 let version = ctx.get_version("cold-restart", 1, 2)?;
14058 Ok(json!({"captured": captured, "version": version}))
14059 }
14060 });
14061 worker
14062 }
14063
14064 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
14065 WorkflowTask {
14066 task_id: "wft-side-effect-version".to_string(),
14067 workflow_command_id: None,
14068 workflow_id: Some("wf-side-effect-version".to_string()),
14069 run_id: Some("run-side-effect-version".to_string()),
14070 workflow_type: "rust.side-effect-version".to_string(),
14071 cancel_requested: false,
14072 payload_codec: DEFAULT_CODEC.to_string(),
14073 arguments: Some(
14074 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
14075 ),
14076 history_events,
14077 total_history_events: None,
14078 history_size_bytes: None,
14079 continue_as_new_recommended: None,
14080 history_budget_pressure: None,
14081 next_history_page_token: None,
14082 workflow_task_attempt: 1,
14083 workflow_signal_id: None,
14084 signal_name: None,
14085 signal_arguments: None,
14086 workflow_update_id: None,
14087 update_name: None,
14088 lease_owner: Some("rust-worker".to_string()),
14089 }
14090 }
14091
14092 let calls = Arc::new(AtomicUsize::new(0));
14093 let initial = worker(Arc::clone(&calls))
14094 .execute_workflow_task(task(Vec::new()))
14095 .expect("initial execution");
14096 assert_eq!(
14097 initial
14098 .iter()
14099 .map(|command| &command["type"])
14100 .collect::<Vec<_>>(),
14101 vec![
14102 "record_side_effect",
14103 "record_version_marker",
14104 "complete_workflow"
14105 ]
14106 );
14107 assert_eq!(calls.load(Ordering::SeqCst), 1);
14108
14109 let restarted = worker(Arc::clone(&calls));
14110 let replayed = restarted
14111 .execute_workflow_task(task(vec![
14112 history_event(
14113 "SideEffectRecorded",
14114 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
14115 ),
14116 history_event(
14117 "VersionMarkerRecorded",
14118 json!({
14119 "sequence": 2,
14120 "change_id": "cold-restart",
14121 "version": 2,
14122 "min_supported": 1,
14123 "max_supported": 2,
14124 }),
14125 ),
14126 ]))
14127 .expect("cold replay");
14128 assert_eq!(replayed.len(), 1);
14129 assert_eq!(replayed[0]["type"], "complete_workflow");
14130 assert_eq!(calls.load(Ordering::SeqCst), 1);
14131 }
14132
14133 #[test]
14134 fn side_effect_replay_rejects_changed_rust_value_type() {
14135 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
14136 let ctx = workflow_context(vec![history_event(
14137 "SideEffectRecorded",
14138 json!({"sequence": 1, "result": result}),
14139 )]);
14140 let error = ctx
14141 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
14142 .expect_err("changed type must fail replay");
14143 assert!(matches!(
14144 error,
14145 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
14146 if reason == "side_effect_type_mismatch"
14147 ));
14148 }
14149
14150 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
14151 vec![
14152 history_event(
14153 "ActivityScheduled",
14154 json!({
14155 "sequence": 1,
14156 "activity_type": "flaky",
14157 "activity_execution_id": "act-1",
14158 "activity": {
14159 "id": "act-1",
14160 "sequence": 1,
14161 "type": "flaky",
14162 "queue": "critical-activities",
14163 "execution_mode": null,
14164 "retry_policy": {
14165 "snapshot_version": 1,
14166 "max_attempts": 3,
14167 "backoff_seconds": [2, 4],
14168 "start_to_close_timeout": 30,
14169 "schedule_to_start_timeout": 5,
14170 "schedule_to_close_timeout": 90,
14171 "heartbeat_timeout": 10,
14172 "non_retryable_error_types": ["PermanentError"]
14173 }
14174 }
14175 }),
14176 ),
14177 history_event(
14178 "ActivityStarted",
14179 json!({
14180 "sequence": 1,
14181 "activity_type": "flaky",
14182 "activity_execution_id": "act-1",
14183 "activity_attempt_id": "attempt-1",
14184 "attempt_number": 1
14185 }),
14186 ),
14187 history_event(
14188 "ActivityRetryScheduled",
14189 json!({
14190 "sequence": 1,
14191 "activity_type": "flaky",
14192 "activity_execution_id": "act-1",
14193 "activity_attempt_id": "attempt-1",
14194 "attempt_number": 1,
14195 "retry_after_attempt": 1,
14196 "retry_backoff_seconds": 2,
14197 "failure_category": "activity",
14198 "exception_type": "TransientError"
14199 }),
14200 ),
14201 history_event(
14202 "ActivityStarted",
14203 json!({
14204 "sequence": 1,
14205 "activity_type": "flaky",
14206 "activity_execution_id": "act-1",
14207 "activity_attempt_id": "attempt-2",
14208 "attempt_number": 2
14209 }),
14210 ),
14211 history_event(
14212 "ActivityCompleted",
14213 json!({
14214 "sequence": 1,
14215 "activity_type": "flaky",
14216 "activity_execution_id": "act-1",
14217 "activity_attempt_id": "attempt-2",
14218 "attempt_number": 2,
14219 "payload_codec": DEFAULT_CODEC,
14220 "result": fixture_envelope(json!({"status":"recovered"}))
14221 }),
14222 ),
14223 ]
14224 }
14225
14226 fn retry_activity_options() -> ActivityOptions {
14227 ActivityOptions::new()
14228 .task_queue("critical-activities")
14229 .retry_policy(
14230 ActivityRetryPolicy::new(3)
14231 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
14232 .non_retryable_error_type("PermanentError"),
14233 )
14234 .start_to_close_timeout(Duration::from_secs(30))
14235 .schedule_to_start_timeout(Duration::from_secs(5))
14236 .schedule_to_close_timeout(Duration::from_secs(90))
14237 .heartbeat_timeout(Duration::from_secs(10))
14238 }
14239
14240 #[test]
14241 fn fixed_avro_value_round_trips_json_values() {
14242 let value = json!({"greeting": "hello", "count": 3, "ok": true});
14243 let envelope = PayloadEnvelope::avro(&value).expect("encode");
14244 assert_eq!(envelope.codec, DEFAULT_CODEC);
14245 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
14246 }
14247
14248 #[tokio::test]
14249 async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
14250 let client = Client::new("http://127.0.0.1:8080").expect("client");
14251 let mut worker = Worker::new(client, "rust-workers");
14252 worker.register_typed_workflow(
14253 "typed.contract.workflow",
14254 |_ctx, input: TypedContract| async move { Ok(input) },
14255 );
14256 worker.register_typed_activity(
14257 "typed.contract.activity",
14258 |_ctx, input: TypedContract| async move { Ok(input) },
14259 );
14260
14261 let expected = typed_contract();
14262 let arguments = AvroValue::Array(vec![
14263 AvroValue::from_serialize(&expected).expect("typed request")
14264 ]);
14265 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
14266 let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
14267 workflow.arguments = Some(envelope.clone());
14268 let commands = worker
14269 .execute_workflow_task(workflow)
14270 .expect("typed workflow task");
14271 let workflow_result: TypedContract =
14272 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
14273 .expect("workflow result envelope")
14274 .deserialize()
14275 .expect("workflow result type");
14276 assert_eq!(workflow_result, expected);
14277
14278 let activity = ActivityTask {
14279 task_id: "typed-contract-activity".to_string(),
14280 activity_attempt_id: Some("typed-contract-attempt".to_string()),
14281 attempt_id: None,
14282 activity_type: "typed.contract.activity".to_string(),
14283 payload_codec: DEFAULT_CODEC.to_string(),
14284 arguments: Some(envelope),
14285 attempt_number: 1,
14286 lease_owner: Some("rust-worker".to_string()),
14287 };
14288 let activity_result: TypedContract = worker
14289 .execute_activity_task(activity)
14290 .await
14291 .expect("typed activity task")
14292 .deserialize()
14293 .expect("activity result type");
14294 assert_eq!(activity_result, expected);
14295 }
14296
14297 #[tokio::test]
14298 async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
14299 let client = Client::new("http://127.0.0.1:8080").expect("client");
14300 let mut worker = Worker::new(client, "rust-workers");
14301 worker.register_typed_workflow(
14302 "typed.shape.workflow",
14303 |_ctx, input: TypedContract| async move { Ok(input) },
14304 );
14305 worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
14306 Ok(f64::NAN)
14307 });
14308
14309 let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
14310 workflow.arguments = Some(
14311 encode_typed_envelope(
14312 &AvroValue::Array(vec![
14313 AvroValue::String("first".to_string()),
14314 AvroValue::String("second".to_string()),
14315 ]),
14316 DEFAULT_CODEC,
14317 )
14318 .expect("malformed typed arguments"),
14319 );
14320 let commands = worker
14321 .execute_workflow_task(workflow)
14322 .expect("shape mismatch becomes a workflow failure");
14323 let message = commands[0]["message"].as_str().expect("failure message");
14324 assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
14325 assert!(message.contains(type_name::<TypedContract>()));
14326 assert!(message.contains("task carried 2 arguments"));
14327
14328 let activity = ActivityTask {
14329 task_id: "typed-unsupported-activity".to_string(),
14330 activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
14331 attempt_id: None,
14332 activity_type: "typed.unsupported.activity".to_string(),
14333 payload_codec: DEFAULT_CODEC.to_string(),
14334 arguments: Some(
14335 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
14336 .expect("unit arguments"),
14337 ),
14338 attempt_number: 1,
14339 lease_owner: Some("rust-worker".to_string()),
14340 };
14341 let Error::HandlerType {
14342 handler_kind,
14343 handler_name,
14344 value_kind,
14345 rust_type,
14346 message,
14347 } = worker
14348 .execute_activity_task(activity)
14349 .await
14350 .expect_err("non-finite handler output must fail")
14351 else {
14352 panic!("expected contextual handler type failure");
14353 };
14354 assert_eq!(handler_kind, HandlerKind::Activity);
14355 assert_eq!(handler_name, "typed.unsupported.activity");
14356 assert_eq!(value_kind, HandlerValueKind::Result);
14357 assert_eq!(rust_type, type_name::<f64>());
14358 assert!(message.contains("non_finite_float"));
14359 }
14360
14361 #[tokio::test]
14362 async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
14363 #[derive(Clone, Default)]
14364 struct State {
14365 observed: Option<TypedContract>,
14366 }
14367
14368 let client = Client::new("http://127.0.0.1:8080").expect("client");
14369 let mut worker = Worker::new(client, "rust-workers");
14370 worker.register_typed_replayed_workflow(
14371 "typed.contract.replayed",
14372 State::default,
14373 |ctx, input: TypedContract, state| async move {
14374 let result: TypedContract =
14375 ctx.activity_typed("typed.contract.activity", input).await?;
14376 state.update(|current| current.observed = Some(result.clone()))?;
14377 Ok(result)
14378 },
14379 );
14380 worker.register_replayed_query::<State, _, _>(
14381 "typed.contract.replayed",
14382 "observed",
14383 |_ctx, state, _args| async move {
14384 Ok(json!(state.observed.as_ref().map(|value| value.signed)))
14385 },
14386 );
14387
14388 let expected = typed_contract();
14389 let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
14390 let workflow_arguments =
14391 encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
14392 .expect("workflow arguments");
14393 let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
14394 let task = QueryTask {
14395 query_task_id: "typed-replay-query".to_string(),
14396 query_task_attempt: 1,
14397 lease_owner: Some("rust-worker".to_string()),
14398 workflow_id: Some("typed-replay".to_string()),
14399 run_id: Some("typed-replay-run".to_string()),
14400 workflow_type: "typed.contract.replayed".to_string(),
14401 query_name: "observed".to_string(),
14402 payload_codec: DEFAULT_CODEC.to_string(),
14403 workflow_arguments: Some(workflow_arguments),
14404 query_arguments: Some(
14405 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
14406 .expect("query arguments"),
14407 ),
14408 history_events: vec![
14409 history_event(
14410 "ActivityScheduled",
14411 json!({
14412 "sequence": 1,
14413 "activity_type": "typed.contract.activity"
14414 }),
14415 ),
14416 history_event(
14417 "ActivityCompleted",
14418 json!({
14419 "sequence": 1,
14420 "activity_type": "typed.contract.activity",
14421 "payload_codec": DEFAULT_CODEC,
14422 "result": result
14423 }),
14424 ),
14425 ],
14426 history_export: None,
14427 run_status: Some("completed".to_string()),
14428 };
14429
14430 assert_eq!(
14431 worker
14432 .execute_query_task(task)
14433 .await
14434 .expect("typed replay query")
14435 .deserialize::<i64>()
14436 .expect("query result"),
14437 expected.signed
14438 );
14439 }
14440
14441 #[tokio::test]
14442 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
14443 let client = Client::new("http://127.0.0.1:8080").expect("client");
14444 let mut worker = Worker::new(client, "rust-workers");
14445 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
14446 worker
14447 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
14448 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
14449 Ok(input)
14450 });
14451 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
14452 Ok(input)
14453 });
14454 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
14455 Ok(AvroValue::Array(
14456 ctx.wait_signal_avro_value("changed").await?,
14457 ))
14458 });
14459
14460 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
14461 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
14462
14463 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
14464 workflow.arguments = Some(envelope.clone());
14465 let commands = worker
14466 .execute_workflow_task(workflow)
14467 .expect("typed workflow task");
14468 assert_eq!(commands[0]["type"], "complete_workflow");
14469 assert_eq!(
14470 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
14471 .expect("typed workflow result"),
14472 arguments
14473 );
14474
14475 let activity = ActivityTask {
14476 task_id: "activity-typed".to_string(),
14477 activity_attempt_id: Some("attempt-typed".to_string()),
14478 attempt_id: None,
14479 activity_type: "typed.activity".to_string(),
14480 payload_codec: DEFAULT_CODEC.to_string(),
14481 arguments: Some(envelope.clone()),
14482 attempt_number: 1,
14483 lease_owner: Some("rust-worker".to_string()),
14484 };
14485 assert_eq!(
14486 worker
14487 .execute_activity_task(activity)
14488 .await
14489 .expect("typed activity result"),
14490 arguments
14491 );
14492
14493 let query = QueryTask {
14494 query_task_id: "query-typed".to_string(),
14495 query_task_attempt: 1,
14496 lease_owner: Some("rust-worker".to_string()),
14497 workflow_id: Some("typed-1".to_string()),
14498 run_id: Some("run-typed".to_string()),
14499 workflow_type: "typed.echo".to_string(),
14500 query_name: "inspect".to_string(),
14501 payload_codec: DEFAULT_CODEC.to_string(),
14502 workflow_arguments: Some(
14503 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
14504 .expect("workflow input"),
14505 ),
14506 query_arguments: Some(envelope.clone()),
14507 history_events: Vec::new(),
14508 history_export: None,
14509 run_status: Some("running".to_string()),
14510 };
14511 assert_eq!(
14512 worker
14513 .execute_query_task(query)
14514 .await
14515 .expect("typed query result"),
14516 arguments
14517 );
14518
14519 let mut update = workflow_task(
14520 "typed.echo",
14521 vec![history_event(
14522 "UpdateAccepted",
14523 json!({
14524 "update_id": "update-typed",
14525 "update_name": "replace",
14526 "arguments": envelope.clone(),
14527 }),
14528 )],
14529 DEFAULT_CODEC,
14530 );
14531 update.workflow_update_id = Some("update-typed".to_string());
14532 update.update_name = Some("replace".to_string());
14533 let commands = worker
14534 .execute_workflow_task(update)
14535 .expect("typed update task");
14536 assert_eq!(commands[0]["type"], "complete_update");
14537 assert_eq!(
14538 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
14539 .expect("typed update result"),
14540 arguments
14541 );
14542
14543 let mut signal = workflow_task(
14544 "typed.signal",
14545 vec![history_event(
14546 "SignalReceived",
14547 json!({
14548 "signal_id": "signal-typed",
14549 "signal_name": "changed",
14550 "arguments": envelope.clone(),
14551 }),
14552 )],
14553 DEFAULT_CODEC,
14554 );
14555 signal.workflow_signal_id = Some("signal-typed".to_string());
14556 signal.signal_name = Some("changed".to_string());
14557 signal.signal_arguments = Some(envelope);
14558 let commands = worker
14559 .execute_workflow_task(signal)
14560 .expect("typed signal resume");
14561 assert_eq!(
14562 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
14563 .expect("typed signal result"),
14564 arguments
14565 );
14566 }
14567
14568 #[tokio::test]
14569 async fn typed_helpers_never_parse_json_inspection_projection() {
14570 let collision_values = projection_collision_probe();
14571 let expected = AvroValue::Array(collision_values.clone());
14572 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
14573
14574 let activity_context = workflow_context_with_codec(
14575 vec![history_event(
14576 "ActivityCompleted",
14577 json!({
14578 "sequence": 1,
14579 "activity_type": "collision.activity",
14580 "payload_codec": DEFAULT_CODEC,
14581 "result": envelope.clone(),
14582 }),
14583 )],
14584 DEFAULT_CODEC,
14585 );
14586 assert_eq!(
14587 activity_context
14588 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
14589 .await
14590 .expect("typed activity collision result"),
14591 expected
14592 );
14593
14594 let signal_context = workflow_context_with_codec(
14595 vec![
14596 history_event(
14597 "SignalWaitOpened",
14598 json!({"sequence": 1, "signal_name": "collision"}),
14599 ),
14600 history_event(
14601 "SignalApplied",
14602 json!({
14603 "sequence": 1,
14604 "signal_name": "collision",
14605 "payload_codec": DEFAULT_CODEC,
14606 "value": envelope.clone(),
14607 }),
14608 ),
14609 ],
14610 DEFAULT_CODEC,
14611 );
14612 assert_eq!(
14613 signal_context
14614 .wait_signal_avro_value("collision")
14615 .await
14616 .expect("typed signal collision arguments"),
14617 collision_values
14618 );
14619
14620 let child_context = workflow_context_with_codec(
14621 vec![
14622 history_event(
14623 "ChildWorkflowScheduled",
14624 json!({
14625 "sequence": 1,
14626 "child_workflow_instance_id": "collision-child",
14627 "child_workflow_run_id": "collision-run",
14628 "child_workflow_type": "collision.child",
14629 }),
14630 ),
14631 history_event(
14632 "ChildRunCompleted",
14633 json!({
14634 "sequence": 1,
14635 "child_workflow_instance_id": "collision-child",
14636 "child_workflow_run_id": "collision-run",
14637 "child_workflow_type": "collision.child",
14638 "payload_codec": DEFAULT_CODEC,
14639 "result": envelope,
14640 }),
14641 ),
14642 ],
14643 DEFAULT_CODEC,
14644 );
14645 let child = child_context
14646 .start_child_workflow_avro_value(
14647 "collision.child",
14648 ChildWorkflowOptions::new("collision-workers"),
14649 AvroValue::Array(Vec::new()),
14650 )
14651 .await
14652 .expect("typed child collision result");
14653 assert_eq!(child.result, expected);
14654 }
14655
14656 #[tokio::test]
14657 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
14658 let client = Client::new("http://127.0.0.1:8080").expect("client");
14659 let mut worker = Worker::new(client, "rust-workers");
14660 worker.register_replayed_workflow_avro_value(
14661 "typed.replayed",
14662 || (),
14663 |_ctx, input, _state| async move { Ok(input) },
14664 );
14665 worker.register_replayed_query_avro_value::<(), _, _>(
14666 "typed.replayed",
14667 "inspect",
14668 |ctx, _state, args| async move {
14669 let mut signals = ctx.signals_avro_value("collision");
14670 let signal = signals
14671 .pop()
14672 .map(AvroValue::Array)
14673 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
14674 Ok(AvroValue::Array(vec![
14675 ctx.workflow_input_avro_value().clone(),
14676 signal,
14677 args,
14678 ]))
14679 },
14680 );
14681 let arguments = AvroValue::Array(projection_collision_probe());
14682 let signal_arguments =
14683 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
14684 let task = QueryTask {
14685 query_task_id: "query-typed-replay".to_string(),
14686 query_task_attempt: 1,
14687 lease_owner: Some("rust-worker".to_string()),
14688 workflow_id: Some("typed-replay".to_string()),
14689 run_id: Some("run-typed-replay".to_string()),
14690 workflow_type: "typed.replayed".to_string(),
14691 query_name: "inspect".to_string(),
14692 payload_codec: DEFAULT_CODEC.to_string(),
14693 workflow_arguments: Some(
14694 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
14695 ),
14696 query_arguments: Some(
14697 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
14698 ),
14699 history_events: vec![history_event(
14700 "SignalReceived",
14701 json!({
14702 "signal_id": "collision-signal",
14703 "signal_name": "collision",
14704 "workflow_sequence": 1,
14705 "payload_codec": DEFAULT_CODEC,
14706 "arguments": signal_arguments,
14707 }),
14708 )],
14709 history_export: None,
14710 run_status: Some("completed".to_string()),
14711 };
14712
14713 assert_eq!(
14714 worker
14715 .execute_query_task(task)
14716 .await
14717 .expect("typed replay query"),
14718 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
14719 );
14720 }
14721
14722 #[test]
14723 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
14724 let value = BTreeMap::from([(1_i32, "integer key")]);
14725 let error = PayloadEnvelope::avro(&value)
14726 .expect_err("integer map keys must fail")
14727 .to_string();
14728
14729 assert!(error.contains("invalid_map_key"));
14730 }
14731
14732 #[test]
14733 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
14734 let envelope = PayloadEnvelope {
14735 codec: "json".to_string(),
14736 blob: r#"{"greeting":"hello"}"#.to_string(),
14737 };
14738
14739 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
14740 let diagnostic = error.to_string();
14741 assert!(diagnostic.contains("unsupported_payload_codec"));
14742 assert!(diagnostic.contains("codec=\"avro\""));
14743 assert!(diagnostic.contains("HTTP document transport"));
14744 }
14745
14746 #[test]
14747 fn untagged_json_payload_value_fails_closed() {
14748 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
14749 .expect_err("untagged JSON payload values must fail");
14750 let diagnostic = error.to_string();
14751 assert!(diagnostic.contains("unsupported_payload_codec"));
14752 assert!(diagnostic.contains("untagged durable payload"));
14753 assert!(diagnostic.contains("HTTP document transport"));
14754 }
14755
14756 #[test]
14757 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
14758 let envelope = PayloadEnvelope {
14759 codec: DEFAULT_CODEC.to_string(),
14760 blob: BASE64.encode([0x01]),
14761 };
14762
14763 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
14764 assert!(error.to_string().contains("invalid_payload_framing"));
14765 }
14766
14767 #[tokio::test]
14768 async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
14769 let server = MockWorkerServer::start();
14770 let client = Client::builder(server.base_url())
14771 .timeout(Duration::from_secs(2))
14772 .build()
14773 .expect("client");
14774 let invalid_commands = [
14775 json!({
14776 "type": "complete_workflow",
14777 "result": {"codec": "json", "blob": null}
14778 }),
14779 json!({
14780 "type": "schedule_activity",
14781 "arguments": {"codec": "yaml", "blob": "ignored"}
14782 }),
14783 json!({
14784 "type": "start_child_workflow",
14785 "arguments": {"codec": DEFAULT_CODEC, "blob": null}
14786 }),
14787 json!({"type": "continue_as_new", "arguments": []}),
14788 json!({"type": "complete_update"}),
14789 json!({"type": "record_side_effect", "result": null}),
14790 json!({
14791 "type": "start_service_operation",
14792 "payload_codec": DEFAULT_CODEC,
14793 "request_payload": "raw-avro-bytes"
14794 }),
14795 ];
14796
14797 for command in invalid_commands {
14798 let error = client
14799 .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
14800 .await
14801 .expect_err("invalid durable payload must fail locally");
14802 let diagnostic = error.to_string();
14803 assert!(
14804 diagnostic.contains("unsupported_payload_codec")
14805 || diagnostic.contains("invalid_payload_envelope")
14806 || diagnostic.contains("untagged durable payload"),
14807 "unexpected validation diagnostic: {diagnostic}"
14808 );
14809 }
14810
14811 assert_eq!(
14812 server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
14813 0,
14814 "invalid command payloads must not reach HTTP transport"
14815 );
14816 }
14817
14818 #[test]
14819 fn workflow_completion_validates_only_protocol_owned_payload_slots() {
14820 let envelope = fixture_envelope(json!({"codec": "customer-value"}));
14821 let commands = [
14822 json!({"type": "complete_workflow", "result": envelope.clone()}),
14823 json!({"type": "schedule_activity", "arguments": envelope.clone()}),
14824 json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
14825 json!({"type": "continue_as_new", "arguments": envelope.clone()}),
14826 json!({"type": "complete_update", "result": envelope.clone()}),
14827 json!({"type": "record_side_effect", "result": envelope.clone()}),
14828 json!({
14829 "type": "start_service_operation",
14830 "payload_codec": DEFAULT_CODEC,
14831 "request_payload": envelope.clone()
14832 }),
14833 json!({
14834 "type": "complete_workflow",
14835 "result": envelope,
14836 "metadata": {
14837 "codec": "json",
14838 "payload_codec": "customer-codec",
14839 "result": {"codec": "yaml", "blob": null}
14840 }
14841 }),
14842 ];
14843
14844 validate_workflow_task_commands(&commands)
14845 .expect("customer metadata must not become a protocol codec declaration");
14846 }
14847
14848 #[test]
14849 fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
14850 assert_eq!(
14851 decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
14852 AvroValue::Array(Vec::new())
14853 );
14854 assert_eq!(
14855 decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
14856 AvroValue::Array(Vec::new())
14857 );
14858
14859 let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
14860 signal.signal_name = Some("empty-signal".to_string());
14861 signal.signal_arguments = None;
14862 let decoded = decode_resume_signal(&signal)
14863 .expect("valid Avro signal")
14864 .expect("named signal resumes the workflow");
14865 assert!(decoded.arguments.is_empty());
14866 }
14867
14868 #[tokio::test]
14869 async fn malformed_task_level_codecs_become_pre_handler_failures() {
14870 let client = Client::new("http://127.0.0.1:8080").expect("client");
14871 let mut worker = Worker::new(client, "rust-workers");
14872 let handler_calls = Arc::new(AtomicUsize::new(0));
14873
14874 let calls = Arc::clone(&handler_calls);
14875 worker.register_workflow("codec.workflow", move |_ctx, _args| {
14876 calls.fetch_add(1, Ordering::SeqCst);
14877 async move { Ok(Value::Null) }
14878 });
14879 let calls = Arc::clone(&handler_calls);
14880 worker.register_activity("codec.activity", move |_ctx, _args| {
14881 calls.fetch_add(1, Ordering::SeqCst);
14882 async move { Ok(Value::Null) }
14883 });
14884 let calls = Arc::clone(&handler_calls);
14885 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
14886 calls.fetch_add(1, Ordering::SeqCst);
14887 async move { Ok(Value::Null) }
14888 });
14889
14890 let mut failures = Vec::new();
14891 for codec_case in [
14892 InvalidTaskPayloadCodec::Missing,
14893 InvalidTaskPayloadCodec::Null,
14894 InvalidTaskPayloadCodec::NonString,
14895 ] {
14896 let mut workflow = json!({
14897 "task_id": format!("workflow-{}", codec_case.label()),
14898 "workflow_type": "codec.workflow"
14899 });
14900 codec_case.apply(&mut workflow);
14901 match serde_json::from_value::<WorkflowTask>(workflow) {
14902 Ok(task) => match worker.execute_workflow_task(task) {
14903 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
14904 outcome => failures.push(format!(
14905 "workflow {} codec returned {outcome:?}",
14906 codec_case.label()
14907 )),
14908 },
14909 Err(error) => failures.push(format!(
14910 "workflow {} codec failed transport deserialization: {error}",
14911 codec_case.label()
14912 )),
14913 }
14914
14915 let mut activity = json!({
14916 "task_id": format!("activity-{}", codec_case.label()),
14917 "activity_attempt_id": format!("attempt-{}", codec_case.label()),
14918 "activity_type": "codec.activity",
14919 "attempt_number": 1
14920 });
14921 codec_case.apply(&mut activity);
14922 match serde_json::from_value::<ActivityTask>(activity) {
14923 Ok(task) => match worker.execute_activity_task(task).await {
14924 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
14925 outcome => failures.push(format!(
14926 "activity {} codec returned {outcome:?}",
14927 codec_case.label()
14928 )),
14929 },
14930 Err(error) => failures.push(format!(
14931 "activity {} codec failed transport deserialization: {error}",
14932 codec_case.label()
14933 )),
14934 }
14935
14936 let mut query = json!({
14937 "query_task_id": format!("query-{}", codec_case.label()),
14938 "workflow_type": "codec.workflow",
14939 "query_name": "known"
14940 });
14941 codec_case.apply(&mut query);
14942 match serde_json::from_value::<QueryTask>(query) {
14943 Ok(task) => match worker.execute_query_task(task).await {
14944 Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
14945 outcome => failures.push(format!(
14946 "query {} codec returned {outcome:?}",
14947 codec_case.label()
14948 )),
14949 },
14950 Err(error) => failures.push(format!(
14951 "query {} codec failed transport deserialization: {error}",
14952 codec_case.label()
14953 )),
14954 }
14955 }
14956
14957 assert!(failures.is_empty(), "{}", failures.join("\n"));
14958 assert_eq!(
14959 handler_calls.load(Ordering::SeqCst),
14960 0,
14961 "invalid task codecs must not invoke a handler"
14962 );
14963 }
14964
14965 #[tokio::test]
14966 async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
14967 for codec_case in [
14968 InvalidTaskPayloadCodec::Missing,
14969 InvalidTaskPayloadCodec::Null,
14970 InvalidTaskPayloadCodec::NonString,
14971 ] {
14972 let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
14973 let client = Client::builder(server.base_url())
14974 .timeout(Duration::from_secs(2))
14975 .build()
14976 .expect("client");
14977 let mut worker = Worker::new(client, "rust-workers")
14978 .worker_id("codec-worker")
14979 .poll_timeout(Duration::from_millis(10));
14980 let handler_calls = Arc::new(AtomicUsize::new(0));
14981
14982 let calls = Arc::clone(&handler_calls);
14983 worker.register_workflow("codec.workflow", move |_ctx, _args| {
14984 calls.fetch_add(1, Ordering::SeqCst);
14985 async move { Ok(Value::Null) }
14986 });
14987 let calls = Arc::clone(&handler_calls);
14988 worker.register_activity("codec.activity", move |_ctx, _args| {
14989 calls.fetch_add(1, Ordering::SeqCst);
14990 async move { Ok(Value::Null) }
14991 });
14992 let calls = Arc::clone(&handler_calls);
14993 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
14994 calls.fetch_add(1, Ordering::SeqCst);
14995 async move { Ok(Value::Null) }
14996 });
14997
14998 assert_eq!(
14999 worker.run_once().await.expect("invalid tasks are settled"),
15000 3,
15001 "all {} codec tasks must be handled",
15002 codec_case.label()
15003 );
15004 assert_eq!(
15005 handler_calls.load(Ordering::SeqCst),
15006 0,
15007 "{} task codecs must fail before every handler",
15008 codec_case.label()
15009 );
15010
15011 for path in [
15012 "/api/worker/workflow-tasks/codec-workflow/fail",
15013 "/api/worker/activity-tasks/codec-activity/fail",
15014 "/api/worker/query-tasks/codec-query/fail",
15015 ] {
15016 let body = server.request_body(path);
15017 assert!(
15018 body["failure"]["message"]
15019 .as_str()
15020 .is_some_and(|message| message.contains("unsupported_payload_codec")),
15021 "{path} must receive the stable codec diagnostic for the {} case: {body}",
15022 codec_case.label()
15023 );
15024 }
15025 assert_eq!(
15026 server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
15027 ["reason"],
15028 "query_payload_decode_failed"
15029 );
15030 for path in [
15031 "/api/worker/workflow-tasks/codec-workflow/complete",
15032 "/api/worker/activity-tasks/codec-activity/complete",
15033 "/api/worker/query-tasks/codec-query/complete",
15034 ] {
15035 assert_eq!(
15036 server.request_count(path),
15037 0,
15038 "invalid {} codec task reached {path}",
15039 codec_case.label()
15040 );
15041 }
15042 }
15043 }
15044
15045 #[tokio::test]
15046 async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
15047 let client = Client::new("http://127.0.0.1:8080").expect("client");
15048 let mut worker = Worker::new(client, "rust-workers");
15049 let handler_calls = Arc::new(AtomicUsize::new(0));
15050
15051 let calls = Arc::clone(&handler_calls);
15052 worker.register_workflow("codec.workflow", move |_ctx, _args| {
15053 calls.fetch_add(1, Ordering::SeqCst);
15054 async move { Ok(Value::Null) }
15055 });
15056 let calls = Arc::clone(&handler_calls);
15057 worker.register_activity("codec.activity", move |_ctx, _args| {
15058 calls.fetch_add(1, Ordering::SeqCst);
15059 async move { Ok(Value::Null) }
15060 });
15061 let calls = Arc::clone(&handler_calls);
15062 worker.register_update("codec.workflow", "known", move |_ctx, _args| {
15063 calls.fetch_add(1, Ordering::SeqCst);
15064 async move { Ok(Value::Null) }
15065 });
15066 let calls = Arc::clone(&handler_calls);
15067 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
15068 calls.fetch_add(1, Ordering::SeqCst);
15069 async move { Ok(Value::Null) }
15070 });
15071
15072 let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
15073 workflow.payload_codec = "json".to_string();
15074 workflow.arguments = None;
15075 let error = worker
15076 .execute_workflow_task(workflow)
15077 .expect_err("task codec must be checked before workflow invocation");
15078 assert!(error.to_string().contains("unsupported_payload_codec"));
15079
15080 let activity = ActivityTask {
15081 task_id: "activity-invalid-codec".to_string(),
15082 activity_attempt_id: None,
15083 attempt_id: None,
15084 activity_type: "codec.activity".to_string(),
15085 payload_codec: "unknown".to_string(),
15086 arguments: None,
15087 attempt_number: 1,
15088 lease_owner: None,
15089 };
15090 let error = worker
15091 .execute_activity_task(activity)
15092 .await
15093 .expect_err("task codec must be checked before activity invocation");
15094 assert!(error.to_string().contains("unsupported_payload_codec"));
15095
15096 let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
15097 update.workflow_update_id = Some("update-invalid-codec".to_string());
15098 update.update_name = Some("known".to_string());
15099 update.history_events.push(history_event(
15100 "UpdateAccepted",
15101 json!({
15102 "update_id": "update-invalid-codec",
15103 "update_name": "known",
15104 "arguments": {"codec": "json", "blob": null}
15105 }),
15106 ));
15107 let error = worker
15108 .execute_workflow_task(update)
15109 .expect_err("nested update codec must be checked before handler lookup");
15110 assert!(error.to_string().contains("unsupported_payload_codec"));
15111
15112 let query: QueryTask = serde_json::from_value(json!({
15113 "query_task_id": "query-invalid-codec",
15114 "workflow_type": "codec.workflow",
15115 "query_name": "known",
15116 "payload_codec": DEFAULT_CODEC,
15117 "workflow_arguments": null,
15118 "query_arguments": null,
15119 "history_export": {
15120 "payloads": {"codec": DEFAULT_CODEC},
15121 "signals": [{
15122 "name": "empty",
15123 "payload_codec": "json",
15124 "arguments": null
15125 }]
15126 }
15127 }))
15128 .expect("query task");
15129 let failure = worker
15130 .execute_query_task(query)
15131 .await
15132 .expect_err("exported signal codec must be checked before query invocation");
15133 assert_eq!(failure.reason, "query_payload_decode_failed");
15134 assert!(failure.message.contains("unsupported_payload_codec"));
15135
15136 let exported_history: QueryTask = serde_json::from_value(json!({
15137 "query_task_id": "query-invalid-history-codec",
15138 "workflow_type": "codec.workflow",
15139 "query_name": "known",
15140 "payload_codec": DEFAULT_CODEC,
15141 "history_export": {
15142 "payloads": {"codec": DEFAULT_CODEC},
15143 "history_events": [{
15144 "type": "ActivityCompleted",
15145 "payload": {"payload_codec": "unknown", "result": null}
15146 }]
15147 }
15148 }))
15149 .expect("query task");
15150 let failure = worker
15151 .execute_query_task(exported_history)
15152 .await
15153 .expect_err("exported history codec must be checked before query invocation");
15154 assert_eq!(failure.reason, "query_payload_decode_failed");
15155 assert!(failure.message.contains("unsupported_payload_codec"));
15156 assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
15157
15158 let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
15159 unknown_workflow.arguments = None;
15160 unknown_workflow.history_events.push(history_event(
15161 "SignalReceived",
15162 json!({
15163 "signal_name": "empty",
15164 "payload_codec": "json",
15165 "arguments": null
15166 }),
15167 ));
15168 let error = worker
15169 .execute_workflow_task(unknown_workflow)
15170 .expect_err("history codec must precede unknown workflow outcome");
15171 assert!(error.to_string().contains("unsupported_payload_codec"));
15172
15173 let unknown_activity = ActivityTask {
15174 task_id: "activity-unknown".to_string(),
15175 activity_attempt_id: None,
15176 attempt_id: None,
15177 activity_type: "missing".to_string(),
15178 payload_codec: "json".to_string(),
15179 arguments: None,
15180 attempt_number: 1,
15181 lease_owner: None,
15182 };
15183 let error = worker
15184 .execute_activity_task(unknown_activity)
15185 .await
15186 .expect_err("codec must precede unknown activity outcome");
15187 assert!(error.to_string().contains("unsupported_payload_codec"));
15188
15189 let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
15190 unknown_update.payload_codec = "json".to_string();
15191 unknown_update.arguments = None;
15192 unknown_update.workflow_update_id = Some("update-unknown".to_string());
15193 unknown_update.update_name = Some("missing".to_string());
15194 let error = worker
15195 .execute_workflow_task(unknown_update)
15196 .expect_err("codec must precede fail_update shortcut");
15197 assert!(error.to_string().contains("unsupported_payload_codec"));
15198
15199 let unknown_query: QueryTask = serde_json::from_value(json!({
15200 "query_task_id": "query-unknown",
15201 "workflow_type": "missing",
15202 "query_name": "missing",
15203 "payload_codec": "json",
15204 "workflow_arguments": null,
15205 "query_arguments": null
15206 }))
15207 .expect("query task");
15208 let failure = worker
15209 .execute_query_task(unknown_query)
15210 .await
15211 .expect_err("codec must precede unknown query outcome");
15212 assert_eq!(failure.reason, "query_payload_decode_failed");
15213 assert!(failure.message.contains("unsupported_payload_codec"));
15214 }
15215
15216 #[tokio::test]
15217 async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
15218 let client = Client::new("http://127.0.0.1:8080").expect("client");
15219 let worker = Worker::new(client, "rust-workers");
15220
15221 for event_type in ["SignalReceived", "SignalApplied"] {
15222 for (payload_field, codec) in [
15223 ("value", "json"),
15224 ("input", "unknown"),
15225 ("arguments", "json"),
15226 ] {
15227 let payload = json!({
15228 "signal_name": "empty",
15229 payload_field: {"codec": codec, "blob": null}
15230 });
15231 let workflow = workflow_task(
15232 "missing",
15233 vec![history_event(event_type, payload.clone())],
15234 DEFAULT_CODEC,
15235 );
15236 let error = worker
15237 .execute_workflow_task(workflow)
15238 .expect_err("signal payload codec must precede unknown workflow outcome");
15239 assert!(
15240 error.to_string().contains("unsupported_payload_codec"),
15241 "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
15242 );
15243
15244 let query: QueryTask = serde_json::from_value(json!({
15245 "query_task_id": format!("query-{event_type}-{payload_field}"),
15246 "workflow_type": "missing",
15247 "query_name": "missing",
15248 "payload_codec": DEFAULT_CODEC,
15249 "workflow_arguments": null,
15250 "query_arguments": null,
15251 "history_events": [{
15252 "event_type": event_type,
15253 "payload": payload
15254 }]
15255 }))
15256 .expect("query task");
15257 let failure = worker
15258 .execute_query_task(query)
15259 .await
15260 .expect_err("signal payload codec must precede unknown query outcome");
15261 assert_eq!(
15262 failure.reason, "query_payload_decode_failed",
15263 "{event_type}.{payload_field} returned an unrelated query outcome"
15264 );
15265 assert!(
15266 failure.message.contains("unsupported_payload_codec"),
15267 "{event_type}.{payload_field} returned an unrelated query error: {}",
15268 failure.message
15269 );
15270 }
15271 }
15272 }
15273
15274 #[test]
15275 fn workflow_context_schedules_activity_until_completion_is_in_history() {
15276 let ctx = WorkflowContext {
15277 state: Arc::new(Mutex::new(
15278 WorkflowState::new_with_identity(
15279 Vec::new(),
15280 Some("wf-parent".to_string()),
15281 Some("run-parent".to_string()),
15282 "rust-workers".to_string(),
15283 DEFAULT_CODEC.to_string(),
15284 None,
15285 )
15286 .expect("workflow state"),
15287 )),
15288 };
15289
15290 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
15291 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15292 assert!(matches!(
15293 call.as_mut().poll(&mut task_context),
15294 Poll::Pending
15295 ));
15296
15297 let commands = ctx.take_commands().expect("commands");
15298 assert_eq!(commands[0]["type"], "schedule_activity");
15299 assert_eq!(commands[0]["activity_type"], "hello.activity");
15300 }
15301
15302 #[test]
15303 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
15304 let ctx = workflow_context(Vec::new());
15305 let options = ActivityOptions::new()
15306 .task_queue("payments")
15307 .retry_policy(
15308 ActivityRetryPolicy::new(4)
15309 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
15310 .non_retryable_error_type("ValidationError"),
15311 )
15312 .start_to_close_timeout(Duration::from_secs(120))
15313 .schedule_to_start_timeout(Duration::from_secs(10))
15314 .schedule_to_close_timeout(Duration::from_secs(300))
15315 .heartbeat_timeout(Duration::from_secs(15));
15316 let mut call = Box::pin(ctx.activity_with_options(
15317 "charge-card",
15318 options,
15319 json!([{"order_id": "o-1"}]),
15320 ));
15321 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15322
15323 assert!(matches!(
15324 call.as_mut().poll(&mut task_context),
15325 Poll::Pending
15326 ));
15327 assert!(matches!(
15328 call.as_mut().poll(&mut task_context),
15329 Poll::Pending
15330 ));
15331
15332 let commands = ctx.take_commands().expect("activity command");
15333 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
15334 assert_eq!(commands[0]["queue"], "payments");
15335 assert_eq!(
15336 commands[0]["retry_policy"],
15337 json!({
15338 "max_attempts": 4,
15339 "backoff_seconds": [1, 3, 9],
15340 "non_retryable_error_types": ["ValidationError"],
15341 })
15342 );
15343 assert_eq!(commands[0]["start_to_close_timeout"], 120);
15344 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
15345 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
15346 assert_eq!(commands[0]["heartbeat_timeout"], 15);
15347 }
15348
15349 #[test]
15350 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
15351 let ctx = workflow_context(Vec::new());
15352 let options = ActivityOptions::new().retry_policy(
15353 ActivityRetryPolicy::new(3)
15354 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
15355 );
15356 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
15357 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15358
15359 assert!(matches!(
15360 call.as_mut().poll(&mut task_context),
15361 Poll::Pending
15362 ));
15363 assert_eq!(
15364 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
15365 json!([1, 2])
15366 );
15367 }
15368
15369 #[test]
15370 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
15371 let cases = [
15372 (
15373 ActivityOptions::new().task_queue(" "),
15374 ActivityOptionsErrorKind::EmptyTaskQueue,
15375 ),
15376 (
15377 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
15378 ActivityOptionsErrorKind::EmptyRetryPolicy,
15379 ),
15380 (
15381 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
15382 ActivityOptionsErrorKind::InvalidMaxAttempts,
15383 ),
15384 (
15385 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
15386 max_attempts: None,
15387 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
15388 non_retryable_error_types: Vec::new(),
15389 }),
15390 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
15391 ),
15392 (
15393 ActivityOptions::new().retry_policy(
15394 ActivityRetryPolicy::new(2)
15395 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
15396 ),
15397 ActivityOptionsErrorKind::TooManyBackoffIntervals,
15398 ),
15399 (
15400 ActivityOptions::new().retry_policy(
15401 ActivityRetryPolicy::new(2).exponential_backoff(
15402 Duration::from_secs(1),
15403 0,
15404 None,
15405 ),
15406 ),
15407 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
15408 ),
15409 (
15410 ActivityOptions::new()
15411 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
15412 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
15413 ),
15414 (
15415 ActivityOptions::new().retry_policy(
15416 ActivityRetryPolicy::new(10_002).exponential_backoff(
15417 Duration::from_secs(1),
15418 1,
15419 None,
15420 ),
15421 ),
15422 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
15423 ),
15424 (
15425 ActivityOptions::new().retry_policy(
15426 ActivityRetryPolicy::new(2)
15427 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
15428 ),
15429 ActivityOptionsErrorKind::BackoffOverflow,
15430 ),
15431 ];
15432
15433 for (options, expected_kind) in cases {
15434 let ctx = workflow_context(Vec::new());
15435 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
15436 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15437 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
15438 call.as_mut().poll(&mut task_context)
15439 else {
15440 panic!("expected typed activity validation error");
15441 };
15442 assert_eq!(error.kind, expected_kind);
15443 assert!(ctx.take_commands().expect("commands").is_empty());
15444 }
15445 }
15446
15447 #[test]
15448 fn activity_options_validate_positive_and_ordered_timeouts() {
15449 let zero_timeout_cases = [
15450 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
15451 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
15452 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
15453 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
15454 ];
15455 for options in zero_timeout_cases {
15456 assert_eq!(
15457 options.validate().expect_err("zero timeout").kind,
15458 ActivityOptionsErrorKind::TimeoutNotPositive
15459 );
15460 }
15461
15462 let ordering_cases = [
15463 ActivityOptions::new()
15464 .heartbeat_timeout(Duration::from_secs(11))
15465 .start_to_close_timeout(Duration::from_secs(10)),
15466 ActivityOptions::new()
15467 .start_to_close_timeout(Duration::from_secs(31))
15468 .schedule_to_close_timeout(Duration::from_secs(30)),
15469 ActivityOptions::new()
15470 .schedule_to_start_timeout(Duration::from_secs(31))
15471 .schedule_to_close_timeout(Duration::from_secs(30)),
15472 ];
15473 for options in ordering_cases {
15474 assert_eq!(
15475 options.validate().expect_err("timeout order").kind,
15476 ActivityOptionsErrorKind::TimeoutOrder
15477 );
15478 }
15479
15480 assert_eq!(
15481 ActivityOptions::new()
15482 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
15483 .validate()
15484 .expect_err("protocol integer overflow")
15485 .kind,
15486 ActivityOptionsErrorKind::TimeoutOverflow
15487 );
15488 }
15489
15490 #[test]
15491 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
15492 let ctx = workflow_context(completed_retry_activity_history());
15493 let mut call =
15494 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
15495 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15496
15497 assert!(matches!(
15498 call.as_mut().poll(&mut task_context),
15499 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
15500 ));
15501 assert!(ctx.take_commands().expect("commands").is_empty());
15502 ctx.ensure_history_consumed().expect("history consumed");
15503 }
15504
15505 #[test]
15506 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
15507 let mut options = retry_activity_options();
15508 options
15509 .retry_policy
15510 .as_mut()
15511 .expect("retry policy")
15512 .non_retryable_error_types
15513 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
15514
15515 let new_ctx = workflow_context(Vec::new());
15516 let mut new_call =
15517 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
15518 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15519 assert!(matches!(
15520 new_call.as_mut().poll(&mut task_context),
15521 Poll::Pending
15522 ));
15523 let commands = new_ctx.take_commands().expect("commands");
15524 assert_eq!(commands.len(), 1);
15525 assert_eq!(
15526 commands[0]["retry_policy"]["non_retryable_error_types"],
15527 json!(["PermanentError"])
15528 );
15529
15530 let replay_ctx = workflow_context(completed_retry_activity_history());
15531 let mut replay_call =
15532 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
15533 assert!(matches!(
15534 replay_call.as_mut().poll(&mut task_context),
15535 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
15536 ));
15537 assert!(replay_ctx.take_commands().expect("commands").is_empty());
15538 replay_ctx
15539 .ensure_history_consumed()
15540 .expect("history consumed");
15541 }
15542
15543 #[test]
15544 fn replayed_intermediate_retry_remains_pending_across_restarts() {
15545 let history = completed_retry_activity_history()
15546 .into_iter()
15547 .take(3)
15548 .collect::<Vec<_>>();
15549
15550 for _restart in 0..2 {
15551 let ctx = workflow_context(history.clone());
15552 let mut call =
15553 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
15554 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15555 assert!(matches!(
15556 call.as_mut().poll(&mut task_context),
15557 Poll::Pending
15558 ));
15559 assert!(ctx.take_commands().expect("commands").is_empty());
15560 }
15561 }
15562
15563 #[test]
15564 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
15565 let mut changed_queue = retry_activity_options();
15566 changed_queue.task_queue = Some("different-queue".to_string());
15567
15568 let mut changed_max_attempts = retry_activity_options();
15569 let retry_policy = changed_max_attempts
15570 .retry_policy
15571 .as_mut()
15572 .expect("retry policy");
15573 retry_policy.max_attempts = Some(4);
15574
15575 let mut changed_backoff = retry_activity_options();
15576 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
15577 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
15578 Duration::from_secs(3),
15579 Duration::from_secs(4),
15580 ]));
15581
15582 let mut changed_non_retryable_types = retry_activity_options();
15583 let retry_policy = changed_non_retryable_types
15584 .retry_policy
15585 .as_mut()
15586 .expect("retry policy");
15587 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
15588
15589 let mut changed_start_to_close = retry_activity_options();
15590 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
15591 let mut changed_schedule_to_start = retry_activity_options();
15592 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
15593 let mut changed_schedule_to_close = retry_activity_options();
15594 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
15595 let mut changed_heartbeat = retry_activity_options();
15596 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
15597
15598 let cases = [
15599 (changed_queue, "activity_task_queue_mismatch"),
15600 (changed_max_attempts, "activity_retry_policy_mismatch"),
15601 (changed_backoff, "activity_retry_policy_mismatch"),
15602 (
15603 changed_non_retryable_types,
15604 "activity_retry_policy_mismatch",
15605 ),
15606 (changed_start_to_close, "activity_retry_policy_mismatch"),
15607 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
15608 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
15609 (changed_heartbeat, "activity_retry_policy_mismatch"),
15610 ];
15611
15612 for (options, expected_reason) in cases {
15613 let ctx = workflow_context(completed_retry_activity_history());
15614 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
15615 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15616 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15617 call.as_mut().poll(&mut task_context)
15618 else {
15619 panic!("changed activity options must fail replay");
15620 };
15621 assert_eq!(failure.reason, expected_reason);
15622 assert_eq!(failure.sequence, Some(1));
15623 assert!(ctx.take_commands().expect("commands").is_empty());
15624 }
15625 }
15626
15627 #[test]
15628 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
15629 let cases = [
15630 (
15631 "execution_mode",
15632 json!("local"),
15633 "activity_execution_mode_mismatch",
15634 ),
15635 (
15636 "snapshot_version",
15637 json!(2),
15638 "activity_retry_policy_mismatch",
15639 ),
15640 ];
15641
15642 for (field, value, expected_reason) in cases {
15643 let mut history = completed_retry_activity_history();
15644 let activity = history[0].payload["activity"]
15645 .as_object_mut()
15646 .expect("activity snapshot");
15647 if field == "execution_mode" {
15648 activity.insert(field.to_string(), value);
15649 } else {
15650 activity["retry_policy"]
15651 .as_object_mut()
15652 .expect("retry snapshot")
15653 .insert(field.to_string(), value);
15654 }
15655
15656 let ctx = workflow_context(history);
15657 let mut call =
15658 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
15659 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15660 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15661 call.as_mut().poll(&mut task_context)
15662 else {
15663 panic!("changed {field} must fail replay");
15664 };
15665 assert_eq!(failure.reason, expected_reason);
15666 assert_eq!(failure.sequence, Some(1));
15667 assert!(ctx.take_commands().expect("commands").is_empty());
15668 }
15669 }
15670
15671 #[test]
15672 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
15673 let mut history = completed_retry_activity_history();
15674 let activity = history[0].payload["activity"]
15675 .as_object_mut()
15676 .expect("activity snapshot");
15677 activity.remove("execution_mode");
15678 activity.remove("retry_policy");
15679
15680 let mut current = retry_activity_options();
15681 current.start_to_close_timeout = Some(Duration::from_secs(45));
15682 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
15683 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
15684 current.heartbeat_timeout = Some(Duration::from_secs(12));
15685
15686 let ctx = workflow_context(history);
15687 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
15688 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15689 assert!(matches!(
15690 call.as_mut().poll(&mut task_context),
15691 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
15692 ));
15693 assert!(ctx.take_commands().expect("commands").is_empty());
15694 ctx.ensure_history_consumed().expect("history consumed");
15695 }
15696
15697 #[test]
15698 fn terminal_activity_failed_after_start_returns_typed_failure() {
15699 let history = vec![
15700 history_event(
15701 "ActivityScheduled",
15702 json!({
15703 "sequence": 1,
15704 "activity_type": "flaky",
15705 "activity_execution_id": "act-terminal",
15706 "activity": {
15707 "id": "act-terminal",
15708 "sequence": 1,
15709 "type": "flaky",
15710 "queue": "critical-activities",
15711 "retry_policy": {
15712 "snapshot_version": 1,
15713 "max_attempts": 3,
15714 "backoff_seconds": [2, 4],
15715 "non_retryable_error_types": ["PermanentError"]
15716 }
15717 }
15718 }),
15719 ),
15720 history_event(
15721 "ActivityStarted",
15722 json!({
15723 "sequence": 1,
15724 "activity_type": "flaky",
15725 "activity_execution_id": "act-terminal",
15726 "activity_attempt_id": "attempt-1",
15727 "attempt_number": 1
15728 }),
15729 ),
15730 history_event(
15731 "ActivityFailed",
15732 json!({
15733 "sequence": 1,
15734 "activity_type": "flaky",
15735 "activity_execution_id": "act-terminal",
15736 "activity_attempt_id": "attempt-1",
15737 "attempt_number": 1,
15738 "failure_id": "failure-terminal",
15739 "failure_category": "activity",
15740 "exception_type": "PermanentError",
15741 "message": "cannot retry",
15742 "non_retryable": true
15743 }),
15744 ),
15745 ];
15746 let ctx = workflow_context(history);
15747 let mut call =
15748 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
15749 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15750
15751 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
15752 call.as_mut().poll(&mut task_context)
15753 else {
15754 panic!("terminal ActivityFailed must settle the activity future");
15755 };
15756 assert_eq!(failure.kind, ActivityFailureKind::Failed);
15757 assert_eq!(
15758 failure.activity_execution_id.as_deref(),
15759 Some("act-terminal")
15760 );
15761 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
15762 assert!(failure.non_retryable);
15763 assert!(ctx.take_commands().expect("commands").is_empty());
15764 ctx.ensure_history_consumed().expect("history consumed");
15765 }
15766
15767 #[test]
15768 fn activity_terminal_events_return_machine_readable_failures() {
15769 let cases = [
15770 (
15771 "ActivityFailed",
15772 json!({
15773 "sequence": 1,
15774 "activity_type": "charge-card",
15775 "activity_execution_id": "act-1",
15776 "activity_attempt_id": "attempt-2",
15777 "attempt_number": 2,
15778 "failure_id": "failure-1",
15779 "failure_category": "activity",
15780 "exception_type": "PaymentDeclined",
15781 "exception_class": "payments.PaymentDeclined",
15782 "message": "card declined",
15783 "non_retryable": true
15784 }),
15785 ActivityFailureKind::Failed,
15786 "activity",
15787 ),
15788 (
15789 "ActivityCancelled",
15790 json!({
15791 "sequence": 1,
15792 "activity_type": "charge-card",
15793 "activity_execution_id": "act-1",
15794 "activity_attempt_id": "attempt-1"
15795 }),
15796 ActivityFailureKind::Cancelled,
15797 "cancelled",
15798 ),
15799 ];
15800
15801 for (event_type, payload, expected_kind, expected_reason) in cases {
15802 let ctx = workflow_context(vec![history_event(event_type, payload)]);
15803 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
15804 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15805 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
15806 call.as_mut().poll(&mut task_context)
15807 else {
15808 panic!("expected terminal activity failure");
15809 };
15810 assert_eq!(failure.kind, expected_kind);
15811 assert_eq!(failure.reason, expected_reason);
15812 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
15813 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
15814 }
15815 }
15816
15817 #[test]
15818 fn every_activity_timeout_class_is_typed() {
15819 for timeout_kind in [
15820 "start_to_close",
15821 "schedule_to_start",
15822 "schedule_to_close",
15823 "heartbeat",
15824 ] {
15825 let ctx = workflow_context(vec![history_event(
15826 "ActivityTimedOut",
15827 json!({
15828 "sequence": 1,
15829 "activity_type": "slow",
15830 "activity_execution_id": "act-timeout",
15831 "activity_attempt_id": "attempt-timeout",
15832 "failure_category": "timeout",
15833 "timeout_kind": timeout_kind,
15834 "message": "deadline expired"
15835 }),
15836 )]);
15837 let mut call = Box::pin(ctx.activity("slow", json!([])));
15838 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15839 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
15840 call.as_mut().poll(&mut task_context)
15841 else {
15842 panic!("expected timeout failure");
15843 };
15844 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
15845 assert_eq!(failure.reason, timeout_kind);
15846 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
15847 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
15848 }
15849 }
15850
15851 #[test]
15852 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
15853 let ctx = workflow_context(Vec::new());
15854 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
15855 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15856
15857 assert!(matches!(
15858 sleep.as_mut().poll(&mut task_context),
15859 Poll::Pending
15860 ));
15861 assert!(matches!(
15862 sleep.as_mut().poll(&mut task_context),
15863 Poll::Pending
15864 ));
15865
15866 let commands = ctx.take_commands().expect("timer command");
15867 assert_eq!(
15868 commands,
15869 vec![json!({
15870 "type": "start_timer",
15871 "delay_seconds": 2,
15872 })]
15873 );
15874 }
15875
15876 #[test]
15877 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
15878 let history = vec![
15879 history_event(
15880 "TimerScheduled",
15881 json!({
15882 "sequence": 1,
15883 "timer_id": "timer-1",
15884 "delay_seconds": 5,
15885 "fire_at": "2026-07-11T12:00:05Z",
15886 }),
15887 ),
15888 history_event(
15889 "TimerFired",
15890 json!({
15891 "sequence": 1,
15892 "timer_id": "timer-1",
15893 "delay_seconds": 5,
15894 "fire_at": "2026-07-11T12:00:05Z",
15895 "fired_at": "2026-07-11T12:00:05Z",
15896 }),
15897 ),
15898 ];
15899
15900 for _restart in 0..2 {
15901 let ctx = workflow_context(history.clone());
15902 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
15903 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15904 assert!(matches!(
15905 sleep.as_mut().poll(&mut task_context),
15906 Poll::Ready(Ok(()))
15907 ));
15908 assert!(ctx.take_commands().expect("commands").is_empty());
15909 ctx.ensure_history_consumed().expect("history consumed");
15910 }
15911 }
15912
15913 #[test]
15914 fn workflow_sleep_rejects_changed_delay_during_replay() {
15915 let ctx = workflow_context(vec![
15916 history_event(
15917 "TimerScheduled",
15918 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15919 ),
15920 history_event(
15921 "TimerFired",
15922 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15923 ),
15924 ]);
15925 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
15926 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15927
15928 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15929 sleep.as_mut().poll(&mut task_context)
15930 else {
15931 panic!("changed timer delay must be rejected");
15932 };
15933 assert_eq!(failure.reason, "timer_delay_mismatch");
15934 assert_eq!(failure.sequence, Some(1));
15935 }
15936
15937 #[test]
15938 fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
15939 let ctx = workflow_context(Vec::new());
15940 let mut wait = Box::pin(
15941 ctx.wait_condition(
15942 ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
15943 .timeout(Duration::from_millis(60_001)),
15944 || Ok(false),
15945 ),
15946 );
15947 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15948
15949 assert!(matches!(
15950 wait.as_mut().poll(&mut task_context),
15951 Poll::Pending
15952 ));
15953 assert!(matches!(
15954 wait.as_mut().poll(&mut task_context),
15955 Poll::Pending
15956 ));
15957 assert_eq!(
15958 ctx.take_commands().expect("condition command"),
15959 vec![json!({
15960 "type": "open_condition_wait",
15961 "condition_wait_occurrence_id": "rust:condition-wait:0",
15962 "condition_key": "approval.ready",
15963 "condition_definition_fingerprint": "sha256:approval-v1",
15964 "timeout_seconds": 61,
15965 })]
15966 );
15967 }
15968
15969 #[test]
15970 fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
15971 let ctx = workflow_context(Vec::new());
15972 let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
15973 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15974 assert!(matches!(
15975 satisfied.as_mut().poll(&mut task_context),
15976 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
15977 ));
15978
15979 let mut timed_out = Box::pin(wait_condition!(
15980 ctx,
15981 "no-wait",
15982 timeout: Duration::ZERO,
15983 || Ok(false),
15984 ));
15985 assert!(matches!(
15986 timed_out.as_mut().poll(&mut task_context),
15987 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
15988 ));
15989 assert!(ctx.take_commands().expect("commands").is_empty());
15990 }
15991
15992 #[test]
15993 fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
15994 let signal_history = vec![
15995 history_event(
15996 "ConditionWaitOpened",
15997 json!({
15998 "sequence": 4,
15999 "condition_wait_id": "condition:4",
16000 "condition_wait_occurrence_id": "rust:condition-wait:0",
16001 "condition_key": "approval",
16002 "condition_definition_fingerprint": "sha256:approval-v1",
16003 "timeout_seconds": 30,
16004 }),
16005 ),
16006 history_event(
16007 "SignalReceived",
16008 json!({
16009 "workflow_sequence": 4,
16010 "signal_name": "approve",
16011 "arguments": fixture_envelope(json!(["Ada"])),
16012 }),
16013 ),
16014 ];
16015 for _worker_before_or_after_restart in 0..2 {
16016 let ctx = workflow_context(signal_history.clone());
16017 let predicate_ctx = ctx.clone();
16018 let mut wait = Box::pin(
16019 ctx.wait_condition(
16020 ConditionWaitOptions::new("approval", "sha256:approval-v1")
16021 .timeout(Duration::from_secs(30)),
16022 move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
16023 ),
16024 );
16025 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16026 assert!(matches!(
16027 wait.as_mut().poll(&mut task_context),
16028 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16029 ));
16030 assert!(ctx.take_commands().expect("commands").is_empty());
16031 ctx.ensure_history_consumed().expect("condition consumed");
16032 }
16033
16034 let update_history = vec![
16035 history_event(
16036 "ConditionWaitOpened",
16037 json!({
16038 "sequence": 7,
16039 "condition_wait_id": "condition:7",
16040 "condition_wait_occurrence_id": "rust:condition-wait:0",
16041 "condition_key": "update-approval",
16042 "condition_definition_fingerprint": "sha256:update-approval-v1",
16043 }),
16044 ),
16045 history_event(
16046 "UpdateApplied",
16047 json!({
16048 "sequence": 7,
16049 "update_id": "update-1",
16050 "update_name": "approve",
16051 "arguments": fixture_envelope(json!([true])),
16052 }),
16053 ),
16054 ];
16055 let ctx = workflow_context(update_history);
16056 let predicate_ctx = ctx.clone();
16057 let mut wait = Box::pin(ctx.wait_condition(
16058 ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
16059 move || {
16060 Ok(predicate_ctx
16061 .updates("approve")?
16062 .first()
16063 .and_then(|arguments| arguments.first())
16064 .and_then(Value::as_bool)
16065 == Some(true))
16066 },
16067 ));
16068 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16069 assert!(matches!(
16070 wait.as_mut().poll(&mut task_context),
16071 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16072 ));
16073 assert!(ctx.take_commands().expect("commands").is_empty());
16074 ctx.ensure_history_consumed().expect("condition consumed");
16075 }
16076
16077 #[test]
16078 fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
16079 let open_history = vec![
16080 history_event(
16081 "ConditionWaitOpened",
16082 json!({
16083 "sequence": 3,
16084 "condition_wait_id": "condition:3",
16085 "condition_wait_occurrence_id": "rust:condition-wait:0",
16086 "condition_key": "two-votes",
16087 "condition_definition_fingerprint": "sha256:two-votes-v1",
16088 "timeout_seconds": 120,
16089 }),
16090 ),
16091 history_event(
16092 "SignalReceived",
16093 json!({
16094 "workflow_sequence": 3,
16095 "signal_name": "vote",
16096 "arguments": fixture_envelope(json!(["first"])),
16097 }),
16098 ),
16099 ];
16100 for _worker_before_or_after_restart in 0..2 {
16101 let ctx = workflow_context(open_history.clone());
16102 let predicate_ctx = ctx.clone();
16103 let mut wait = Box::pin(
16104 ctx.wait_condition(
16105 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
16106 .timeout(Duration::from_secs(120)),
16107 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
16108 ),
16109 );
16110 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16111 assert!(matches!(
16112 wait.as_mut().poll(&mut task_context),
16113 Poll::Pending
16114 ));
16115 assert_eq!(
16116 ctx.take_commands().expect("reopened condition"),
16117 vec![json!({
16118 "type": "open_condition_wait",
16119 "condition_wait_occurrence_id": "rust:condition-wait:0",
16120 "condition_key": "two-votes",
16121 "condition_definition_fingerprint": "sha256:two-votes-v1",
16122 "timeout_seconds": 120,
16123 })]
16124 );
16125 }
16126
16127 let satisfied_ctx = workflow_context(vec![
16128 history_event(
16129 "ConditionWaitOpened",
16130 json!({
16131 "sequence": 5,
16132 "condition_wait_id": "condition:5",
16133 "condition_wait_occurrence_id": "rust:condition-wait:0",
16134 "condition_key": "approval",
16135 "condition_definition_fingerprint": "sha256:approval-v1",
16136 }),
16137 ),
16138 history_event(
16139 "ConditionWaitSatisfied",
16140 json!({
16141 "sequence": 5,
16142 "condition_wait_id": "condition:5",
16143 "condition_wait_occurrence_id": "rust:condition-wait:0",
16144 "condition_key": "approval",
16145 "condition_definition_fingerprint": "sha256:approval-v1",
16146 }),
16147 ),
16148 ]);
16149 let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
16150 ConditionWaitOptions::new("approval", "sha256:approval-v1"),
16151 || Ok(false),
16152 ));
16153 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16154 assert!(matches!(
16155 satisfied.as_mut().poll(&mut task_context),
16156 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16157 ));
16158
16159 let timed_out_ctx = workflow_context(vec![
16160 history_event(
16161 "ConditionWaitOpened",
16162 json!({
16163 "sequence": 8,
16164 "condition_wait_id": "condition:8",
16165 "condition_wait_occurrence_id": "rust:condition-wait:0",
16166 "condition_key": "approval-timeout",
16167 "condition_definition_fingerprint": "sha256:approval-timeout-v1",
16168 "timeout_seconds": 5,
16169 }),
16170 ),
16171 history_event(
16172 "TimerScheduled",
16173 json!({
16174 "sequence": 9,
16175 "timer_id": "condition-timer:9",
16176 "timer_kind": "condition_timeout",
16177 "condition_wait_id": "condition:8",
16178 "delay_seconds": 5,
16179 }),
16180 ),
16181 history_event(
16182 "TimerFired",
16183 json!({
16184 "sequence": 9,
16185 "timer_id": "condition-timer:9",
16186 "timer_kind": "condition_timeout",
16187 "condition_wait_id": "condition:8",
16188 "delay_seconds": 5,
16189 }),
16190 ),
16191 ]);
16192 let mut timed_out = Box::pin(
16193 timed_out_ctx.wait_condition(
16194 ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
16195 .timeout(Duration::from_secs(5)),
16196 || Ok(true),
16197 ),
16198 );
16199 assert!(matches!(
16200 timed_out.as_mut().poll(&mut task_context),
16201 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
16202 ));
16203 }
16204
16205 #[test]
16206 fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
16207 let history = vec![
16208 history_event(
16209 "ConditionWaitOpened",
16210 json!({
16211 "sequence": 3,
16212 "condition_wait_id": "condition:3",
16213 "condition_wait_occurrence_id": "rust:condition-wait:0",
16214 "condition_key": "two-votes",
16215 "condition_definition_fingerprint": "sha256:two-votes-v1",
16216 }),
16217 ),
16218 history_event(
16219 "SignalReceived",
16220 json!({
16221 "workflow_sequence": 3,
16222 "signal_name": "vote",
16223 "arguments": fixture_envelope(json!(["first"])),
16224 }),
16225 ),
16226 history_event(
16227 "ConditionWaitSatisfied",
16228 json!({
16229 "sequence": 3,
16230 "condition_wait_id": "condition:3",
16231 "condition_wait_occurrence_id": "rust:condition-wait:0",
16232 "condition_key": "two-votes",
16233 "condition_definition_fingerprint": "sha256:two-votes-v1",
16234 }),
16235 ),
16236 history_event(
16237 "ConditionWaitOpened",
16238 json!({
16239 "sequence": 5,
16240 "condition_wait_id": "condition:5",
16241 "condition_wait_occurrence_id": "rust:condition-wait:0",
16242 "condition_key": "two-votes",
16243 "condition_definition_fingerprint": "sha256:two-votes-v1",
16244 }),
16245 ),
16246 history_event(
16247 "SignalReceived",
16248 json!({
16249 "workflow_sequence": 5,
16250 "signal_name": "vote",
16251 "arguments": fixture_envelope(json!(["second"])),
16252 }),
16253 ),
16254 history_event(
16255 "ConditionWaitSatisfied",
16256 json!({
16257 "sequence": 5,
16258 "condition_wait_id": "condition:5",
16259 "condition_wait_occurrence_id": "rust:condition-wait:0",
16260 "condition_key": "two-votes",
16261 "condition_definition_fingerprint": "sha256:two-votes-v1",
16262 }),
16263 ),
16264 ];
16265 for _cold_worker_or_restart in 0..2 {
16266 let ctx = workflow_context(history.clone());
16267 let predicate_ctx = ctx.clone();
16268 let mut wait = Box::pin(ctx.wait_condition(
16269 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
16270 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
16271 ));
16272 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16273
16274 assert!(matches!(
16275 wait.as_mut().poll(&mut task_context),
16276 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16277 ));
16278 assert!(ctx.take_commands().expect("commands").is_empty());
16279 ctx.ensure_history_consumed()
16280 .expect("every physical wait-open is consumed");
16281 }
16282 }
16283
16284 #[test]
16285 fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
16286 let history = vec![
16287 history_event(
16288 "ConditionWaitOpened",
16289 json!({
16290 "sequence": 3,
16291 "condition_wait_id": "condition:3",
16292 "condition_wait_occurrence_id": "rust:condition-wait:0",
16293 "condition_key": "approved",
16294 "condition_definition_fingerprint": "sha256:approved-v1",
16295 }),
16296 ),
16297 history_event(
16298 "UpdateApplied",
16299 json!({
16300 "sequence": 3,
16301 "update_id": "update-1",
16302 "update_name": "approve",
16303 "arguments": fixture_envelope(json!([false])),
16304 }),
16305 ),
16306 history_event(
16307 "ConditionWaitOpened",
16308 json!({
16309 "sequence": 5,
16310 "condition_wait_id": "condition:5",
16311 "condition_wait_occurrence_id": "rust:condition-wait:0",
16312 "condition_key": "approved",
16313 "condition_definition_fingerprint": "sha256:approved-v1",
16314 }),
16315 ),
16316 history_event(
16317 "UpdateApplied",
16318 json!({
16319 "sequence": 5,
16320 "update_id": "update-2",
16321 "update_name": "approve",
16322 "arguments": fixture_envelope(json!([true])),
16323 }),
16324 ),
16325 ];
16326
16327 for _cold_worker_or_restart in 0..2 {
16328 let ctx = workflow_context(history.clone());
16329 let predicate_ctx = ctx.clone();
16330 let mut wait = Box::pin(ctx.wait_condition(
16331 ConditionWaitOptions::new("approved", "sha256:approved-v1"),
16332 move || {
16333 Ok(predicate_ctx
16334 .updates("approve")?
16335 .last()
16336 .and_then(|arguments| arguments.first())
16337 .and_then(Value::as_bool)
16338 == Some(true))
16339 },
16340 ));
16341 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16342
16343 assert!(matches!(
16344 wait.as_mut().poll(&mut task_context),
16345 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16346 ));
16347 assert!(ctx.take_commands().expect("commands").is_empty());
16348 ctx.ensure_history_consumed()
16349 .expect("every update-driven reopen is consumed");
16350 }
16351 }
16352
16353 #[test]
16354 fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
16355 for (first_key, first_fingerprint, second_key, second_fingerprint) in [
16356 ("shared", "sha256:first", "shared", "sha256:second"),
16357 ("first", "sha256:shared", "second", "sha256:shared"),
16358 ("shared", "sha256:shared", "shared", "sha256:shared"),
16359 ("first", "sha256:first", "second", "sha256:second"),
16360 ] {
16361 let history = vec![
16362 history_event(
16363 "ConditionWaitOpened",
16364 json!({
16365 "sequence": 3,
16366 "condition_wait_id": "condition:3",
16367 "condition_wait_occurrence_id": "rust:condition-wait:0",
16368 "condition_key": first_key,
16369 "condition_definition_fingerprint": first_fingerprint,
16370 }),
16371 ),
16372 history_event(
16373 "ConditionWaitSatisfied",
16374 json!({
16375 "sequence": 3,
16376 "condition_wait_id": "condition:3",
16377 "condition_wait_occurrence_id": "rust:condition-wait:0",
16378 "condition_key": first_key,
16379 "condition_definition_fingerprint": first_fingerprint,
16380 }),
16381 ),
16382 history_event(
16383 "ConditionWaitOpened",
16384 json!({
16385 "sequence": 4,
16386 "condition_wait_id": "condition:4",
16387 "condition_wait_occurrence_id": "rust:condition-wait:1",
16388 "condition_key": second_key,
16389 "condition_definition_fingerprint": second_fingerprint,
16390 }),
16391 ),
16392 history_event(
16393 "ConditionWaitSatisfied",
16394 json!({
16395 "sequence": 4,
16396 "condition_wait_id": "condition:4",
16397 "condition_wait_occurrence_id": "rust:condition-wait:1",
16398 "condition_key": second_key,
16399 "condition_definition_fingerprint": second_fingerprint,
16400 }),
16401 ),
16402 ];
16403 for _cold_worker_or_restart in 0..2 {
16404 let ctx = workflow_context(history.clone());
16405 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16406 let mut first = Box::pin(ctx.wait_condition(
16407 ConditionWaitOptions::new(first_key, first_fingerprint),
16408 || Ok(false),
16409 ));
16410 assert!(matches!(
16411 first.as_mut().poll(&mut task_context),
16412 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16413 ));
16414
16415 let mut second = Box::pin(ctx.wait_condition(
16416 ConditionWaitOptions::new(second_key, second_fingerprint),
16417 || Ok(false),
16418 ));
16419 assert!(matches!(
16420 second.as_mut().poll(&mut task_context),
16421 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
16422 ));
16423 assert!(ctx.take_commands().expect("commands").is_empty());
16424 ctx.ensure_history_consumed()
16425 .expect("each authored wait consumes one occurrence");
16426 }
16427 }
16428 }
16429
16430 #[test]
16431 fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
16432 fn worker() -> Worker {
16433 let client = Client::new("http://127.0.0.1:8080").expect("client");
16434 let mut worker = Worker::new(client, "rust-workers");
16435 worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
16436 let mut outcomes = Vec::new();
16437 for _ in 0..2 {
16438 outcomes.push(
16439 ctx.wait_condition(
16440 ConditionWaitOptions::new("shared", "sha256:shared"),
16441 || Ok(false),
16442 )
16443 .await?,
16444 );
16445 }
16446 Ok(json!(outcomes))
16447 });
16448 worker
16449 }
16450
16451 let task = workflow_task(
16452 "rust.condition-loop",
16453 vec![
16454 history_event(
16455 "ConditionWaitOpened",
16456 json!({
16457 "sequence": 1,
16458 "condition_wait_id": "condition:1",
16459 "condition_wait_occurrence_id": "rust:condition-wait:0",
16460 "condition_key": "shared",
16461 "condition_definition_fingerprint": "sha256:shared",
16462 }),
16463 ),
16464 history_event(
16465 "ConditionWaitSatisfied",
16466 json!({
16467 "sequence": 1,
16468 "condition_wait_id": "condition:1",
16469 "condition_wait_occurrence_id": "rust:condition-wait:0",
16470 "condition_key": "shared",
16471 "condition_definition_fingerprint": "sha256:shared",
16472 }),
16473 ),
16474 history_event(
16475 "ConditionWaitOpened",
16476 json!({
16477 "sequence": 2,
16478 "condition_wait_id": "condition:2",
16479 "condition_wait_occurrence_id": "rust:condition-wait:1",
16480 "condition_key": "shared",
16481 "condition_definition_fingerprint": "sha256:shared",
16482 }),
16483 ),
16484 history_event(
16485 "ConditionWaitSatisfied",
16486 json!({
16487 "sequence": 2,
16488 "condition_wait_id": "condition:2",
16489 "condition_wait_occurrence_id": "rust:condition-wait:1",
16490 "condition_key": "shared",
16491 "condition_definition_fingerprint": "sha256:shared",
16492 }),
16493 ),
16494 ],
16495 DEFAULT_CODEC,
16496 );
16497
16498 for _cold_worker_or_restart in 0..2 {
16499 let commands = worker()
16500 .execute_workflow_task(task.clone())
16501 .expect("adjacent loop waits replay deterministically");
16502 assert_eq!(commands.len(), 1);
16503 assert_eq!(commands[0]["type"], "complete_workflow");
16504 assert_eq!(
16505 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
16506 json!(["satisfied", "satisfied"])
16507 );
16508 }
16509 }
16510
16511 #[test]
16512 fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
16513 let history = vec![history_event(
16514 "ConditionWaitOpened",
16515 json!({
16516 "sequence": 12,
16517 "condition_wait_id": "condition:12",
16518 "condition_wait_occurrence_id": "rust:condition-wait:0",
16519 "condition_key": "approval",
16520 "condition_definition_fingerprint": "sha256:approval-v1",
16521 "timeout_seconds": 30,
16522 }),
16523 )];
16524 for (options, expected_reason) in [
16525 (
16526 ConditionWaitOptions::new("changed", "sha256:approval-v1")
16527 .timeout(Duration::from_secs(30)),
16528 "condition_wait_key_mismatch",
16529 ),
16530 (
16531 ConditionWaitOptions::new("approval", "sha256:approval-v2")
16532 .timeout(Duration::from_secs(30)),
16533 "condition_wait_predicate_mismatch",
16534 ),
16535 (
16536 ConditionWaitOptions::new("approval", "sha256:approval-v1")
16537 .timeout(Duration::from_secs(29)),
16538 "condition_wait_timeout_mismatch",
16539 ),
16540 ] {
16541 let ctx = workflow_context(history.clone());
16542 let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
16543 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16544 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
16545 wait.as_mut().poll(&mut task_context)
16546 else {
16547 panic!("changed condition definition must fail replay");
16548 };
16549 assert_eq!(failure.reason, expected_reason);
16550 assert_eq!(failure.sequence, Some(12));
16551 }
16552 }
16553
16554 #[test]
16555 fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
16556 let error = WorkflowState::new(
16557 vec![history_event(
16558 "ConditionWaitOpened",
16559 json!({
16560 "sequence": 12,
16561 "condition_wait_id": "condition:12",
16562 "condition_wait_occurrence_id": "rust:condition-wait:0",
16563 "condition_key": "approval",
16564 }),
16565 )],
16566 "rust-workers".to_string(),
16567 DEFAULT_CODEC.to_string(),
16568 None,
16569 )
16570 .expect_err("condition history without a predicate fingerprint must fail");
16571
16572 assert!(matches!(
16573 error,
16574 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16575 if reason == "condition_wait_predicate_fingerprint_missing"
16576 ));
16577 }
16578
16579 #[test]
16580 fn condition_wait_history_requires_authored_occurrence_identity() {
16581 let error = WorkflowState::new(
16582 vec![history_event(
16583 "ConditionWaitOpened",
16584 json!({
16585 "sequence": 12,
16586 "condition_wait_id": "condition:12",
16587 "condition_key": "approval",
16588 "condition_definition_fingerprint": "sha256:approval-v1",
16589 }),
16590 )],
16591 "rust-workers".to_string(),
16592 DEFAULT_CODEC.to_string(),
16593 None,
16594 )
16595 .expect_err("condition history without occurrence identity must fail");
16596
16597 assert!(matches!(
16598 error,
16599 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16600 if reason == "condition_wait_occurrence_id_missing"
16601 ));
16602 }
16603
16604 #[test]
16605 fn typed_search_attribute_updates_validate_emit_and_replay() {
16606 let update = SearchAttributeUpdate::new()
16607 .keyword("OrderStatus", " waiting ")
16608 .expect("keyword")
16609 .int("Attempt", 3)
16610 .expect("int")
16611 .bool("Escalated", false)
16612 .expect("bool")
16613 .keyword_list("Regions", ["us-east", "eu-west"])
16614 .expect("list")
16615 .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
16616 .expect("datetime")
16617 .delete("LegacyStatus")
16618 .expect("delete");
16619 let ctx = workflow_context(Vec::new());
16620 ctx.upsert_search_attributes(update.clone())
16621 .expect("typed update");
16622 assert_eq!(
16623 ctx.take_commands().expect("search-attribute command"),
16624 vec![json!({
16625 "type": "upsert_search_attributes",
16626 "attributes": {
16627 "Attempt": 3,
16628 "Escalated": false,
16629 "LegacyStatus": null,
16630 "OrderStatus": "waiting",
16631 "Regions": ["us-east", "eu-west"],
16632 "UpdatedAt": "2026-08-22T04:00:00Z",
16633 },
16634 "attribute_types": {
16635 "Attempt": "int",
16636 "Escalated": "bool",
16637 "OrderStatus": "keyword",
16638 "Regions": "keyword_list",
16639 "UpdatedAt": "datetime",
16640 },
16641 })]
16642 );
16643
16644 let replay = workflow_context(vec![history_event(
16645 "SearchAttributesUpserted",
16646 json!({
16647 "sequence": 6,
16648 "attributes": {
16649 "Attempt": 3,
16650 "Escalated": false,
16651 "LegacyStatus": null,
16652 "OrderStatus": "waiting",
16653 "Regions": ["us-east", "eu-west"],
16654 "UpdatedAt": "2026-08-22T04:00:00Z",
16655 },
16656 "attribute_types": {
16657 "Attempt": "int",
16658 "Escalated": "bool",
16659 "OrderStatus": "keyword",
16660 "Regions": "keyword_list",
16661 "UpdatedAt": "datetime",
16662 },
16663 "merged": {},
16664 }),
16665 )]);
16666 replay
16667 .upsert_search_attributes(update)
16668 .expect("matching update replays");
16669 assert!(replay.take_commands().expect("commands").is_empty());
16670 replay.ensure_history_consumed().expect("history consumed");
16671
16672 let type_drift = workflow_context(vec![history_event(
16673 "SearchAttributesUpserted",
16674 json!({
16675 "sequence": 7,
16676 "attributes": {"OrderStatus": "waiting"},
16677 "attribute_types": {"OrderStatus": "keyword"},
16678 "merged": {"OrderStatus": "waiting"},
16679 }),
16680 )]);
16681 let error = type_drift
16682 .upsert_search_attributes(
16683 SearchAttributeUpdate::new()
16684 .string("OrderStatus", "waiting")
16685 .expect("string update"),
16686 )
16687 .expect_err("same JSON value with a changed type must fail replay");
16688 assert!(matches!(
16689 error,
16690 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16691 if reason == "search_attribute_type_mismatch"
16692 ));
16693
16694 let malformed_types = WorkflowState::new(
16695 vec![history_event(
16696 "SearchAttributesUpserted",
16697 json!({
16698 "sequence": 8,
16699 "attributes": {"OrderStatus": "waiting"},
16700 "attribute_types": {"OrderStatus": "unsupported"},
16701 "merged": {"OrderStatus": "waiting"},
16702 }),
16703 )],
16704 "rust-workers".to_string(),
16705 DEFAULT_CODEC.to_string(),
16706 None,
16707 )
16708 .expect_err("unsupported search-attribute type metadata must fail");
16709 assert!(matches!(
16710 malformed_types,
16711 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16712 if reason == "search_attribute_types_malformed"
16713 ));
16714
16715 assert!(matches!(
16716 SearchAttributeUpdate::new().keyword("bad key", "value"),
16717 Err(SearchAttributeUpdateError::InvalidKey(_))
16718 ));
16719 assert!(matches!(
16720 SearchAttributeUpdate::new().float("Ratio", f64::NAN),
16721 Err(SearchAttributeUpdateError::NonFiniteFloat(_))
16722 ));
16723 assert!(matches!(
16724 SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
16725 Err(SearchAttributeUpdateError::ValueTooLong { .. })
16726 ));
16727 assert!(matches!(
16728 SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
16729 Err(SearchAttributeUpdateError::InvalidDateTime(_))
16730 ));
16731 assert!(matches!(
16732 workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
16733 Err(Error::InvalidSearchAttributeUpdate(
16734 SearchAttributeUpdateError::Empty
16735 ))
16736 ));
16737 }
16738
16739 #[test]
16740 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
16741 let lone_fire = WorkflowState::new(
16742 vec![history_event(
16743 "TimerFired",
16744 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16745 )],
16746 "rust-workers".to_string(),
16747 DEFAULT_CODEC.to_string(),
16748 None,
16749 )
16750 .expect_err("TimerFired requires TimerScheduled");
16751 assert!(matches!(
16752 lone_fire,
16753 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16754 if reason == "timer_schedule_missing_or_duplicate"
16755 ));
16756
16757 let wrong_identity = WorkflowState::new(
16758 vec![
16759 history_event(
16760 "TimerScheduled",
16761 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16762 ),
16763 history_event(
16764 "TimerFired",
16765 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
16766 ),
16767 ],
16768 "rust-workers".to_string(),
16769 DEFAULT_CODEC.to_string(),
16770 None,
16771 )
16772 .expect_err("fire must match scheduled timer identity");
16773 assert!(matches!(
16774 wrong_identity,
16775 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16776 if reason == "timer_identity_mismatch"
16777 ));
16778
16779 let duplicate_fire = WorkflowState::new(
16780 vec![
16781 history_event(
16782 "TimerScheduled",
16783 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16784 ),
16785 history_event(
16786 "TimerFired",
16787 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16788 ),
16789 history_event(
16790 "TimerFired",
16791 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16792 ),
16793 ],
16794 "rust-workers".to_string(),
16795 DEFAULT_CODEC.to_string(),
16796 None,
16797 )
16798 .expect_err("a durable timer cannot fire twice");
16799 assert!(matches!(
16800 duplicate_fire,
16801 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16802 if reason == "duplicate_timer_fire"
16803 ));
16804
16805 let wrong_fired_delay = WorkflowState::new(
16806 vec![
16807 history_event(
16808 "TimerScheduled",
16809 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16810 ),
16811 history_event(
16812 "TimerFired",
16813 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
16814 ),
16815 ],
16816 "rust-workers".to_string(),
16817 DEFAULT_CODEC.to_string(),
16818 None,
16819 )
16820 .expect_err("timer schedule and fire delays must agree");
16821 assert!(matches!(
16822 wrong_fired_delay,
16823 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16824 if reason == "timer_history_delay_mismatch"
16825 ));
16826 }
16827
16828 #[test]
16829 fn replay_rejects_activity_moved_before_recorded_timer() {
16830 let ctx = workflow_context(vec![
16831 history_event(
16832 "TimerScheduled",
16833 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16834 ),
16835 history_event(
16836 "TimerFired",
16837 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16838 ),
16839 history_event(
16840 "ActivityCompleted",
16841 json!({
16842 "sequence": 2,
16843 "activity_type": "after-timer",
16844 "payload_codec": DEFAULT_CODEC,
16845 "result": fixture_envelope(json!("done")),
16846 }),
16847 ),
16848 ]);
16849 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
16850 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16851
16852 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
16853 activity.as_mut().poll(&mut task_context)
16854 else {
16855 panic!("reordered durable command must be rejected");
16856 };
16857 assert_eq!(failure.reason, "recorded_command_mismatch");
16858 assert_eq!(failure.sequence, Some(1));
16859 assert_eq!(failure.expected.as_deref(), Some("timer"));
16860 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
16861 }
16862
16863 #[test]
16864 fn workflow_context_emits_a_typed_named_signal_wait() {
16865 let ctx = workflow_context(Vec::new());
16866 let mut signal = Box::pin(ctx.wait_signal("finish"));
16867 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16868
16869 assert!(matches!(
16870 signal.as_mut().poll(&mut task_context),
16871 Poll::Pending
16872 ));
16873 assert_eq!(
16874 ctx.take_commands().expect("signal-wait command"),
16875 vec![json!({
16876 "type": "open_signal_wait",
16877 "signal_name": "finish",
16878 })]
16879 );
16880 }
16881
16882 #[test]
16883 fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
16884 let ctx = workflow_context(Vec::new());
16885 let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
16886 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16887
16888 let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
16889 else {
16890 panic!("runtime-reserved signal should be rejected");
16891 };
16892 assert!(message.contains("reserved by the workflow runtime"));
16893 assert!(ctx.take_commands().expect("commands").is_empty());
16894 }
16895
16896 #[tokio::test]
16897 async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
16898 let client = Client::builder("http://127.0.0.1:9")
16899 .build()
16900 .expect("client");
16901 let error = client
16902 .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
16903 .await
16904 .expect_err("runtime-reserved signal should be rejected before transport");
16905
16906 assert!(
16907 matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
16908 );
16909 }
16910
16911 #[test]
16912 fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
16913 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
16914 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
16915 value.to_string(),
16916 )]))
16917 .expect("message payload");
16918 json!({
16919 "schema": MESSAGE_STREAM_SCHEMA,
16920 "stream_name": "orders",
16921 "message_id": message_id,
16922 "position": position,
16923 "payload_envelope": payload,
16924 })
16925 }
16926
16927 fn opened(sequence: u64) -> HistoryEvent {
16928 history_event(
16929 "SignalWaitOpened",
16930 json!({
16931 "sequence": sequence,
16932 "signal_name": MESSAGE_STREAM_SIGNAL,
16933 }),
16934 )
16935 }
16936
16937 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
16938 history_event(
16939 "SignalApplied",
16940 json!({
16941 "sequence": sequence,
16942 "signal_name": MESSAGE_STREAM_SIGNAL,
16943 "value": fixture_envelope(json!([delivery])),
16944 }),
16945 )
16946 }
16947
16948 fn received(delivery: Value) -> HistoryEvent {
16949 history_event(
16950 "SignalReceived",
16951 json!({
16952 "signal_name": MESSAGE_STREAM_SIGNAL,
16953 "arguments": fixture_envelope(json!([delivery])),
16954 "payload_codec": DEFAULT_CODEC,
16955 }),
16956 )
16957 }
16958
16959 let client = Client::new("http://127.0.0.1:8080").expect("client");
16960 let mut worker = Worker::new(client, "rust-workers");
16961 worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
16962 let messages = ctx.message_stream("orders")?.receive(2).await?;
16963 Ok(json!(messages
16964 .into_iter()
16965 .map(|message| message.message_id)
16966 .collect::<Vec<_>>()))
16967 });
16968
16969 let first = delivery("message-1", 1, "one");
16970 let second = delivery("message-2", 2, "two");
16971 let batch = worker
16972 .execute_workflow_task_decision(workflow_task(
16973 "rust.message-stream-batch",
16974 vec![
16975 opened(1),
16976 received(first.clone()),
16977 applied(1, first.clone()),
16978 received(first.clone()),
16979 received(second),
16980 ],
16981 DEFAULT_CODEC,
16982 ))
16983 .expect("worker task consumes the available batch");
16984
16985 assert_eq!(batch.commands.len(), 1);
16986 assert_eq!(batch.commands[0]["type"], "complete_workflow");
16987 assert_eq!(
16988 decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
16989 .expect("workflow result"),
16990 json!(["message-1", "message-2"])
16991 );
16992 assert_eq!(
16993 batch.message_stream_cursors,
16994 vec![json!({"stream_name": "orders", "through_position": 2})]
16995 );
16996 assert!(batch.message_stream_waits.is_empty());
16997
16998 let partial = worker
16999 .execute_workflow_task_decision(workflow_task(
17000 "rust.message-stream-batch",
17001 vec![opened(1), received(first.clone()), applied(1, first)],
17002 DEFAULT_CODEC,
17003 ))
17004 .expect("worker task returns without waiting for a missing second item");
17005 assert_eq!(partial.commands.len(), 1);
17006 assert_eq!(partial.commands[0]["type"], "complete_workflow");
17007 assert_eq!(
17008 decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
17009 .expect("workflow result"),
17010 json!(["message-1"])
17011 );
17012 assert_eq!(
17013 partial.message_stream_cursors,
17014 vec![json!({"stream_name": "orders", "through_position": 1})]
17015 );
17016 assert!(partial.message_stream_waits.is_empty());
17017 }
17018
17019 #[test]
17020 fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
17021 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
17022 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
17023 value.to_string(),
17024 )]))
17025 .expect("message payload");
17026 json!({
17027 "schema": MESSAGE_STREAM_SCHEMA,
17028 "stream_name": "orders",
17029 "message_id": message_id,
17030 "position": position,
17031 "payload_envelope": payload,
17032 })
17033 }
17034
17035 fn opened(sequence: u64) -> HistoryEvent {
17036 history_event(
17037 "SignalWaitOpened",
17038 json!({
17039 "sequence": sequence,
17040 "signal_name": MESSAGE_STREAM_SIGNAL,
17041 }),
17042 )
17043 }
17044
17045 fn received(delivery: Value) -> HistoryEvent {
17046 history_event(
17047 "SignalReceived",
17048 json!({
17049 "signal_name": MESSAGE_STREAM_SIGNAL,
17050 "arguments": fixture_envelope(json!([delivery])),
17051 "payload_codec": DEFAULT_CODEC,
17052 }),
17053 )
17054 }
17055
17056 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
17057 history_event(
17058 "SignalApplied",
17059 json!({
17060 "sequence": sequence,
17061 "signal_name": MESSAGE_STREAM_SIGNAL,
17062 "value": fixture_envelope(json!([delivery])),
17063 }),
17064 )
17065 }
17066
17067 let client = Client::new("http://127.0.0.1:8080").expect("client");
17068 let mut worker = Worker::new(client, "rust-workers");
17069 worker.register_workflow(
17070 "rust.message-stream-partial-batches",
17071 |ctx, _input| async move {
17072 let stream = ctx.message_stream("orders")?;
17073 let first = stream.receive(10).await?;
17074 let second = stream.receive(10).await?;
17075 Ok(json!([
17076 first
17077 .into_iter()
17078 .map(|message| message.message_id)
17079 .collect::<Vec<_>>(),
17080 second
17081 .into_iter()
17082 .map(|message| message.message_id)
17083 .collect::<Vec<_>>(),
17084 ]))
17085 },
17086 );
17087
17088 let first = delivery("message-1", 1, "one");
17089 let second = delivery("message-2", 2, "two");
17090 let decision = worker
17091 .execute_workflow_task_decision(workflow_task(
17092 "rust.message-stream-partial-batches",
17093 vec![
17094 opened(1),
17095 received(first.clone()),
17096 applied(1, first),
17097 opened(2),
17098 received(second.clone()),
17099 applied(2, second),
17100 ],
17101 DEFAULT_CODEC,
17102 ))
17103 .expect("cold replay preserves both authored receive boundaries");
17104
17105 assert_eq!(decision.commands.len(), 1);
17106 assert_eq!(decision.commands[0]["type"], "complete_workflow");
17107 assert_eq!(
17108 decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
17109 .expect("workflow result"),
17110 json!([["message-1"], ["message-2"]])
17111 );
17112 assert_eq!(
17113 decision.message_stream_cursors,
17114 vec![json!({"stream_name": "orders", "through_position": 2})]
17115 );
17116 assert!(decision.message_stream_waits.is_empty());
17117 }
17118
17119 #[test]
17120 fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
17121 let ctx = workflow_context(Vec::new());
17122 let stream = ctx.message_stream("orders").expect("message stream");
17123 let mut receive = Box::pin(stream.receive(10));
17124 let mut task_context = TaskContext::from_waker(noop_waker_ref());
17125
17126 assert!(matches!(
17127 receive.as_mut().poll(&mut task_context),
17128 Poll::Pending
17129 ));
17130 assert_eq!(
17131 ctx.take_commands().expect("message-stream wait command"),
17132 vec![json!({
17133 "type": "open_signal_wait",
17134 "signal_name": MESSAGE_STREAM_SIGNAL,
17135 })]
17136 );
17137 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
17138 assert!(cursors.is_empty());
17139 assert_eq!(
17140 waits,
17141 vec![json!({"stream_name": "orders", "after_position": 0})]
17142 );
17143 }
17144
17145 #[test]
17146 fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
17147 let ctx = workflow_context(vec![history_event(
17148 "SignalReceived",
17149 json!({
17150 "signal_name": MESSAGE_STREAM_SIGNAL,
17151 "arguments": fixture_envelope(json!([{
17152 "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
17153 "stream_name": "orders",
17154 "through_position": 2,
17155 }])),
17156 "payload_codec": DEFAULT_CODEC,
17157 }),
17158 )]);
17159 let stream = ctx.message_stream("orders").expect("message stream");
17160 let mut receive = Box::pin(stream.receive(10));
17161 let mut task_context = TaskContext::from_waker(noop_waker_ref());
17162
17163 assert!(matches!(
17164 receive.as_mut().poll(&mut task_context),
17165 Poll::Pending
17166 ));
17167 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
17168 assert_eq!(
17169 cursors,
17170 vec![json!({"stream_name": "orders", "through_position": 2})]
17171 );
17172 assert_eq!(
17173 waits,
17174 vec![json!({"stream_name": "orders", "after_position": 2})]
17175 );
17176 }
17177
17178 #[test]
17179 fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
17180 let mut empty_map = BTreeMap::new();
17181 let mut nested = BTreeMap::new();
17182 nested.insert(
17183 "value".to_string(),
17184 AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
17185 );
17186 let values = vec![
17187 AvroValue::Bytes(vec![0, 255]),
17188 AvroValue::Long(1),
17189 AvroValue::Double(1.0),
17190 AvroValue::Array(Vec::new()),
17191 AvroValue::Map(std::mem::take(&mut empty_map)),
17192 AvroValue::Map(nested),
17193 ];
17194 let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
17195 let transport = vec![json!({
17196 "schema": MESSAGE_STREAM_SCHEMA,
17197 "stream_name": "orders",
17198 "message_id": "message-1",
17199 "position": 1,
17200 "payload_envelope": payload,
17201 })];
17202
17203 for _ in 0..2 {
17204 let Some(MessageStreamDelivery::Message(message)) =
17205 decode_message_stream_delivery(transport.clone()).expect("delivery")
17206 else {
17207 panic!("message delivery expected");
17208 };
17209 assert_eq!(message.arguments, values);
17210 assert!(matches!(message.arguments[1], AvroValue::Long(1)));
17211 assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
17212 }
17213 }
17214
17215 #[test]
17216 fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
17217 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
17218 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
17219 value.to_string(),
17220 )]))
17221 .expect("message payload");
17222 json!({
17223 "schema": MESSAGE_STREAM_SCHEMA,
17224 "stream_name": "orders",
17225 "message_id": message_id,
17226 "position": position,
17227 "payload_envelope": payload,
17228 })
17229 }
17230
17231 fn opened(sequence: u64) -> HistoryEvent {
17232 history_event(
17233 "SignalWaitOpened",
17234 json!({
17235 "sequence": sequence,
17236 "signal_name": MESSAGE_STREAM_SIGNAL,
17237 }),
17238 )
17239 }
17240
17241 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
17242 history_event(
17243 "SignalApplied",
17244 json!({
17245 "sequence": sequence,
17246 "signal_name": MESSAGE_STREAM_SIGNAL,
17247 "value": fixture_envelope(json!([delivery])),
17248 }),
17249 )
17250 }
17251
17252 fn worker() -> Worker {
17253 let client = Client::new("http://127.0.0.1:8080").expect("client");
17254 let mut worker = Worker::new(client, "rust-workers");
17255 worker.register_workflow("rust.message-stream", |ctx, _input| async move {
17256 let stream = ctx.message_stream("orders")?;
17257 let first = stream.receive_one().await?;
17258 let second = stream.receive_one().await?;
17259 Ok(json!([first.message_id, second.message_id]))
17260 });
17261 worker
17262 }
17263
17264 fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
17265 let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
17266 task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
17267 task.signal_arguments = Some(fixture_envelope(json!([delivery])));
17268 task
17269 }
17270
17271 let waiting = worker()
17272 .execute_workflow_task_decision(workflow_task(
17273 "rust.message-stream",
17274 Vec::new(),
17275 DEFAULT_CODEC,
17276 ))
17277 .expect("first worker opens the stream wait");
17278 assert_eq!(
17279 waiting.commands,
17280 vec![json!({
17281 "type": "open_signal_wait",
17282 "signal_name": MESSAGE_STREAM_SIGNAL,
17283 })]
17284 );
17285 assert!(waiting.message_stream_cursors.is_empty());
17286 assert_eq!(
17287 waiting.message_stream_waits,
17288 vec![json!({"stream_name": "orders", "after_position": 0})]
17289 );
17290
17291 let first_delivery = delivery("message-1", 1, "one");
17292 let first_arrival = worker()
17293 .execute_workflow_task_decision(task_with_resume(
17294 vec![opened(1)],
17295 first_delivery.clone(),
17296 ))
17297 .expect("replacement worker consumes the first arrival");
17298 assert_eq!(
17299 first_arrival.commands,
17300 vec![json!({
17301 "type": "open_signal_wait",
17302 "signal_name": MESSAGE_STREAM_SIGNAL,
17303 })]
17304 );
17305 assert_eq!(
17306 first_arrival.message_stream_cursors,
17307 vec![json!({"stream_name": "orders", "through_position": 1})]
17308 );
17309 assert_eq!(
17310 first_arrival.message_stream_waits,
17311 vec![json!({"stream_name": "orders", "after_position": 1})]
17312 );
17313
17314 let second_delivery = delivery("message-2", 2, "two");
17315 let first_applied = applied(1, first_delivery);
17316 let completed = worker()
17317 .execute_workflow_task_decision(task_with_resume(
17318 vec![opened(1), first_applied.clone(), opened(2)],
17319 second_delivery.clone(),
17320 ))
17321 .expect("next replacement worker consumes the second arrival");
17322 assert_eq!(completed.commands.len(), 1);
17323 assert_eq!(completed.commands[0]["type"], "complete_workflow");
17324 assert_eq!(
17325 decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
17326 .expect("workflow result"),
17327 json!(["message-1", "message-2"])
17328 );
17329 assert_eq!(
17330 completed.message_stream_cursors,
17331 vec![json!({"stream_name": "orders", "through_position": 2})]
17332 );
17333 assert!(completed.message_stream_waits.is_empty());
17334
17335 let replay_history = vec![
17336 opened(1),
17337 first_applied,
17338 opened(2),
17339 applied(2, second_delivery),
17340 ];
17341 for _cold_worker_or_restart in 0..2 {
17342 let replayed = worker()
17343 .execute_workflow_task_decision(workflow_task(
17344 "rust.message-stream",
17345 replay_history.clone(),
17346 DEFAULT_CODEC,
17347 ))
17348 .expect("cold worker replays each logical message exactly once");
17349 assert_eq!(replayed.commands.len(), 1);
17350 assert_eq!(
17351 decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
17352 .expect("replayed workflow result"),
17353 json!(["message-1", "message-2"])
17354 );
17355 assert_eq!(
17356 replayed.message_stream_cursors,
17357 vec![json!({"stream_name": "orders", "through_position": 2})]
17358 );
17359 assert!(replayed.message_stream_waits.is_empty());
17360 }
17361 }
17362
17363 #[test]
17364 fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
17365 assert!(!worker_protocol_supports_message_streams("1.14"));
17366 assert!(worker_protocol_supports_message_streams("1.15"));
17367 assert!(worker_protocol_supports_message_streams("1.16"));
17368 assert!(worker_protocol_supports_message_streams(
17369 WORKER_PROTOCOL_VERSION
17370 ));
17371 assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
17372 }
17373
17374 #[test]
17375 fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
17376 let ctx = workflow_context(vec![
17377 history_event(
17378 "ConditionWaitOpened",
17379 json!({
17380 "sequence": 1,
17381 "condition_wait_id": "condition:1",
17382 "condition_wait_occurrence_id": "rust:condition-wait:0",
17383 "condition_key": "signal:finish",
17384 "condition_definition_fingerprint": "sha256:signal-finish-v1",
17385 }),
17386 ),
17387 history_event(
17388 "ConditionWaitSatisfied",
17389 json!({
17390 "sequence": 1,
17391 "condition_wait_id": "condition:1",
17392 "condition_wait_occurrence_id": "rust:condition-wait:0",
17393 "condition_key": "signal:finish",
17394 "condition_definition_fingerprint": "sha256:signal-finish-v1",
17395 }),
17396 ),
17397 history_event(
17398 "SignalReceived",
17399 json!({"signal_name": "finish", "arguments": []}),
17400 ),
17401 ]);
17402 let mut signal = Box::pin(ctx.wait_signal("finish"));
17403 let mut task_context = TaskContext::from_waker(noop_waker_ref());
17404
17405 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
17406 signal.as_mut().poll(&mut task_context)
17407 else {
17408 panic!("condition history must not resolve as a typed signal wait");
17409 };
17410 assert_eq!(failure.reason, "recorded_command_mismatch");
17411 assert_eq!(failure.expected.as_deref(), Some("condition wait"));
17412 }
17413
17414 #[test]
17415 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
17416 let signal_then_timer = vec![
17417 history_event(
17418 "SignalWaitOpened",
17419 json!({"sequence": 1, "signal_name": "go"}),
17420 ),
17421 history_event(
17422 "SignalApplied",
17423 json!({
17424 "sequence": 1,
17425 "signal_name": "go",
17426 "value": fixture_envelope(json!(["now"])),
17427 }),
17428 ),
17429 history_event(
17430 "TimerScheduled",
17431 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
17432 ),
17433 history_event(
17434 "TimerFired",
17435 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
17436 ),
17437 ];
17438
17439 let ctx = workflow_context(signal_then_timer.clone());
17440 let mut signal = Box::pin(ctx.wait_signal("go"));
17441 let mut task_context = TaskContext::from_waker(noop_waker_ref());
17442 assert!(matches!(
17443 signal.as_mut().poll(&mut task_context),
17444 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
17445 ));
17446 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
17447 assert!(matches!(
17448 timer.as_mut().poll(&mut task_context),
17449 Poll::Ready(Ok(()))
17450 ));
17451 ctx.ensure_history_consumed()
17452 .expect("signal and timer history consumed in order");
17453
17454 let reordered = workflow_context(signal_then_timer);
17455 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
17456 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
17457 timer_first.as_mut().poll(&mut task_context)
17458 else {
17459 panic!("timer cannot consume signal-wait-first history");
17460 };
17461 assert_eq!(failure.reason, "recorded_command_mismatch");
17462 assert_eq!(failure.sequence, Some(1));
17463 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
17464
17465 let timer_then_signal = vec![
17466 history_event(
17467 "TimerScheduled",
17468 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
17469 ),
17470 history_event(
17471 "TimerFired",
17472 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
17473 ),
17474 history_event(
17475 "SignalWaitOpened",
17476 json!({"sequence": 2, "signal_name": "go"}),
17477 ),
17478 history_event(
17479 "SignalApplied",
17480 json!({
17481 "sequence": 2,
17482 "signal_name": "go",
17483 "value": fixture_envelope(json!([])),
17484 }),
17485 ),
17486 ];
17487 let reordered = workflow_context(timer_then_signal);
17488 let mut signal_first = Box::pin(reordered.wait_signal("go"));
17489 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
17490 signal_first.as_mut().poll(&mut task_context)
17491 else {
17492 panic!("signal wait cannot consume timer-first history");
17493 };
17494 assert_eq!(failure.reason, "recorded_command_mismatch");
17495 assert_eq!(failure.sequence, Some(1));
17496 assert_eq!(failure.expected.as_deref(), Some("timer"));
17497 }
17498
17499 #[test]
17500 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
17501 let duplicate_timer = WorkflowState::new(
17502 vec![
17503 history_event(
17504 "TimerScheduled",
17505 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
17506 ),
17507 history_event(
17508 "TimerScheduled",
17509 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
17510 ),
17511 ],
17512 "rust-workers".to_string(),
17513 DEFAULT_CODEC.to_string(),
17514 None,
17515 )
17516 .expect_err("one workflow sequence cannot schedule two timers");
17517 assert!(matches!(
17518 duplicate_timer,
17519 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17520 if reason == "timer_schedule_missing_or_duplicate"
17521 ));
17522
17523 let colliding_kinds = WorkflowState::new(
17524 vec![
17525 history_event(
17526 "TimerScheduled",
17527 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
17528 ),
17529 history_event(
17530 "ActivityCompleted",
17531 json!({"sequence": 1, "activity_type": "same-sequence"}),
17532 ),
17533 ],
17534 "rust-workers".to_string(),
17535 DEFAULT_CODEC.to_string(),
17536 None,
17537 )
17538 .expect_err("one workflow sequence cannot identify two command kinds");
17539 assert!(matches!(
17540 colliding_kinds,
17541 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17542 if reason == "durable_command_sequence_collision"
17543 ));
17544
17545 let duplicate_signal_wait = WorkflowState::new(
17546 vec![
17547 history_event(
17548 "SignalWaitOpened",
17549 json!({"sequence": 1, "signal_name": "go"}),
17550 ),
17551 history_event(
17552 "SignalWaitOpened",
17553 json!({"sequence": 1, "signal_name": "go"}),
17554 ),
17555 ],
17556 "rust-workers".to_string(),
17557 DEFAULT_CODEC.to_string(),
17558 None,
17559 )
17560 .expect_err("one workflow sequence cannot open two signal waits");
17561 assert!(matches!(
17562 duplicate_signal_wait,
17563 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17564 if reason == "signal_wait_open_missing_or_duplicate"
17565 ));
17566 }
17567
17568 #[test]
17569 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
17570 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
17571 .expect("side-effect result");
17572 let ctx = workflow_context(vec![history_event(
17573 "SideEffectRecorded",
17574 json!({"sequence": 99, "result": result}),
17575 )]);
17576
17577 let replayed: Value = ctx
17578 .side_effect(|| panic!("recorded side effect must not run"))
17579 .expect("positive global workflow sequence is valid");
17580 assert_eq!(replayed, json!({"captured": true}));
17581 ctx.ensure_history_consumed().expect("history consumed");
17582 }
17583
17584 #[test]
17585 fn workflow_history_rejects_zero_and_descending_command_sequences() {
17586 let result =
17587 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
17588 let zero = WorkflowState::new(
17589 vec![history_event(
17590 "SideEffectRecorded",
17591 json!({"sequence": 0, "result": result.clone()}),
17592 )],
17593 "rust-workers".to_string(),
17594 DEFAULT_CODEC.to_string(),
17595 None,
17596 )
17597 .expect_err("durable command sequences must be positive");
17598 assert!(matches!(
17599 zero,
17600 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17601 if reason == "durable_command_sequence_invalid"
17602 ));
17603
17604 let descending = WorkflowState::new(
17605 vec![
17606 history_event(
17607 "SideEffectRecorded",
17608 json!({"sequence": 3, "result": result}),
17609 ),
17610 history_event(
17611 "VersionMarkerRecorded",
17612 json!({
17613 "sequence": 2,
17614 "change_id": "descending-marker",
17615 "version": 1,
17616 "min_supported": 1,
17617 "max_supported": 1,
17618 }),
17619 ),
17620 ],
17621 "rust-workers".to_string(),
17622 DEFAULT_CODEC.to_string(),
17623 None,
17624 )
17625 .expect_err("new durable commands must remain strictly ordered");
17626 let Error::NonDeterministicReplay(failure) = descending else {
17627 panic!("expected typed replay failure");
17628 };
17629 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
17630 assert_eq!(failure.sequence, Some(2));
17631 assert_eq!(
17632 failure.expected.as_deref(),
17633 Some("workflow sequence greater than 3")
17634 );
17635 assert_eq!(failure.actual.as_deref(), Some("2"));
17636 }
17637
17638 #[test]
17639 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
17640 fn worker() -> Worker {
17641 let client = Client::new("http://127.0.0.1:8080").expect("client");
17642 let mut worker = Worker::new(client, "rust-workers");
17643 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
17644 ctx.wait_signal("finish").await?;
17645 let marker: String =
17646 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
17647 assert_eq!(marker, "after-finish");
17648 Ok(json!("finished"))
17649 });
17650 worker
17651 }
17652
17653 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
17654 .expect("side-effect result");
17655 let task = workflow_task(
17656 "rust.finish-after-gaps",
17657 vec![
17658 history_event(
17659 "SignalWaitOpened",
17660 json!({"sequence": 1, "signal_name": "finish"}),
17661 ),
17662 history_event(
17663 "SignalReceived",
17664 json!({
17665 "signal_id": "increment-3",
17666 "signal_name": "increment",
17667 "workflow_sequence": 2,
17668 "payload_codec": DEFAULT_CODEC,
17669 "arguments": fixture_envelope(json!([3])),
17670 }),
17671 ),
17672 history_event(
17673 "SignalReceived",
17674 json!({
17675 "signal_id": "increment-5",
17676 "signal_name": "increment",
17677 "workflow_sequence": 3,
17678 "payload_codec": DEFAULT_CODEC,
17679 "arguments": fixture_envelope(json!([5])),
17680 }),
17681 ),
17682 history_event(
17683 "SignalReceived",
17684 json!({
17685 "signal_id": "finish",
17686 "signal_name": "finish",
17687 "workflow_sequence": 4,
17688 "payload_codec": DEFAULT_CODEC,
17689 "arguments": fixture_envelope(json!([])),
17690 }),
17691 ),
17692 history_event(
17693 "SignalApplied",
17694 json!({
17695 "sequence": 1,
17696 "signal_id": "finish",
17697 "signal_name": "finish",
17698 "payload_codec": DEFAULT_CODEC,
17699 "value": fixture_envelope(json!([])),
17700 }),
17701 ),
17702 history_event(
17703 "SideEffectRecorded",
17704 json!({"sequence": 5, "result": marker}),
17705 ),
17706 ],
17707 DEFAULT_CODEC,
17708 );
17709
17710 for _original_or_cold_worker in 0..2 {
17711 let commands = worker()
17712 .execute_workflow_task(task.clone())
17713 .expect("signal gaps preserve deterministic replay");
17714 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
17715 assert_eq!(commands[0]["type"], "complete_workflow");
17716 assert_eq!(
17717 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
17718 json!("finished")
17719 );
17720 }
17721 }
17722
17723 #[test]
17724 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
17725 let ctx = workflow_context(Vec::new());
17726 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
17727 let mut task_context = TaskContext::from_waker(noop_waker_ref());
17728 assert!(matches!(
17729 sleep.as_mut().poll(&mut task_context),
17730 Poll::Ready(Err(Error::TimerDurationOverflow))
17731 ));
17732 assert!(ctx.take_commands().expect("commands").is_empty());
17733 }
17734
17735 #[test]
17736 fn workflow_memo_update_emits_canonical_command_and_replays_once() {
17737 let entries = AvroValue::Map(BTreeMap::from([
17738 ("text".to_string(), AvroValue::String("same".to_string())),
17739 (
17740 "nested".to_string(),
17741 AvroValue::Map(BTreeMap::from([
17742 ("beta".to_string(), AvroValue::Long(2)),
17743 ("alpha".to_string(), AvroValue::Long(1)),
17744 ])),
17745 ),
17746 ("long".to_string(), AvroValue::Long(7)),
17747 ("double".to_string(), AvroValue::Double(7.0)),
17748 ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
17749 ]));
17750 let ctx = workflow_context(Vec::new());
17751 ctx.upsert_memo(entries.clone()).expect("valid memo update");
17752 let commands = ctx.take_commands().expect("commands");
17753
17754 assert_eq!(commands.len(), 1);
17755 assert_eq!(commands[0]["type"], "upsert_memo");
17756 let server_entries = json!({
17757 "codec": "avro",
17758 "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
17759 });
17760 assert_eq!(
17761 commands[0]["entries"]
17762 .as_object()
17763 .expect("entries envelope")
17764 .keys()
17765 .collect::<Vec<_>>(),
17766 vec!["blob", "codec"]
17767 );
17768 assert_eq!(commands[0]["entries"], server_entries);
17769 let wire_entries =
17770 decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
17771 assert_eq!(wire_entries, entries);
17772
17773 let history = vec![history_event(
17774 "MemoUpserted",
17775 json!({
17776 "sequence": 1,
17777 "entries": server_entries.clone(),
17778 "merged": server_entries,
17779 }),
17780 )];
17781 let replay = workflow_context(history.clone());
17782 replay
17783 .upsert_memo(entries.clone())
17784 .expect("matching replay identity");
17785 assert!(replay.take_commands().expect("replay commands").is_empty());
17786
17787 let changed_types = AvroValue::Map(BTreeMap::from([
17788 ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
17789 (
17790 "nested".to_string(),
17791 AvroValue::Map(BTreeMap::from([
17792 ("alpha".to_string(), AvroValue::Long(1)),
17793 ("beta".to_string(), AvroValue::Long(2)),
17794 ])),
17795 ),
17796 ("long".to_string(), AvroValue::Double(7.0)),
17797 ("double".to_string(), AvroValue::Long(7)),
17798 ("binary".to_string(), AvroValue::String("same".to_string())),
17799 ]));
17800 let error = workflow_context(history)
17801 .upsert_memo(changed_types)
17802 .expect_err("memo replay identity must preserve Avro value types");
17803 assert!(matches!(
17804 error,
17805 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
17806 ));
17807 }
17808
17809 #[test]
17810 fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
17811 let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
17812 .expect("memo envelope");
17813 let replay = workflow_context(vec![history_event(
17814 "MemoUpserted",
17815 json!({
17816 "sequence": 1,
17817 "entries": original.clone(),
17818 "merged": original
17819 }),
17820 )]);
17821 let error = replay
17822 .upsert_memo(json!({"stage": "changed"}))
17823 .expect_err("changed memo update must fail replay");
17824 assert!(matches!(
17825 error,
17826 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
17827 ));
17828
17829 let invalid = workflow_context(Vec::new())
17830 .upsert_memo(
17831 json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
17832 )
17833 .expect_err("oversized key");
17834 assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
17835 }
17836
17837 #[test]
17838 fn workflow_memo_replay_distinguishes_signed_zero_identity() {
17839 let negative_zero = AvroValue::Map(BTreeMap::from([(
17840 "reading".to_string(),
17841 AvroValue::Double(-0.0),
17842 )]));
17843 let negative_zero_envelope =
17844 encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
17845 let history = vec![history_event(
17846 "MemoUpserted",
17847 json!({
17848 "sequence": 1,
17849 "entries": negative_zero_envelope.clone(),
17850 "merged": negative_zero_envelope,
17851 }),
17852 )];
17853
17854 workflow_context(history.clone())
17855 .upsert_memo(negative_zero)
17856 .expect("matching negative-zero history identity");
17857
17858 let error = workflow_context(history)
17859 .upsert_memo(AvroValue::Map(BTreeMap::from([(
17860 "reading".to_string(),
17861 AvroValue::Double(0.0),
17862 )])))
17863 .expect_err("positive zero must not consume negative-zero memo history");
17864 assert!(matches!(
17865 error,
17866 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
17867 ));
17868 }
17869
17870 #[test]
17871 fn workflow_memo_capability_requires_flag_and_command_advertisement() {
17872 let supported = json!({
17873 "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
17874 "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
17875 });
17876 assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
17877 assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
17878 "workflow_memo_updates": {"supported": false},
17879 "supported_workflow_task_commands": ["upsert_memo"]
17880 }))));
17881 assert!(commands_use_workflow_memo_updates(&[json!({
17882 "type": "upsert_memo",
17883 "entries": {"stage": "processing"}
17884 })]));
17885 }
17886
17887 #[test]
17888 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
17889 let client = Client::new("http://127.0.0.1:8080").expect("client");
17890 let mut worker = Worker::new(client, "rust-workers");
17891 worker.register_workflow("rust.timer", |ctx, _input| async move {
17892 ctx.sleep(Duration::from_secs(5)).await?;
17893 ctx.activity("after-timer", json!([])).await
17894 });
17895
17896 let task = |history_events| WorkflowTask {
17897 task_id: "wft-rust-timer-1".to_string(),
17898 workflow_command_id: None,
17899 workflow_id: Some("wf-rust-timer".to_string()),
17900 run_id: Some("run-rust-timer".to_string()),
17901 workflow_type: "rust.timer".to_string(),
17902 cancel_requested: false,
17903 payload_codec: DEFAULT_CODEC.to_string(),
17904 arguments: Some(
17905 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
17906 ),
17907 history_events,
17908 total_history_events: None,
17909 history_size_bytes: None,
17910 continue_as_new_recommended: None,
17911 history_budget_pressure: None,
17912 next_history_page_token: None,
17913 workflow_task_attempt: 1,
17914 workflow_signal_id: None,
17915 signal_name: None,
17916 signal_arguments: None,
17917 workflow_update_id: None,
17918 update_name: None,
17919 lease_owner: Some("rust-worker".to_string()),
17920 };
17921
17922 let initial = worker
17923 .execute_workflow_task(task(Vec::new()))
17924 .expect("initial timer task");
17925 assert_eq!(
17926 initial,
17927 vec![json!({"type": "start_timer", "delay_seconds": 5})]
17928 );
17929
17930 let activity_result =
17931 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
17932 let replayed = worker
17933 .execute_workflow_task(task(vec![
17934 history_event(
17935 "TimerScheduled",
17936 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
17937 ),
17938 history_event(
17939 "TimerFired",
17940 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
17941 ),
17942 history_event(
17943 "ActivityCompleted",
17944 json!({
17945 "sequence": 2,
17946 "activity_type": "after-timer",
17947 "payload_codec": DEFAULT_CODEC,
17948 "result": activity_result,
17949 }),
17950 ),
17951 ]))
17952 .expect("replayed workflow task");
17953 assert_eq!(replayed.len(), 1);
17954 assert_eq!(replayed[0]["type"], "complete_workflow");
17955 assert_eq!(
17956 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
17957 json!("done")
17958 );
17959 }
17960
17961 #[test]
17962 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
17963 let client = Client::new("http://127.0.0.1:8080").expect("client");
17964 let mut worker = Worker::new(client, "rust-workers");
17965 worker.register_workflow("rust.continue", |ctx, _input| async move {
17966 ctx.continue_as_new_with_options(
17967 ContinueAsNewOptions::new()
17968 .workflow_type("rust.next")
17969 .task_queue("next-workers"),
17970 json!([2, {"cursor": "next"}]),
17971 )
17972 });
17973
17974 let commands = worker
17975 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
17976 .expect("continue-as-new command");
17977
17978 assert_eq!(commands.len(), 1);
17979 assert_eq!(commands[0]["type"], "continue_as_new");
17980 assert_eq!(commands[0]["workflow_type"], "rust.next");
17981 assert_eq!(commands[0]["queue"], "next-workers");
17982 assert_eq!(
17983 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
17984 .expect("continue-as-new arguments"),
17985 json!([2, {"cursor": "next"}])
17986 );
17987 }
17988
17989 #[test]
17990 fn continue_as_new_preserves_typed_arguments() {
17991 let client = Client::new("http://127.0.0.1:8080").expect("client");
17992 let mut worker = Worker::new(client, "rust-workers");
17993 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
17994 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
17995 unreachable!("continue-as-new returns a control-flow error")
17996 });
17997
17998 let commands = worker
17999 .execute_workflow_task(workflow_task(
18000 "rust.typed-continue",
18001 Vec::new(),
18002 DEFAULT_CODEC,
18003 ))
18004 .expect("typed continue-as-new command");
18005
18006 assert_eq!(commands[0]["type"], "continue_as_new");
18007 assert_eq!(
18008 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
18009 .expect("typed continue arguments"),
18010 AvroValue::Array(vec![typed_fidelity_probe()])
18011 );
18012 }
18013
18014 #[test]
18015 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
18016 let client = Client::new("http://127.0.0.1:8080").expect("client");
18017 let mut worker = Worker::new(client, "rust-workers");
18018 worker.register_workflow("rust.continue", |ctx, _input| async move {
18019 ctx.continue_as_new(json!([2]))
18020 });
18021 let task = workflow_task(
18022 "rust.continue",
18023 vec![history_event(
18024 "WorkflowContinuedAsNew",
18025 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
18026 )],
18027 DEFAULT_CODEC,
18028 );
18029
18030 for _worker_restart_or_redelivery in 0..2 {
18031 let commands = worker
18032 .execute_workflow_task(task.clone())
18033 .expect("recorded transition replays");
18034 assert!(
18035 commands.is_empty(),
18036 "replay must not emit another successor"
18037 );
18038 }
18039 }
18040
18041 #[test]
18042 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
18043 let ctx = workflow_context(Vec::new());
18044 let error = ctx
18045 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
18046 .expect_err("blank queue must be rejected");
18047
18048 let Error::InvalidContinueAsNewOptions(error) = error else {
18049 panic!("expected typed continue-as-new validation error");
18050 };
18051 assert_eq!(error.field, "task_queue");
18052 assert!(ctx.take_commands().expect("commands").is_empty());
18053 }
18054
18055 #[test]
18056 fn workflow_context_exposes_server_history_budget() {
18057 let client = Client::new("http://127.0.0.1:8080").expect("client");
18058 let mut worker = Worker::new(client, "rust-workers");
18059 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
18060 let budget = ctx.history_budget()?;
18061 Ok(json!({
18062 "events": budget.event_count,
18063 "bytes": budget.size_bytes,
18064 "recommended": budget.continue_as_new_recommended,
18065 "pressure": budget.pressure,
18066 }))
18067 });
18068 let task: WorkflowTask = serde_json::from_value(json!({
18069 "task_id": "task-history-budget",
18070 "workflow_type": "rust.history-budget",
18071 "payload_codec": DEFAULT_CODEC,
18072 "history_events": [],
18073 "total_history_events": 480,
18074 "history_size_bytes": 1_048_576,
18075 "continue_as_new_recommended": true,
18076 "history_budget_pressure": "continue_as_new_recommended",
18077 }))
18078 .expect("published workflow task");
18079
18080 let commands = worker
18081 .execute_workflow_task(task)
18082 .expect("history-budget workflow");
18083 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
18084 assert_eq!(result["events"], 480);
18085 assert_eq!(result["bytes"], 1_048_576);
18086 assert_eq!(result["recommended"], true);
18087 assert_eq!(result["pressure"], "continue_as_new_recommended");
18088 }
18089
18090 #[test]
18091 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
18092 let client = Client::new("http://127.0.0.1:8080").expect("client");
18093 let mut worker = Worker::new(client, "rust-workers");
18094 worker.register_workflow("rust.failing", |_ctx, _input| async move {
18095 Err(Error::Codec("rust_conformance_failure".to_string()))
18096 });
18097 let task = WorkflowTask {
18098 task_id: "wft-rust-failing-1".to_string(),
18099 workflow_command_id: None,
18100 workflow_id: Some("wf-rust-failing".to_string()),
18101 run_id: Some("run-rust-failing".to_string()),
18102 workflow_type: "rust.failing".to_string(),
18103 cancel_requested: false,
18104 payload_codec: DEFAULT_CODEC.to_string(),
18105 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
18106 history_events: Vec::new(),
18107 total_history_events: Some(0),
18108 history_size_bytes: None,
18109 continue_as_new_recommended: None,
18110 history_budget_pressure: None,
18111 next_history_page_token: None,
18112 workflow_task_attempt: 1,
18113 workflow_signal_id: None,
18114 signal_name: None,
18115 signal_arguments: None,
18116 workflow_update_id: None,
18117 update_name: None,
18118 lease_owner: Some("rust-worker".to_string()),
18119 };
18120
18121 let commands = worker
18122 .execute_workflow_task(task)
18123 .expect("handler failure becomes a workflow command");
18124
18125 assert_eq!(commands.len(), 1);
18126 assert_eq!(commands[0]["type"], "fail_workflow");
18127 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
18128 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
18129 assert_eq!(commands[0]["non_retryable"], false);
18130 assert_eq!(
18131 commands[0]["message"],
18132 "codec error: rust_conformance_failure"
18133 );
18134 assert_eq!(
18135 commands[0]["exception"]["message"],
18136 "codec error: rust_conformance_failure"
18137 );
18138 }
18139
18140 #[test]
18141 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
18142 let client = Client::new("http://127.0.0.1:8080").expect("client");
18143 let mut worker = Worker::new(client, "rust-workers");
18144 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
18145 let _: String = ctx.side_effect(|| "captured".to_string())?;
18146 Err(Error::WorkerLoop("application failure".to_string()))
18147 });
18148
18149 let commands = worker
18150 .execute_workflow_task(workflow_task(
18151 "rust.failing-after-side-effect",
18152 Vec::new(),
18153 DEFAULT_CODEC,
18154 ))
18155 .expect("ordinary failure remains a workflow decision");
18156
18157 assert_eq!(commands.len(), 2);
18158 assert_eq!(commands[0]["type"], "record_side_effect");
18159 assert_eq!(commands[1]["type"], "fail_workflow");
18160 }
18161
18162 #[test]
18163 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
18164 let client = Client::new("http://127.0.0.1:8080").expect("client");
18165 let mut worker = Worker::new(client, "rust-workers");
18166 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
18167 Err(Error::WorkerLoop("application failure".to_string()))
18168 });
18169 let result =
18170 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
18171
18172 let error = worker
18173 .execute_workflow_task(workflow_task(
18174 "rust.removed-side-effect",
18175 vec![history_event(
18176 "SideEffectRecorded",
18177 json!({"sequence": 1, "result": result}),
18178 )],
18179 DEFAULT_CODEC,
18180 ))
18181 .expect_err("removed committed history must not become fail_workflow");
18182
18183 let Error::NonDeterministicReplay(failure) = error else {
18184 panic!("expected typed replay failure");
18185 };
18186 assert_eq!(failure.reason, "recorded_commands_unconsumed");
18187 assert_eq!(failure.sequence, Some(1));
18188 assert_eq!(failure.expected.as_deref(), Some("side effect"));
18189 }
18190
18191 #[test]
18192 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
18193 let client = Client::new("http://127.0.0.1:8080").expect("client");
18194 let mut worker = Worker::new(client, "rust-workers");
18195 worker.register_workflow(
18196 "rust.side-effect-before-marker-error",
18197 |ctx, _input| async move {
18198 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
18199 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
18200 ctx.get_version("restart-safe", 2, 2)?;
18201 Ok(Value::Null)
18202 },
18203 );
18204
18205 let error = worker
18206 .execute_workflow_task(workflow_task(
18207 "rust.side-effect-before-marker-error",
18208 vec![history_event(
18209 "VersionMarkerRecorded",
18210 json!({
18211 "sequence": 1,
18212 "change_id": "restart-safe",
18213 "version": 1,
18214 "min_supported": 1,
18215 "max_supported": 1,
18216 }),
18217 )],
18218 DEFAULT_CODEC,
18219 ))
18220 .expect_err("replay error must return no queued workflow commands");
18221
18222 let Error::NonDeterministicReplay(failure) = error else {
18223 panic!("expected typed replay failure");
18224 };
18225 assert_eq!(failure.reason, "version_marker_incompatible_range");
18226 assert_eq!(failure.sequence, Some(1));
18227 }
18228
18229 #[test]
18230 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
18231 let client = Client::new("http://127.0.0.1:8080").expect("client");
18232 let mut worker = Worker::new(client, "rust-workers");
18233 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
18234 ctx.sleep(Duration::from_secs(5)).await?;
18235 Ok(json!({"status": "timer fired"}))
18236 });
18237
18238 let task = WorkflowTask {
18239 task_id: "wft-rust-timer-pending".to_string(),
18240 workflow_command_id: None,
18241 workflow_id: Some("wf-rust-timer".to_string()),
18242 run_id: Some("run-rust-timer".to_string()),
18243 workflow_type: "rust.timer.pending".to_string(),
18244 cancel_requested: false,
18245 payload_codec: DEFAULT_CODEC.to_string(),
18246 arguments: Some(
18247 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
18248 ),
18249 history_events: vec![history_event(
18250 "TimerScheduled",
18251 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18252 )],
18253 total_history_events: Some(1),
18254 history_size_bytes: None,
18255 continue_as_new_recommended: None,
18256 history_budget_pressure: None,
18257 next_history_page_token: None,
18258 workflow_task_attempt: 1,
18259 workflow_signal_id: None,
18260 signal_name: None,
18261 signal_arguments: None,
18262 workflow_update_id: None,
18263 update_name: None,
18264 lease_owner: Some("rust-worker".to_string()),
18265 };
18266
18267 for _redelivery_or_restart in 0..2 {
18268 let commands = worker
18269 .execute_workflow_task(task.clone())
18270 .expect("recorded timer remains pending");
18271 assert!(
18272 commands.is_empty(),
18273 "recorded timer must not be rescheduled"
18274 );
18275 }
18276 }
18277
18278 #[test]
18279 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
18280 let client = Client::new("http://127.0.0.1:8080").expect("client");
18281 let mut worker = Worker::new(client, "rust-workers");
18282 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
18283 Ok(json!({"status": "completed"}))
18284 });
18285 let task = WorkflowTask {
18286 task_id: "wft-rust-timer-removed".to_string(),
18287 workflow_command_id: None,
18288 workflow_id: Some("wf-rust-timer".to_string()),
18289 run_id: Some("run-rust-timer".to_string()),
18290 workflow_type: "rust.timer.removed".to_string(),
18291 cancel_requested: false,
18292 payload_codec: DEFAULT_CODEC.to_string(),
18293 arguments: Some(
18294 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
18295 ),
18296 history_events: vec![
18297 history_event(
18298 "TimerScheduled",
18299 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18300 ),
18301 history_event(
18302 "TimerFired",
18303 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18304 ),
18305 ],
18306 total_history_events: Some(2),
18307 history_size_bytes: None,
18308 continue_as_new_recommended: None,
18309 history_budget_pressure: None,
18310 next_history_page_token: None,
18311 workflow_task_attempt: 1,
18312 workflow_signal_id: None,
18313 signal_name: None,
18314 signal_arguments: None,
18315 workflow_update_id: None,
18316 update_name: None,
18317 lease_owner: Some("rust-worker".to_string()),
18318 };
18319
18320 let Error::NonDeterministicReplay(failure) = worker
18321 .execute_workflow_task(task)
18322 .expect_err("removed timer must fail replay")
18323 else {
18324 panic!("expected typed replay failure");
18325 };
18326 assert_eq!(failure.reason, "recorded_commands_unconsumed");
18327 assert_eq!(failure.sequence, Some(1));
18328 }
18329
18330 #[test]
18331 fn workflow_context_emits_explicit_child_workflow_contract() {
18332 let ctx = WorkflowContext {
18333 state: Arc::new(Mutex::new(
18334 WorkflowState::new_with_identity(
18335 Vec::new(),
18336 Some("wf-parent".to_string()),
18337 Some("run-parent".to_string()),
18338 "parent-workers".to_string(),
18339 DEFAULT_CODEC.to_string(),
18340 None,
18341 )
18342 .expect("workflow state"),
18343 )),
18344 };
18345 let options = ChildWorkflowOptions::new("python-workers")
18346 .parent_close_policy(ParentClosePolicy::RequestCancel)
18347 .retry_policy(ChildWorkflowRetryPolicy {
18348 max_attempts: Some(3),
18349 backoff_seconds: vec![1, 5],
18350 non_retryable_error_types: vec!["ValidationError".to_string()],
18351 })
18352 .execution_timeout_seconds(600)
18353 .run_timeout_seconds(120);
18354 let mut call = Box::pin(ctx.start_child_workflow(
18355 "python.fulfil-order",
18356 options,
18357 json!([{"order_id": "order-42"}]),
18358 ));
18359 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18360
18361 assert!(matches!(
18362 call.as_mut().poll(&mut task_context),
18363 Poll::Pending
18364 ));
18365 let commands = ctx.take_commands().expect("commands");
18366 assert_eq!(commands.len(), 1);
18367 let command = &commands[0];
18368 assert_eq!(command["type"], "start_child_workflow");
18369 assert_eq!(command["workflow_type"], "python.fulfil-order");
18370 assert_eq!(command["queue"], "python-workers");
18371 assert_eq!(command["parent_close_policy"], "request_cancel");
18372 assert_eq!(command["retry_policy"]["max_attempts"], 3);
18373 assert_eq!(command["execution_timeout_seconds"], 600);
18374 assert_eq!(command["run_timeout_seconds"], 120);
18375 assert_eq!(
18376 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
18377 json!([{"order_id": "order-42"}])
18378 );
18379 }
18380
18381 fn child_parent_worker() -> Worker {
18382 let client = Client::new("http://127.0.0.1:8080").expect("client");
18383 let mut worker = Worker::new(client, "rust-parent-workers");
18384 worker.register_workflow("rust.parent", |ctx, _input| async move {
18385 let child = ctx
18386 .start_child_workflow(
18387 "python.child",
18388 ChildWorkflowOptions::new("python-child-workers")
18389 .parent_close_policy(ParentClosePolicy::Terminate),
18390 json!([{"codec_probe": [1, true, "rust"]}]),
18391 )
18392 .await?;
18393 Ok(json!({
18394 "parent_workflow_id": child.parent.workflow_id,
18395 "parent_run_id": child.parent.run_id,
18396 "child_workflow_id": child.child.workflow_id,
18397 "child_run_id": child.child.run_id,
18398 "child_workflow_type": child.child_workflow_type,
18399 "result": child.result,
18400 }))
18401 });
18402 worker
18403 }
18404
18405 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
18406 WorkflowTask {
18407 task_id: "wft-child-parent".to_string(),
18408 workflow_command_id: None,
18409 workflow_id: Some("wf-parent".to_string()),
18410 run_id: Some("run-parent".to_string()),
18411 workflow_type: "rust.parent".to_string(),
18412 cancel_requested: false,
18413 payload_codec: DEFAULT_CODEC.to_string(),
18414 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
18415 history_events: vec![
18416 HistoryEvent {
18417 event_type: "ChildWorkflowScheduled".to_string(),
18418 payload: json!({
18419 "sequence": 1,
18420 "child_call_id": "call-child",
18421 "child_workflow_instance_id": "wf-child",
18422 "child_workflow_run_id": "run-child",
18423 "child_workflow_type": "python.child",
18424 }),
18425 raw: HashMap::new(),
18426 },
18427 HistoryEvent {
18428 event_type: event_type.to_string(),
18429 payload,
18430 raw: HashMap::new(),
18431 },
18432 ],
18433 total_history_events: Some(2),
18434 history_size_bytes: None,
18435 continue_as_new_recommended: None,
18436 history_budget_pressure: None,
18437 next_history_page_token: None,
18438 workflow_task_attempt: 1,
18439 workflow_signal_id: None,
18440 signal_name: None,
18441 signal_arguments: None,
18442 workflow_update_id: None,
18443 update_name: None,
18444 lease_owner: Some("rust-worker".to_string()),
18445 }
18446 }
18447
18448 #[test]
18449 fn committed_child_result_replays_without_starting_a_duplicate() {
18450 let worker = child_parent_worker();
18451 let task = child_parent_task(
18452 "ChildRunCompleted",
18453 json!({
18454 "sequence": 1,
18455 "child_call_id": "call-child",
18456 "child_workflow_instance_id": "wf-child",
18457 "child_workflow_run_id": "run-child",
18458 "child_workflow_type": "python.child",
18459 "payload_codec": DEFAULT_CODEC,
18460 "result": fixture_envelope(json!({"from":"python","ok":true})),
18461 }),
18462 );
18463
18464 for _restart in 0..2 {
18465 let commands = worker
18466 .execute_workflow_task(task.clone())
18467 .expect("replayed parent task");
18468 assert_eq!(commands.len(), 1);
18469 assert_eq!(commands[0]["type"], "complete_workflow");
18470 assert!(!commands
18471 .iter()
18472 .any(|command| command["type"] == "start_child_workflow"));
18473 let output =
18474 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
18475 assert_eq!(output["parent_workflow_id"], "wf-parent");
18476 assert_eq!(output["parent_run_id"], "run-parent");
18477 assert_eq!(output["child_workflow_id"], "wf-child");
18478 assert_eq!(output["child_run_id"], "run-child");
18479 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
18480 }
18481 }
18482
18483 #[test]
18484 fn typed_child_arguments_and_results_survive_replay() {
18485 let client = Client::new("http://127.0.0.1:8080").expect("client");
18486 let mut worker = Worker::new(client, "rust-parent-workers");
18487 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
18488 let child = ctx
18489 .start_child_workflow_avro_value(
18490 "python.typed-child",
18491 ChildWorkflowOptions::new("python-workers"),
18492 AvroValue::Array(vec![typed_fidelity_probe()]),
18493 )
18494 .await?;
18495 Ok(child.result)
18496 });
18497
18498 let initial = worker
18499 .execute_workflow_task(workflow_task(
18500 "rust.typed-parent",
18501 Vec::new(),
18502 DEFAULT_CODEC,
18503 ))
18504 .expect("typed child start");
18505 assert_eq!(initial[0]["type"], "start_child_workflow");
18506 assert_eq!(
18507 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
18508 .expect("typed child arguments"),
18509 AvroValue::Array(vec![typed_fidelity_probe()])
18510 );
18511
18512 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
18513 .expect("typed child result");
18514 let task = workflow_task(
18515 "rust.typed-parent",
18516 vec![
18517 history_event(
18518 "ChildWorkflowScheduled",
18519 json!({
18520 "sequence": 1,
18521 "child_call_id": "call-typed",
18522 "child_workflow_instance_id": "wf-child",
18523 "child_workflow_run_id": "run-child",
18524 "child_workflow_type": "python.typed-child",
18525 }),
18526 ),
18527 history_event(
18528 "ChildRunCompleted",
18529 json!({
18530 "sequence": 1,
18531 "child_call_id": "call-typed",
18532 "child_workflow_instance_id": "wf-child",
18533 "child_workflow_run_id": "run-child",
18534 "child_workflow_type": "python.typed-child",
18535 "payload_codec": DEFAULT_CODEC,
18536 "result": result,
18537 }),
18538 ),
18539 ],
18540 DEFAULT_CODEC,
18541 );
18542
18543 let commands = worker
18544 .execute_workflow_task(task)
18545 .expect("typed child replay");
18546 assert_eq!(commands[0]["type"], "complete_workflow");
18547 assert_eq!(
18548 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
18549 .expect("typed parent result"),
18550 typed_fidelity_probe()
18551 );
18552 }
18553
18554 #[test]
18555 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
18556 let worker = child_parent_worker();
18557 let mut task = child_parent_task("unused", Value::Null);
18558 task.history_events.truncate(1);
18559 task.total_history_events = Some(1);
18560
18561 for _redelivery_or_restart in 0..2 {
18562 let commands = worker
18563 .execute_workflow_task(task.clone())
18564 .expect("recorded child remains pending");
18565 assert!(
18566 commands.is_empty(),
18567 "recorded pending child must not be started again"
18568 );
18569 }
18570 }
18571
18572 #[test]
18573 fn child_cancellation_becomes_stable_parent_failure_command() {
18574 let worker = child_parent_worker();
18575 let task = child_parent_task(
18576 "ChildRunCancelled",
18577 json!({
18578 "sequence": 1,
18579 "child_workflow_instance_id": "wf-child",
18580 "child_workflow_run_id": "run-child",
18581 "child_workflow_type": "python.child",
18582 "failure_id": "failure-child",
18583 "failure_category": "cancelled",
18584 "message": "cancelled by parent-close policy",
18585 }),
18586 );
18587
18588 let commands = worker
18589 .execute_workflow_task(task)
18590 .expect("parent settlement");
18591 assert_eq!(commands.len(), 1);
18592 assert_eq!(commands[0]["type"], "fail_workflow");
18593 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
18594 assert_eq!(
18595 commands[0]["exception"]["properties"]["reason"],
18596 "cancelled"
18597 );
18598 assert_eq!(
18599 commands[0]["exception"]["properties"]["child_workflow_run_id"],
18600 "run-child"
18601 );
18602 }
18603
18604 #[test]
18605 fn workflow_can_handle_typed_child_failure() {
18606 let client = Client::new("http://127.0.0.1:8080").expect("client");
18607 let mut worker = Worker::new(client, "rust-parent-workers");
18608 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
18609 match ctx
18610 .start_child_workflow(
18611 "python.child",
18612 ChildWorkflowOptions::new("python-child-workers"),
18613 json!([]),
18614 )
18615 .await
18616 {
18617 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
18618 "reason": failure.reason,
18619 "failure_id": failure.failure_id,
18620 "exception_class": failure.exception_class,
18621 "child_run_id": failure.child_workflow_run_id,
18622 })),
18623 Err(error) => Err(error),
18624 Ok(_) => Err(Error::WorkerLoop(
18625 "child unexpectedly succeeded".to_string(),
18626 )),
18627 }
18628 });
18629 let mut task = child_parent_task(
18630 "ChildRunFailed",
18631 json!({
18632 "sequence": 1,
18633 "child_workflow_instance_id": "wf-child",
18634 "child_workflow_run_id": "run-child",
18635 "child_workflow_type": "python.child",
18636 "failure_id": "failure-child",
18637 "failure_category": "child_workflow",
18638 "message": "payment rejected",
18639 "exception": {
18640 "type": "PaymentRejected",
18641 "class": "payments.PaymentRejected",
18642 "message": "payment rejected"
18643 }
18644 }),
18645 );
18646 task.workflow_type = "rust.handled-parent".to_string();
18647
18648 let commands = worker.execute_workflow_task(task).expect("handled failure");
18649 assert_eq!(commands[0]["type"], "complete_workflow");
18650 let output =
18651 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
18652 assert_eq!(output["reason"], "child_workflow");
18653 assert_eq!(output["failure_id"], "failure-child");
18654 assert_eq!(output["exception_class"], "payments.PaymentRejected");
18655 assert_eq!(output["child_run_id"], "run-child");
18656 }
18657
18658 #[test]
18659 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
18660 let client = Client::new("http://127.0.0.1:8080").expect("client");
18661 let mut worker = Worker::new(client, "rust-workers");
18662
18663 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
18664 let signal = ctx.wait_signal("start").await?;
18665 let name = signal
18666 .first()
18667 .and_then(|value| value.as_str())
18668 .unwrap_or("world");
18669 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
18670 Ok(json!({
18671 "greeting": greeting,
18672 "language": "rust"
18673 }))
18674 });
18675
18676 let signal_arguments =
18677 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
18678 let task = WorkflowTask {
18679 task_id: "wft-rust-signal-1".to_string(),
18680 workflow_command_id: None,
18681 workflow_id: Some("wf-rust-hello".to_string()),
18682 run_id: Some("run-rust-hello".to_string()),
18683 workflow_type: "rust.hello_workflow".to_string(),
18684 cancel_requested: false,
18685 payload_codec: DEFAULT_CODEC.to_string(),
18686 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
18687 history_events: vec![HistoryEvent {
18688 event_type: "SignalReceived".to_string(),
18689 payload: json!({
18690 "signal_id": "sig-rust-1",
18691 "signal_name": "start"
18692 }),
18693 raw: HashMap::new(),
18694 }],
18695 total_history_events: Some(1),
18696 history_size_bytes: None,
18697 continue_as_new_recommended: None,
18698 history_budget_pressure: None,
18699 next_history_page_token: None,
18700 workflow_task_attempt: 1,
18701 workflow_signal_id: Some("sig-rust-1".to_string()),
18702 signal_name: Some("start".to_string()),
18703 signal_arguments: Some(signal_arguments),
18704 workflow_update_id: None,
18705 update_name: None,
18706 lease_owner: Some("rust-worker".to_string()),
18707 };
18708
18709 let commands = worker.execute_workflow_task(task).expect("workflow task");
18710
18711 assert_eq!(commands.len(), 1);
18712 assert_eq!(commands[0]["type"], "schedule_activity");
18713 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
18714 assert_eq!(
18715 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
18716 json!(["Rust"])
18717 );
18718 }
18719
18720 #[test]
18721 fn workflow_task_appends_paginated_history_events() {
18722 let mut task = WorkflowTask {
18723 task_id: "wft-rust-pages-1".to_string(),
18724 workflow_command_id: None,
18725 workflow_id: Some("wf-rust-pages".to_string()),
18726 run_id: Some("run-rust-pages".to_string()),
18727 workflow_type: "rust.hello_workflow".to_string(),
18728 cancel_requested: false,
18729 payload_codec: DEFAULT_CODEC.to_string(),
18730 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
18731 history_events: vec![HistoryEvent {
18732 event_type: "WorkflowStarted".to_string(),
18733 payload: json!({}),
18734 raw: HashMap::new(),
18735 }],
18736 total_history_events: Some(3),
18737 history_size_bytes: None,
18738 continue_as_new_recommended: None,
18739 history_budget_pressure: None,
18740 next_history_page_token: Some("MQ==".to_string()),
18741 workflow_task_attempt: 1,
18742 workflow_signal_id: None,
18743 signal_name: None,
18744 signal_arguments: None,
18745 workflow_update_id: None,
18746 update_name: None,
18747 lease_owner: Some("rust-worker".to_string()),
18748 };
18749
18750 task.append_history_page(WorkflowTaskHistoryPage {
18751 history_events: vec![
18752 HistoryEvent {
18753 event_type: "SignalReceived".to_string(),
18754 payload: json!({
18755 "signal_id": "sig-rust-1",
18756 "signal_name": "start",
18757 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
18758 .expect("signal arguments")
18759 }),
18760 raw: HashMap::new(),
18761 },
18762 HistoryEvent {
18763 event_type: "MarkerRecorded".to_string(),
18764 payload: json!({"sequence": 3}),
18765 raw: HashMap::new(),
18766 },
18767 ],
18768 total_history_events: Some(3),
18769 next_history_page_token: None,
18770 });
18771
18772 assert_eq!(task.history_events.len(), 3);
18773 assert_eq!(task.total_history_events, Some(3));
18774 assert_eq!(task.next_history_page_token, None);
18775
18776 let signal = task
18777 .history_events
18778 .iter()
18779 .find(|event| event.event_type == "SignalReceived")
18780 .expect("signal event");
18781 assert_eq!(
18782 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
18783 vec![AvroValue::String("Rust".to_string())]
18784 );
18785 }
18786
18787 #[tokio::test]
18788 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
18789 let client = Client::new("http://127.0.0.1:8080").expect("client");
18790 let mut worker = Worker::new(client, "rust-workers");
18791 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
18792 worker.register_query("counter", "current", |ctx, _args| async move {
18793 let mut count = 0_i64;
18794 for signal in ctx.signal_events() {
18795 let value = signal
18796 .arguments
18797 .first()
18798 .and_then(Value::as_i64)
18799 .unwrap_or_default();
18800 match signal.name.as_str() {
18801 "increment" => count += value,
18802 "set" => count = value,
18803 _ => {}
18804 }
18805 }
18806 Ok(json!(count))
18807 });
18808
18809 let task = QueryTask {
18810 query_task_id: "query-rust-counter".to_string(),
18811 query_task_attempt: 1,
18812 lease_owner: Some("rust-worker".to_string()),
18813 workflow_id: Some("counter-1".to_string()),
18814 run_id: Some("run-counter-1".to_string()),
18815 workflow_type: "counter".to_string(),
18816 query_name: "current".to_string(),
18817 payload_codec: DEFAULT_CODEC.to_string(),
18818 workflow_arguments: Some(
18819 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
18820 ),
18821 query_arguments: Some(
18822 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
18823 ),
18824 history_events: vec![
18825 HistoryEvent {
18826 event_type: "SignalReceived".to_string(),
18827 payload: json!({
18828 "signal_id": "php-signal-1",
18829 "signal_name": "increment",
18830 "workflow_sequence": 1,
18831 "payload_codec": DEFAULT_CODEC,
18832 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
18833 }),
18834 raw: HashMap::new(),
18835 },
18836 HistoryEvent {
18837 event_type: "SignalReceived".to_string(),
18838 payload: json!({
18839 "signal_id": "python-signal-2",
18840 "signal_name": "increment",
18841 "workflow_sequence": 2,
18842 "payload_codec": DEFAULT_CODEC,
18843 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
18844 }),
18845 raw: HashMap::new(),
18846 },
18847 HistoryEvent {
18848 event_type: "SignalReceived".to_string(),
18849 payload: json!({
18850 "signal_id": "rust-signal-3",
18851 "signal_name": "set",
18852 "workflow_sequence": 3,
18853 "payload_codec": DEFAULT_CODEC,
18854 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
18855 }),
18856 raw: HashMap::new(),
18857 },
18858 ],
18859 history_export: None,
18860 run_status: Some("completed".to_string()),
18861 };
18862
18863 let result = worker.execute_query_task(task).await.expect("query result");
18864 assert_eq!(result.into_json().expect("query projection"), json!(0));
18865 }
18866
18867 #[tokio::test]
18868 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
18869 let worker = replay_counter_worker();
18870 let running_history = json!([
18871 {
18872 "type": "ActivityCompleted",
18873 "payload": {
18874 "sequence": 1,
18875 "activity_type": "load-counter",
18876 "payload_codec": DEFAULT_CODEC,
18877 "result": fixture_envelope(json!("loaded"))
18878 }
18879 },
18880 {
18881 "type": "SignalWaitOpened",
18882 "payload": {
18883 "sequence": 3,
18884 "signal_name": "increment"
18885 }
18886 },
18887 {
18888 "type": "SignalReceived",
18889 "payload": {
18890 "signal_id": "signal-3",
18891 "signal_name": "increment",
18892 "workflow_sequence": 2,
18893 "payload_codec": DEFAULT_CODEC,
18894 "arguments": fixture_envelope(json!([3]))
18895 }
18896 },
18897 {
18898 "type": "SignalApplied",
18899 "payload": {
18900 "sequence": 3,
18901 "signal_id": "signal-3",
18902 "signal_name": "increment",
18903 "payload_codec": DEFAULT_CODEC,
18904 "value": fixture_envelope(json!([3]))
18905 }
18906 }
18907 ]);
18908
18909 let running = worker
18910 .execute_query_task(replay_counter_query(
18911 "current",
18912 running_history.clone(),
18913 "running",
18914 ))
18915 .await
18916 .expect("running replay query");
18917 assert_eq!(
18918 running.clone().into_json().expect("query projection"),
18919 json!({"loaded": "loaded", "count": 3, "finished": false})
18920 );
18921
18922 let detached = worker
18923 .execute_query_task(replay_counter_query(
18924 "detached-mutation",
18925 running_history.clone(),
18926 "running",
18927 ))
18928 .await
18929 .expect("query mutates only its detached state clone");
18930 assert_eq!(detached.into_json().expect("query projection"), json!(999));
18931 let failed = worker
18932 .execute_query_task(replay_counter_query(
18933 "failed-mutation",
18934 running_history.clone(),
18935 "running",
18936 ))
18937 .await
18938 .expect_err("failed query");
18939 assert_eq!(failed.reason, "query_rejected");
18940 let unchanged = worker
18941 .execute_query_task(replay_counter_query("current", running_history, "running"))
18942 .await
18943 .expect("later query reconstructs unchanged state");
18944 assert_eq!(unchanged, running);
18945
18946 let restarted_worker = replay_counter_worker();
18947 let empty_arguments = fixture_envelope(json!([]));
18948 let loaded_result = fixture_envelope(json!("loaded"));
18949 let signal_three = fixture_blob(json!([3]));
18950 let signal_five = fixture_blob(json!([5]));
18951 let restarted_task: QueryTask = serde_json::from_value(json!({
18952 "query_task_id": "query-after-restart",
18953 "workflow_id": "counter-1",
18954 "run_id": "run-counter-1",
18955 "workflow_type": "replay-counter",
18956 "query_name": "current",
18957 "payload_codec": DEFAULT_CODEC,
18958 "workflow_arguments": empty_arguments.clone(),
18959 "query_arguments": empty_arguments,
18960 "history_events": [],
18961 "history_export": {
18962 "payloads": {"codec": DEFAULT_CODEC},
18963 "history_events": [
18964 {
18965 "type": "ActivityCompleted",
18966 "payload": {
18967 "sequence": 1,
18968 "activity_type": "load-counter",
18969 "payload_codec": DEFAULT_CODEC,
18970 "result": null
18971 }
18972 },
18973 {
18974 "type": "SignalWaitOpened",
18975 "payload": {
18976 "sequence": 3,
18977 "signal_name": "increment"
18978 }
18979 },
18980 {
18981 "type": "SignalReceived",
18982 "payload": {
18983 "signal_id": "signal-3",
18984 "signal_name": "increment",
18985 "workflow_sequence": 2
18986 }
18987 },
18988 {
18989 "type": "SignalApplied",
18990 "payload": {
18991 "sequence": 3,
18992 "signal_id": "signal-3",
18993 "signal_name": "increment"
18994 }
18995 },
18996 {
18997 "type": "SignalWaitOpened",
18998 "payload": {
18999 "sequence": 5,
19000 "signal_name": "increment"
19001 }
19002 },
19003 {
19004 "type": "SignalReceived",
19005 "payload": {
19006 "signal_id": "signal-5",
19007 "signal_name": "increment",
19008 "workflow_sequence": 4
19009 }
19010 },
19011 {
19012 "type": "SignalApplied",
19013 "payload": {
19014 "sequence": 5,
19015 "signal_id": "signal-5",
19016 "signal_name": "increment"
19017 }
19018 }
19019 ],
19020 "activities": [{
19021 "sequence": 1,
19022 "activity_type": "load-counter",
19023 "payload_codec": DEFAULT_CODEC,
19024 "result": loaded_result
19025 }],
19026 "signals": [
19027 {
19028 "id": "signal-3",
19029 "name": "increment",
19030 "workflow_sequence": 2,
19031 "payload_codec": DEFAULT_CODEC,
19032 "arguments": signal_three
19033 },
19034 {
19035 "id": "signal-5",
19036 "name": "increment",
19037 "workflow_sequence": 4,
19038 "payload_codec": DEFAULT_CODEC,
19039 "arguments": signal_five
19040 }
19041 ]
19042 },
19043 "run_status": "completed"
19044 }))
19045 .expect("cold replay query task");
19046 let completed = restarted_worker
19047 .execute_query_task(restarted_task)
19048 .await
19049 .expect("completed cold replay query");
19050 assert_eq!(
19051 completed.into_json().expect("query projection"),
19052 json!({"loaded": "loaded", "count": 8, "finished": true})
19053 );
19054 }
19055
19056 #[tokio::test]
19057 async fn replayed_query_replay_failures_are_machine_readable() {
19058 let worker = replay_counter_worker();
19059 let task = replay_counter_query(
19060 "current",
19061 json!([{
19062 "type": "ActivityCompleted",
19063 "payload": {
19064 "sequence": 1,
19065 "payload_codec": DEFAULT_CODEC,
19066 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
19067 }
19068 }]),
19069 "running",
19070 );
19071 let failure = worker
19072 .execute_query_task(task)
19073 .await
19074 .expect_err("invalid replay history payload");
19075 assert_eq!(failure.reason, "query_payload_decode_failed");
19076 assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
19077 assert!(failure.message.contains("invalid_payload_framing"));
19078 }
19079
19080 #[tokio::test]
19081 async fn query_task_restores_compact_history_from_export() {
19082 let client = Client::new("http://127.0.0.1:8080").expect("client");
19083 let mut worker = Worker::new(client, "rust-workers");
19084 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19085 worker.register_query("counter", "current", |ctx, _args| async move {
19086 Ok(json!(ctx.signals("increment")[0][0]))
19087 });
19088 let empty_arguments = fixture_envelope(json!([]));
19089 let exported_signal = fixture_blob(json!([9]));
19090 let task: QueryTask = serde_json::from_value(json!({
19091 "query_task_id": "query-export",
19092 "workflow_type": "counter",
19093 "query_name": "current",
19094 "payload_codec": DEFAULT_CODEC,
19095 "workflow_arguments": empty_arguments.clone(),
19096 "query_arguments": empty_arguments,
19097 "history_events": [],
19098 "history_export": {
19099 "payloads": {"codec": DEFAULT_CODEC},
19100 "history_events": [{
19101 "type": "SignalReceived",
19102 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
19103 }],
19104 "signals": [{
19105 "id": "signal-export",
19106 "name": "increment",
19107 "status": "applied",
19108 "workflow_sequence": 1,
19109 "payload_codec": DEFAULT_CODEC,
19110 "arguments": exported_signal
19111 }]
19112 }
19113 }))
19114 .expect("query task");
19115
19116 let result = worker.execute_query_task(task).await.expect("query result");
19117 assert_eq!(result.into_json().expect("query projection"), json!(9));
19118 }
19119
19120 #[tokio::test]
19121 async fn query_task_failures_have_stable_reasons() {
19122 let client = Client::new("http://127.0.0.1:8080").expect("client");
19123 let mut worker = Worker::new(client, "rust-workers");
19124 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19125 worker.register_query(
19126 "counter",
19127 "current",
19128 |_ctx, _args| async move { Ok(json!(0)) },
19129 );
19130
19131 let base_task = QueryTask {
19132 query_task_id: "query-errors".to_string(),
19133 query_task_attempt: 1,
19134 lease_owner: None,
19135 workflow_id: Some("counter-errors".to_string()),
19136 run_id: Some("run-errors".to_string()),
19137 workflow_type: "counter".to_string(),
19138 query_name: "missing".to_string(),
19139 payload_codec: DEFAULT_CODEC.to_string(),
19140 workflow_arguments: Some(fixture_envelope(json!([]))),
19141 query_arguments: Some(fixture_envelope(json!([]))),
19142 history_events: Vec::new(),
19143 history_export: None,
19144 run_status: Some("running".to_string()),
19145 };
19146
19147 let unknown = worker
19148 .execute_query_task(base_task.clone())
19149 .await
19150 .expect_err("unknown query");
19151 assert_eq!(unknown.reason, "rejected_unknown_query");
19152
19153 let mut malformed = base_task;
19154 malformed.query_name = "current".to_string();
19155 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
19156 let malformed = worker
19157 .execute_query_task(malformed)
19158 .await
19159 .expect_err("malformed payload");
19160 assert_eq!(malformed.reason, "query_payload_decode_failed");
19161
19162 let client = Client::new("http://127.0.0.1:8080").expect("client");
19163 let mut unavailable_worker = Worker::new(client, "rust-workers");
19164 unavailable_worker
19165 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19166 let empty_arguments = fixture_envelope(json!([]));
19167 let unavailable_task: QueryTask = serde_json::from_value(json!({
19168 "query_task_id": "query-unavailable",
19169 "workflow_type": "counter",
19170 "query_name": "current",
19171 "payload_codec": DEFAULT_CODEC,
19172 "workflow_arguments": empty_arguments.clone(),
19173 "query_arguments": empty_arguments
19174 }))
19175 .expect("query task");
19176 let unavailable = unavailable_worker
19177 .execute_query_task(unavailable_task)
19178 .await
19179 .expect_err("query handler unavailable");
19180 assert_eq!(unavailable.reason, "query_handler_unavailable");
19181 }
19182
19183 #[tokio::test]
19184 async fn client_query_decodes_result_and_typed_failure() {
19185 let server = MockWorkerServer::start();
19186 let client = Client::builder(server.base_url())
19187 .timeout(Duration::from_secs(2))
19188 .build()
19189 .expect("client");
19190
19191 let result = client
19192 .query_workflow("counter-1", "current", json!([]))
19193 .await
19194 .expect("query result");
19195 assert_eq!(result, json!({"count": 8}));
19196
19197 let error = client
19198 .query_workflow("counter-1", "missing", json!([]))
19199 .await
19200 .expect_err("unknown query");
19201 let Error::QueryFailed(failure) = error else {
19202 panic!("expected typed query failure");
19203 };
19204 assert_eq!(failure.status, 404);
19205 assert_eq!(failure.reason, "rejected_unknown_query");
19206 }
19207
19208 #[tokio::test]
19209 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
19210 let server = MockWorkerServer::start();
19211 let client = Client::builder(server.base_url())
19212 .timeout(Duration::from_secs(2))
19213 .build()
19214 .expect("client");
19215 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
19216
19217 client
19218 .start_workflow(
19219 "typed.echo",
19220 "rust-workers",
19221 "typed-start",
19222 arguments.clone(),
19223 )
19224 .await
19225 .expect("typed workflow start");
19226 assert_eq!(
19227 decode_wire_avro_value(
19228 &server.request_body("/api/workflows")["input"],
19229 DEFAULT_CODEC,
19230 )
19231 .expect("typed start input"),
19232 arguments
19233 );
19234
19235 client
19236 .signal_workflow("typed-1", "changed", arguments.clone())
19237 .await
19238 .expect("typed signal");
19239 assert_eq!(
19240 decode_wire_avro_value(
19241 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
19242 DEFAULT_CODEC,
19243 )
19244 .expect("typed signal input"),
19245 arguments
19246 );
19247
19248 assert_eq!(
19249 client
19250 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
19251 .await
19252 .expect("typed query"),
19253 typed_fidelity_probe()
19254 );
19255 assert_eq!(
19256 decode_wire_avro_value(
19257 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
19258 DEFAULT_CODEC,
19259 )
19260 .expect("typed query input"),
19261 arguments
19262 );
19263
19264 assert_eq!(
19265 client
19266 .update_workflow_avro_value(
19267 "typed-1",
19268 "replace",
19269 arguments.clone(),
19270 Some("typed-request"),
19271 )
19272 .await
19273 .expect("typed update"),
19274 typed_fidelity_probe()
19275 );
19276 let update = server.request_body("/api/workflows/typed-1/update/replace");
19277 assert_eq!(update["request_id"], "typed-request");
19278 assert_eq!(
19279 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
19280 arguments
19281 );
19282
19283 let handle = WorkflowHandle {
19284 client: client.clone(),
19285 workflow_id: "typed-1".to_string(),
19286 run_id: Some("run-typed-1".to_string()),
19287 workflow_type: "typed.echo".to_string(),
19288 };
19289 assert_eq!(
19290 handle
19291 .result_avro_value(WorkflowResultOptions::default())
19292 .await
19293 .expect("typed workflow result"),
19294 typed_fidelity_probe()
19295 );
19296
19297 client
19298 .complete_activity_task(
19299 "activity-typed",
19300 "attempt-typed",
19301 "rust-worker",
19302 typed_fidelity_probe(),
19303 DEFAULT_CODEC,
19304 )
19305 .await
19306 .expect("typed activity completion");
19307 assert_eq!(
19308 decode_wire_avro_value(
19309 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
19310 ["result"],
19311 DEFAULT_CODEC,
19312 )
19313 .expect("typed activity result"),
19314 typed_fidelity_probe()
19315 );
19316 client
19317 .fail_activity_task(
19318 "activity-typed",
19319 "attempt-typed",
19320 "rust-worker",
19321 "typed failure",
19322 true,
19323 )
19324 .await
19325 .expect("activity failure");
19326 }
19327
19328 #[tokio::test]
19329 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
19330 let server = MockWorkerServer::start();
19331 let client = Client::builder(server.base_url())
19332 .timeout(Duration::from_secs(2))
19333 .build()
19334 .expect("client");
19335
19336 let options = WorkflowCommandOptions::new()
19337 .reason("cleanup requested")
19338 .request_id("cancel-17");
19339 let cancelled = client
19340 .cancel_workflow("wf-lifecycle", options)
19341 .await
19342 .expect("instance cancellation");
19343 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
19344 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
19345 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
19346 assert_eq!(
19347 server.request_body("/api/workflows/wf-lifecycle/cancel"),
19348 json!({"reason":"cleanup requested","request_id":"cancel-17"})
19349 );
19350
19351 let terminated = client
19352 .terminate_workflow(
19353 "wf-lifecycle",
19354 WorkflowCommandOptions::new().reason("forced stop"),
19355 )
19356 .await
19357 .expect("instance termination");
19358 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
19359 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
19360
19361 client
19362 .cancel_workflow_run(
19363 "wf-lifecycle",
19364 "run-current",
19365 WorkflowCommandOptions::default(),
19366 )
19367 .await
19368 .expect("selected run cancellation");
19369 client
19370 .terminate_workflow_run(
19371 "wf-lifecycle",
19372 "run-current",
19373 WorkflowCommandOptions::default(),
19374 )
19375 .await
19376 .expect("selected run termination");
19377
19378 for (command, error) in [
19379 (
19380 WorkflowCommandKind::Cancel,
19381 client
19382 .cancel_workflow_run(
19383 "wf-lifecycle",
19384 "run-stale",
19385 WorkflowCommandOptions::default(),
19386 )
19387 .await
19388 .expect_err("stale cancellation must be rejected"),
19389 ),
19390 (
19391 WorkflowCommandKind::Terminate,
19392 client
19393 .terminate_workflow_run(
19394 "wf-lifecycle",
19395 "run-stale",
19396 WorkflowCommandOptions::default(),
19397 )
19398 .await
19399 .expect_err("stale termination must be rejected"),
19400 ),
19401 ] {
19402 let Error::WorkflowCommandRejected(rejection) = error else {
19403 panic!("expected typed command rejection");
19404 };
19405 assert_eq!(rejection.command, command);
19406 assert_eq!(rejection.status, 409);
19407 assert_eq!(rejection.reason, "historical_run_command_rejected");
19408 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
19409 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
19410 }
19411 }
19412
19413 #[tokio::test]
19414 async fn workflow_start_options_send_server_enforced_deadlines() {
19415 let server = MockWorkerServer::start();
19416 let client = Client::builder(server.base_url())
19417 .timeout(Duration::from_secs(2))
19418 .build()
19419 .expect("client");
19420
19421 let handle = client
19422 .start_workflow_with_options(
19423 "rust.timeout",
19424 "rust-timeouts",
19425 "wf-start-options",
19426 WorkflowStartOptions::new()
19427 .execution_timeout_seconds(30)
19428 .run_timeout_seconds(1),
19429 json!([]),
19430 )
19431 .await
19432 .expect("workflow start");
19433
19434 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
19435 let body = server.request_body("/api/workflows");
19436 assert_eq!(body["execution_timeout_seconds"], 30);
19437 assert_eq!(body["run_timeout_seconds"], 1);
19438
19439 let invalid = client
19440 .start_workflow_with_options(
19441 "rust.timeout",
19442 "rust-timeouts",
19443 "wf-invalid-options",
19444 WorkflowStartOptions::new()
19445 .execution_timeout_seconds(1)
19446 .run_timeout_seconds(2),
19447 json!([]),
19448 )
19449 .await
19450 .expect_err("invalid deadline ordering");
19451 assert!(invalid
19452 .to_string()
19453 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
19454 }
19455
19456 #[tokio::test]
19457 async fn workflow_result_returns_each_typed_terminal_outcome() {
19458 let server = MockWorkerServer::start();
19459 let client = Client::builder(server.base_url())
19460 .timeout(Duration::from_secs(2))
19461 .build()
19462 .expect("client");
19463 let options = WorkflowResultOptions {
19464 poll_interval: Duration::ZERO,
19465 timeout: Duration::from_secs(1),
19466 };
19467
19468 let failed = WorkflowHandle {
19469 client: client.clone(),
19470 workflow_id: "wf-failed".to_string(),
19471 run_id: Some("run-failed".to_string()),
19472 workflow_type: "failure".to_string(),
19473 }
19474 .result(options)
19475 .await
19476 .expect_err("failed outcome");
19477 let Error::WorkflowFailed(failure) = failed else {
19478 panic!("expected WorkflowFailed");
19479 };
19480 assert_eq!(failure.workflow_id, "wf-failed");
19481 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
19482 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
19483 assert_eq!(failure.failure_category.as_deref(), Some("application"));
19484 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
19485 assert_eq!(
19486 failure.exception_class.as_deref(),
19487 Some("billing::PaymentError")
19488 );
19489 assert_eq!(failure.non_retryable, Some(true));
19490
19491 for (workflow_id, expected_kind, expected_reason) in [
19492 (
19493 "wf-cancelled",
19494 WorkflowTerminalKind::Cancelled,
19495 "cleanup requested",
19496 ),
19497 (
19498 "wf-terminated",
19499 WorkflowTerminalKind::Terminated,
19500 "forced stop",
19501 ),
19502 (
19503 "wf-timed-out",
19504 WorkflowTerminalKind::TimedOut,
19505 "run_timeout",
19506 ),
19507 ] {
19508 let error = WorkflowHandle {
19509 client: client.clone(),
19510 workflow_id: workflow_id.to_string(),
19511 run_id: None,
19512 workflow_type: "terminal".to_string(),
19513 }
19514 .result(options)
19515 .await
19516 .expect_err("typed terminal outcome");
19517 let outcome = match error {
19518 Error::WorkflowCancelled(outcome) => outcome,
19519 Error::WorkflowTerminated(outcome) => outcome,
19520 Error::WorkflowTimedOut(outcome) => outcome,
19521 other => panic!("unexpected terminal error: {other}"),
19522 };
19523 assert_eq!(outcome.kind, expected_kind);
19524 assert_eq!(outcome.workflow_id, workflow_id);
19525 assert_eq!(outcome.reason, expected_reason);
19526 }
19527
19528 let wait_timeout = WorkflowHandle {
19529 client,
19530 workflow_id: "wf-waiting".to_string(),
19531 run_id: Some("run-waiting".to_string()),
19532 workflow_type: "waiting".to_string(),
19533 }
19534 .result(WorkflowResultOptions {
19535 poll_interval: Duration::ZERO,
19536 timeout: Duration::ZERO,
19537 })
19538 .await
19539 .expect_err("client wait timeout");
19540 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
19541 panic!("expected typed client timeout");
19542 };
19543 assert_eq!(timeout.reason, "result_wait_timeout");
19544 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
19545 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
19546 }
19547
19548 #[tokio::test]
19549 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
19550 let server = MockWorkerServer::start();
19551 let client = Client::builder(server.base_url())
19552 .timeout(Duration::from_secs(2))
19553 .build()
19554 .expect("client");
19555
19556 let handle = WorkflowHandle {
19557 client,
19558 workflow_id: "wf-selected".to_string(),
19559 run_id: Some("run-selected".to_string()),
19560 workflow_type: "selected".to_string(),
19561 };
19562 let options = WorkflowResultOptions {
19563 poll_interval: Duration::ZERO,
19564 timeout: Duration::from_secs(1),
19565 };
19566
19567 let current = handle
19568 .result(options)
19569 .await
19570 .expect("instance result follows the current run");
19571 assert_eq!(current, json!("current run output"));
19572
19573 let error = handle
19574 .result_selected_run(options)
19575 .await
19576 .expect_err("the selected run is cancelled even though the current run completed");
19577
19578 let Error::WorkflowCancelled(outcome) = error else {
19579 panic!("expected selected run cancellation");
19580 };
19581 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
19582 assert_eq!(outcome.reason, "selected run cancelled");
19583 assert_eq!(
19584 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
19585 1
19586 );
19587 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
19588 }
19589
19590 #[tokio::test]
19591 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
19592 let server = MockWorkerServer::draining_polls();
19593 let client = Client::builder(server.base_url())
19594 .timeout(Duration::from_secs(2))
19595 .build()
19596 .expect("client");
19597
19598 let workflow = client
19599 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
19600 .await
19601 .expect("workflow drain response");
19602 let activity = client
19603 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
19604 .await
19605 .expect("activity drain response");
19606 let query = client
19607 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
19608 .await
19609 .expect("query drain response");
19610
19611 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
19612 assert_eq!(
19613 outcome,
19614 WorkerPollOutcome::Stop {
19615 poll_status: Some("draining".to_string()),
19616 reason: Some("worker_draining".to_string()),
19617 }
19618 );
19619 }
19620
19621 assert!(client
19622 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
19623 .await
19624 .expect("compatibility poll")
19625 .is_none());
19626 }
19627
19628 #[tokio::test]
19629 async fn managed_worker_honors_drain_stop_for_every_task_family() {
19630 let server = MockWorkerServer::draining_polls();
19631 let client = Client::builder(server.base_url())
19632 .timeout(Duration::from_secs(2))
19633 .build()
19634 .expect("client");
19635
19636 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
19637 .worker_id("draining-workflow-worker")
19638 .poll_timeout(Duration::ZERO);
19639 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
19640 workflow_worker
19641 .run()
19642 .await
19643 .expect("workflow drain is a clean stop");
19644
19645 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
19646 .worker_id("draining-activity-worker")
19647 .poll_timeout(Duration::ZERO);
19648 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
19649 activity_worker
19650 .run()
19651 .await
19652 .expect("activity drain is a clean stop");
19653
19654 let mut query_worker = Worker::new(client, "rust-workers")
19655 .worker_id("draining-query-worker")
19656 .poll_timeout(Duration::ZERO);
19657 query_worker.register_query("counter", "current", |_ctx, _args| async {
19658 Ok(Value::Null)
19659 });
19660 query_worker
19661 .run()
19662 .await
19663 .expect("query drain is a clean stop");
19664 }
19665
19666 #[tokio::test]
19667 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
19668 let server = MockWorkerServer::start();
19669 let client = Client::builder(server.base_url())
19670 .timeout(Duration::from_secs(2))
19671 .build()
19672 .expect("client");
19673
19674 let heartbeat = client
19675 .heartbeat_activity_task(
19676 "activity-cancel",
19677 "attempt-cancel",
19678 "rust-worker",
19679 typed_fidelity_probe(),
19680 )
19681 .await
19682 .expect("cancellation heartbeat");
19683 assert!(heartbeat.cancel_requested);
19684 assert!(heartbeat.should_stop());
19685 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
19686 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
19687 let heartbeat_body =
19688 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
19689 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
19690 assert_eq!(
19691 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
19692 .expect("typed heartbeat details"),
19693 typed_fidelity_probe()
19694 );
19695
19696 let error = client
19697 .complete_activity_task(
19698 "activity-cancel",
19699 "attempt-cancel",
19700 "rust-worker",
19701 json!({"late":true}),
19702 DEFAULT_CODEC,
19703 )
19704 .await
19705 .expect_err("late completion must be refused");
19706 assert!(activity_task_rejection_is_final(&error));
19707 let Error::ActivityTaskRejected(rejection) = error else {
19708 panic!("expected typed activity rejection");
19709 };
19710 assert_eq!(rejection.status, 409);
19711 assert_eq!(rejection.reason, "run_cancelled");
19712 assert!(rejection.cancel_requested);
19713 assert_eq!(rejection.can_continue, Some(false));
19714 }
19715
19716 #[tokio::test]
19717 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
19718 let server = MockWorkerServer::cancelled_activity();
19719 let client = Client::builder(server.base_url())
19720 .timeout(Duration::from_secs(2))
19721 .build()
19722 .expect("client");
19723 let cancellation_observed = Arc::new(AtomicBool::new(false));
19724 let observed = Arc::clone(&cancellation_observed);
19725 let mut worker = Worker::new(client.clone(), "rust-workers")
19726 .worker_id("rust-cancel-worker")
19727 .poll_timeout(Duration::from_millis(10));
19728 worker.register_activity("cancel-aware", move |ctx, _args| {
19729 let observed = Arc::clone(&observed);
19730 async move {
19731 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
19732 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
19733 Ok(json!({"late":"completion"}))
19734 }
19735 });
19736
19737 assert_eq!(
19738 worker.run_once().await.expect("cancelled attempt handled"),
19739 1
19740 );
19741 assert!(cancellation_observed.load(Ordering::SeqCst));
19742 assert_eq!(
19743 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
19744 1
19745 );
19746
19747 let mut restarted = Worker::new(client, "rust-workers")
19748 .worker_id("rust-cancel-worker-restarted")
19749 .poll_timeout(Duration::from_millis(10));
19750 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
19751 assert_eq!(
19752 restarted
19753 .run_once()
19754 .await
19755 .expect("replacement worker continues polling"),
19756 0
19757 );
19758 }
19759
19760 #[tokio::test]
19761 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
19762 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
19763 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
19764 let client = Client::builder(server.base_url())
19765 .timeout(Duration::from_secs(2))
19766 .build()
19767 .expect("client");
19768
19769 let direct_error = client
19770 .complete_workflow_task(
19771 "workflow-timeout-task",
19772 "timeout-worker",
19773 3,
19774 vec![json!({
19775 "type": "complete_workflow",
19776 "result": fixture_envelope(Value::Null)
19777 })],
19778 )
19779 .await
19780 .expect_err("the low-level client preserves the completion rejection");
19781 let Error::Http { status, body } = direct_error else {
19782 panic!("expected the original HTTP completion rejection");
19783 };
19784 assert_eq!(status, reqwest::StatusCode::CONFLICT);
19785 assert_eq!(
19786 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
19787 "run_timed_out"
19788 );
19789
19790 let mut worker = Worker::new(client, "rust-workers")
19791 .worker_id("timeout-worker")
19792 .poll_timeout(Duration::from_millis(10));
19793 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
19794 Ok(json!({"late": "result"}))
19795 });
19796
19797 assert_eq!(
19798 worker
19799 .run_once()
19800 .await
19801 .expect("authoritative selected-run timeout settles the tick"),
19802 1
19803 );
19804 assert_eq!(
19805 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
19806 2,
19807 "both the direct client proof and managed worker must see the rejection"
19808 );
19809 }
19810
19811 #[tokio::test]
19812 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
19813 for (name, status, response) in [
19814 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
19815 (
19816 "command was recorded",
19817 "409 Conflict",
19818 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
19819 ),
19820 (
19821 "lease conflict",
19822 "409 Conflict",
19823 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
19824 ),
19825 (
19826 "nonterminal run",
19827 "409 Conflict",
19828 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
19829 ),
19830 (
19831 "different selected run",
19832 "409 Conflict",
19833 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
19834 ),
19835 (
19836 "different task attempt",
19837 "409 Conflict",
19838 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
19839 ),
19840 (
19841 "authentication failure",
19842 "401 Unauthorized",
19843 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
19844 ),
19845 (
19846 "authorization failure",
19847 "403 Forbidden",
19848 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
19849 ),
19850 (
19851 "protocol failure",
19852 "400 Bad Request",
19853 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
19854 ),
19855 (
19856 "malformed command",
19857 "422 Unprocessable Entity",
19858 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
19859 ),
19860 (
19861 "transient server failure",
19862 "503 Service Unavailable",
19863 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
19864 ),
19865 ] {
19866 let server = MockWorkerServer::workflow_completion(status, response);
19867 let client = Client::builder(server.base_url())
19868 .timeout(Duration::from_secs(2))
19869 .build()
19870 .expect("client");
19871 let mut worker = Worker::new(client, "rust-workers")
19872 .worker_id("timeout-worker")
19873 .poll_timeout(Duration::from_millis(10));
19874 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
19875 Ok(json!({"late": "result"}))
19876 });
19877
19878 let error = worker
19879 .run_once()
19880 .await
19881 .expect_err(&format!("{name} must remain an error"));
19882 assert!(
19883 matches!(error, Error::Http { .. } | Error::Protocol(_)),
19884 "{name} returned an unexpected error variant: {error}"
19885 );
19886 }
19887 }
19888
19889 #[tokio::test]
19890 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
19891 let server = MockWorkerServer::start();
19892 let client = Client::builder(server.base_url())
19893 .worker_token(Some("worker-secret".to_string()))
19894 .namespace("orders")
19895 .timeout(Duration::from_secs(2))
19896 .build()
19897 .expect("client");
19898 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
19899
19900 let result = client
19901 .deregister_worker_registration("worker/α space")
19902 .await
19903 .expect("deregister worker registration");
19904
19905 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
19906 assert_eq!(
19907 server.worker_protocol_for(path).as_deref(),
19908 Some(WORKER_PROTOCOL_VERSION)
19909 );
19910 assert_eq!(server.control_protocol_for(path), None);
19911 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
19912 assert_eq!(
19913 server.authorization_for(path).as_deref(),
19914 Some("Bearer worker-secret")
19915 );
19916 assert_eq!(
19917 result,
19918 WorkerDeregistrationEnvelope {
19919 worker_id: "deregistered-worker".to_string(),
19920 outcome: "deregistered".to_string(),
19921 recovered_workflow_task_count: 2,
19922 }
19923 );
19924 }
19925
19926 #[tokio::test]
19927 async fn low_level_registration_rejects_update_validators_before_transport() {
19928 let server = MockWorkerServer::start();
19929 let client = Client::builder(server.base_url())
19930 .timeout(Duration::from_secs(2))
19931 .build()
19932 .expect("client");
19933
19934 for update_validators in [json!(["approve"]), json!("approve")] {
19935 let error = client
19936 .register_worker_with_command_contracts(
19937 "validator-claiming-worker",
19938 "rust-workers",
19939 vec!["orders".to_string()],
19940 vec![],
19941 1,
19942 1,
19943 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
19944 json!({
19945 "orders": {
19946 "queries": ["current"],
19947 "updates": ["approve"],
19948 "update_validators": update_validators,
19949 },
19950 }),
19951 )
19952 .await
19953 .expect_err("unsupported validator claims must fail before registration");
19954
19955 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
19956 panic!("expected typed unsupported-validator failure");
19957 };
19958 assert_eq!(workflow_type, "orders");
19959 }
19960 assert_eq!(server.request_count("/api/worker/register"), 0);
19961 }
19962
19963 #[tokio::test]
19964 async fn low_level_registration_preserves_query_and_update_contracts() {
19965 let server = MockWorkerServer::start();
19966 let client = Client::builder(server.base_url())
19967 .timeout(Duration::from_secs(2))
19968 .build()
19969 .expect("client");
19970 let contracts = json!({
19971 "orders": {
19972 "queries": ["current"],
19973 "updates": ["approve"],
19974 "update_validators": [],
19975 },
19976 "payments": {
19977 "queries": ["status"],
19978 "updates": ["capture"],
19979 },
19980 });
19981
19982 client
19983 .register_worker_with_command_contracts(
19984 "command-worker",
19985 "rust-workers",
19986 vec!["orders".to_string(), "payments".to_string()],
19987 vec![],
19988 1,
19989 1,
19990 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
19991 contracts.clone(),
19992 )
19993 .await
19994 .expect("query and update contracts must remain supported");
19995
19996 assert_eq!(
19997 server.request_body("/api/worker/register")["workflow_command_contracts"],
19998 contracts
19999 );
20000 }
20001
20002 #[tokio::test]
20003 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
20004 let server = MockWorkerServer::start();
20005 let control_only = Client::builder(server.base_url())
20006 .control_token(Some("control-secret".to_string()))
20007 .build()
20008 .expect("control client");
20009
20010 let error = control_only
20011 .register_worker("worker", "queue", vec![], vec![], 1, 1)
20012 .await
20013 .expect_err("control token must not authorize a worker request");
20014 assert!(matches!(
20015 error,
20016 Error::MissingRoleCredentials { role: "worker", .. }
20017 ));
20018 assert_eq!(server.request_count("/api/worker/register"), 0);
20019
20020 let worker_only = Client::builder(server.base_url())
20021 .worker_token(Some("worker-secret".to_string()))
20022 .build()
20023 .expect("worker client");
20024 let error = worker_only
20025 .health()
20026 .await
20027 .expect_err("worker token must not authorize a control request");
20028 assert!(matches!(
20029 error,
20030 Error::MissingRoleCredentials {
20031 role: "control",
20032 ..
20033 }
20034 ));
20035 assert_eq!(server.request_count("/api/health"), 0);
20036 }
20037
20038 #[tokio::test]
20039 async fn shared_token_supports_worker_and_control_planes() {
20040 let server = MockWorkerServer::start();
20041 let client = Client::builder(server.base_url())
20042 .token(Some("shared-secret".to_string()))
20043 .build()
20044 .expect("client");
20045
20046 client.health().await.expect("control request");
20047 client
20048 .register_worker("worker", "queue", vec![], vec![], 1, 1)
20049 .await
20050 .expect("worker request");
20051
20052 assert_eq!(
20053 server.authorization_for("/api/health").as_deref(),
20054 Some("Bearer shared-secret")
20055 );
20056 assert_eq!(
20057 server.control_protocol_for("/api/health").as_deref(),
20058 Some(CONTROL_PLANE_VERSION)
20059 );
20060 assert_eq!(
20061 server.authorization_for("/api/worker/register").as_deref(),
20062 Some("Bearer shared-secret")
20063 );
20064 assert_eq!(
20065 server
20066 .worker_protocol_for("/api/worker/register")
20067 .as_deref(),
20068 Some(WORKER_PROTOCOL_VERSION)
20069 );
20070 }
20071
20072 #[tokio::test]
20073 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
20074 let server = MockWorkerServer::start();
20075 let client = Client::builder(server.base_url())
20076 .timeout(Duration::from_secs(2))
20077 .build()
20078 .expect("client");
20079
20080 client
20081 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
20082 .await
20083 .expect("register");
20084 client
20085 .heartbeat_worker("capture-worker", 1, 1)
20086 .await
20087 .expect("heartbeat");
20088 client
20089 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
20090 .await
20091 .expect("workflow poll");
20092 client
20093 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
20094 .await
20095 .expect("activity poll");
20096
20097 for path in [
20098 "/api/worker/register",
20099 "/api/worker/heartbeat",
20100 "/api/worker/workflow-tasks/poll",
20101 "/api/worker/activity-tasks/poll",
20102 ] {
20103 assert_eq!(
20104 server.worker_protocol_for(path).as_deref(),
20105 Some(WORKER_PROTOCOL_VERSION),
20106 "unexpected protocol for {path}"
20107 );
20108 }
20109
20110 assert_eq!(
20111 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
20112 1
20113 );
20114 assert_eq!(
20115 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
20116 1
20117 );
20118 assert!(
20119 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
20120 .as_str()
20121 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
20122 );
20123 assert!(
20124 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
20125 .as_str()
20126 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
20127 );
20128 }
20129
20130 #[tokio::test]
20131 async fn query_task_endpoints_send_the_query_feature_protocol() {
20132 let server = MockWorkerServer::start();
20133 let client = Client::builder(server.base_url())
20134 .timeout(Duration::from_secs(2))
20135 .build()
20136 .expect("client");
20137
20138 client
20139 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
20140 .await
20141 .expect("query poll");
20142 client
20143 .complete_query_task(
20144 "query-capture",
20145 "capture-worker",
20146 1,
20147 json!(8),
20148 DEFAULT_CODEC,
20149 )
20150 .await
20151 .expect("query complete");
20152 client
20153 .fail_query_task(
20154 "query-capture",
20155 "capture-worker",
20156 1,
20157 "failed",
20158 "query_rejected",
20159 "QueryFailed",
20160 )
20161 .await
20162 .expect("query fail");
20163
20164 for path in [
20165 "/api/worker/query-tasks/poll",
20166 "/api/worker/query-tasks/query-capture/complete",
20167 "/api/worker/query-tasks/query-capture/fail",
20168 ] {
20169 assert_eq!(
20170 server.worker_protocol_for(path).as_deref(),
20171 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
20172 "unexpected protocol for {path}"
20173 );
20174 }
20175
20176 assert_eq!(
20177 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
20178 1
20179 );
20180 assert!(
20181 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
20182 .as_str()
20183 .is_some_and(|id| id.starts_with("rust-query-poll-"))
20184 );
20185 }
20186
20187 #[tokio::test]
20188 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
20189 let server = MockWorkerServer::transient_worker_failures();
20190 let client = Client::builder(server.base_url())
20191 .timeout(Duration::from_secs(2))
20192 .build()
20193 .expect("client");
20194
20195 client
20196 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
20197 .await
20198 .expect("workflow poll retry");
20199 client
20200 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
20201 .await
20202 .expect("activity poll retry");
20203 client
20204 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
20205 .await
20206 .expect("query poll retry");
20207
20208 for path in [
20209 "/api/worker/workflow-tasks/poll",
20210 "/api/worker/activity-tasks/poll",
20211 "/api/worker/query-tasks/poll",
20212 ] {
20213 let bodies = server.request_bodies(path);
20214 assert_eq!(bodies.len(), 2, "{path} must be retried once");
20215 assert_eq!(
20216 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
20217 "{path} must preserve the request binding across retry"
20218 );
20219 }
20220 }
20221
20222 #[tokio::test]
20223 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
20224 let server = MockWorkerServer::consecutive_poll_failures(2);
20225 let client = Client::builder(server.base_url())
20226 .timeout(Duration::from_secs(2))
20227 .build()
20228 .expect("client");
20229 let mut worker = Worker::new(client, "capture")
20230 .worker_id("capture-worker")
20231 .poll_timeout(Duration::from_millis(10))
20232 .retry_policy(WorkerRetryPolicy {
20233 max_retries: 2,
20234 initial_backoff: Duration::from_millis(1),
20235 max_backoff: Duration::from_millis(1),
20236 });
20237 worker.register_workflow(
20238 "capture.workflow",
20239 |_ctx, _input| async move { Ok(Value::Null) },
20240 );
20241 worker.register_activity(
20242 "capture.activity",
20243 |_ctx, _input| async move { Ok(Value::Null) },
20244 );
20245 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
20246 Ok(Value::Null)
20247 });
20248
20249 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
20250
20251 for path in [
20252 "/api/worker/workflow-tasks/poll",
20253 "/api/worker/activity-tasks/poll",
20254 "/api/worker/query-tasks/poll",
20255 ] {
20256 let bodies = server.request_bodies(path);
20257 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
20258 assert!(
20259 bodies
20260 .iter()
20261 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
20262 "{path} must preserve one request binding across every retry"
20263 );
20264 }
20265 }
20266
20267 #[tokio::test]
20268 async fn query_protocol_rejection_from_older_server_is_typed() {
20269 let server = MockWorkerServer::reject_query_protocol();
20270 let client = Client::builder(server.base_url())
20271 .timeout(Duration::from_secs(2))
20272 .build()
20273 .expect("client");
20274
20275 let error = client
20276 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
20277 .await
20278 .expect_err("server below query protocol floor must reject");
20279 let Error::Protocol(failure) = error else {
20280 panic!("expected typed protocol failure");
20281 };
20282
20283 assert_eq!(failure.status, 400);
20284 assert_eq!(failure.reason, "unsupported_protocol_version");
20285 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
20286 assert_eq!(
20287 failure.requested_version.as_deref(),
20288 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
20289 );
20290 assert_eq!(
20291 server
20292 .worker_protocol_for("/api/worker/query-tasks/poll")
20293 .as_deref(),
20294 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
20295 );
20296 }
20297
20298 #[tokio::test]
20299 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
20300 let server = MockWorkerServer::reject_query_protocol();
20301 let client = Client::builder(server.base_url())
20302 .timeout(Duration::from_secs(2))
20303 .build()
20304 .expect("client");
20305 let mut worker = Worker::new(client, "rust-workers")
20306 .worker_id("baseline-worker")
20307 .poll_timeout(Duration::from_millis(10));
20308
20309 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
20310 Ok(Value::Null)
20311 });
20312
20313 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
20314 assert_eq!(
20315 server
20316 .worker_protocol_for("/api/worker/workflow-tasks/poll")
20317 .as_deref(),
20318 Some(WORKER_PROTOCOL_VERSION)
20319 );
20320 assert_eq!(
20321 server.worker_protocol_for("/api/worker/query-tasks/poll"),
20322 None,
20323 "a worker without query handlers must not use the query-task endpoint"
20324 );
20325 }
20326
20327 #[tokio::test]
20328 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
20329 let server = MockWorkerServer::reject_query_completion();
20330 let client = Client::builder(server.base_url())
20331 .timeout(Duration::from_secs(2))
20332 .build()
20333 .expect("client");
20334
20335 let error = client
20336 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
20337 .await
20338 .expect_err("expired completion must be rejected");
20339 let Error::QueryFailed(failure) = error else {
20340 panic!("expected typed query failure");
20341 };
20342 assert_eq!(failure.status, 409);
20343 assert_eq!(failure.reason, "query_task_timed_out");
20344
20345 let mut worker = Worker::new(client, "rust-workers")
20346 .worker_id("late-worker")
20347 .poll_timeout(Duration::from_millis(10));
20348 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
20349 worker.register_query(
20350 "counter",
20351 "current",
20352 |_ctx, _args| async move { Ok(json!(8)) },
20353 );
20354
20355 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
20356 assert_eq!(
20357 worker
20358 .run_once()
20359 .await
20360 .expect("worker continues after late completion"),
20361 0
20362 );
20363 assert_eq!(
20364 server.request_count("/api/worker/query-tasks/query-late/complete"),
20365 2
20366 );
20367 assert_eq!(
20368 server.request_count("/api/worker/query-tasks/query-late/fail"),
20369 0,
20370 "a server completion rejection must not be reported as an encoding failure"
20371 );
20372 }
20373
20374 #[tokio::test]
20375 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
20376 let server = MockWorkerServer::start();
20377 let client = Client::builder(server.base_url())
20378 .timeout(Duration::from_secs(2))
20379 .build()
20380 .expect("client");
20381 let mut worker = Worker::new(client, "rust-workers")
20382 .worker_id("joined-worker")
20383 .poll_timeout(Duration::from_millis(10));
20384 worker.register_workflow(
20385 "joined.workflow",
20386 |_ctx, _input| async move { Ok(Value::Null) },
20387 );
20388 worker.register_activity(
20389 "joined.activity",
20390 |_ctx, _input| async move { Ok(Value::Null) },
20391 );
20392 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
20393 Ok(Value::Null)
20394 });
20395
20396 worker
20397 .run_until(tokio::time::sleep(Duration::from_millis(20)))
20398 .await
20399 .expect("normal shutdown");
20400
20401 let deregistration_path = "/api/worker/registrations/mock-worker";
20402 assert_eq!(server.request_count(deregistration_path), 1);
20403 for poll_path in [
20404 "/api/worker/workflow-tasks/poll",
20405 "/api/worker/activity-tasks/poll",
20406 "/api/worker/query-tasks/poll",
20407 ] {
20408 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
20409 }
20410 assert_eq!(
20411 server.captured_paths().last().map(String::as_str),
20412 Some(deregistration_path),
20413 "deregistration must start only after every poller has joined"
20414 );
20415 }
20416
20417 #[tokio::test]
20418 async fn registration_failure_does_not_deregister() {
20419 let server = MockWorkerServer::rejected_registration();
20420 let client = Client::builder(server.base_url())
20421 .timeout(Duration::from_secs(2))
20422 .build()
20423 .expect("client");
20424 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
20425
20426 let error = worker
20427 .run_until(async {})
20428 .await
20429 .expect_err("registration must fail");
20430 assert!(matches!(
20431 error,
20432 Error::Http {
20433 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
20434 ..
20435 }
20436 ));
20437 assert!(server
20438 .captured_paths()
20439 .iter()
20440 .all(|path| !path.starts_with("/api/worker/registrations/")));
20441 }
20442
20443 #[tokio::test]
20444 async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
20445 let server = MockWorkerServer::rejected_registration_protocol();
20446 let client = Client::builder(server.base_url())
20447 .timeout(Duration::from_secs(2))
20448 .build()
20449 .expect("client");
20450 let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
20451
20452 let error = worker
20453 .run_until(async {})
20454 .await
20455 .expect_err("a protocol 1.16 server must reject this worker");
20456 let Error::Protocol(failure) = error else {
20457 panic!("expected typed protocol rejection");
20458 };
20459 assert_eq!(failure.reason, "unsupported_protocol_version");
20460 assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
20461 assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
20462 assert_eq!(
20463 server
20464 .worker_protocol_for("/api/worker/register")
20465 .as_deref(),
20466 Some(WORKER_PROTOCOL_VERSION)
20467 );
20468 }
20469
20470 #[tokio::test]
20471 async fn declined_registration_does_not_deregister() {
20472 let server = MockWorkerServer::declined_registration();
20473 let client = Client::builder(server.base_url())
20474 .timeout(Duration::from_secs(2))
20475 .build()
20476 .expect("client");
20477 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
20478
20479 let error = worker
20480 .run_until(async {})
20481 .await
20482 .expect_err("declined registration must fail");
20483 assert!(matches!(error, Error::WorkerLoop(_)));
20484 assert!(error.to_string().contains("was not accepted"));
20485 assert!(server
20486 .captured_paths()
20487 .iter()
20488 .all(|path| !path.starts_with("/api/worker/registrations/")));
20489 }
20490
20491 #[tokio::test]
20492 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
20493 let server = MockWorkerServer::rejected_deregistration();
20494 let client = Client::builder(server.base_url())
20495 .timeout(Duration::from_secs(2))
20496 .build()
20497 .expect("client");
20498 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
20499
20500 let error = worker
20501 .run_until(async {})
20502 .await
20503 .expect_err("deregistration must fail");
20504 assert!(matches!(
20505 error,
20506 Error::Http {
20507 status: reqwest::StatusCode::FORBIDDEN,
20508 ..
20509 }
20510 ));
20511 assert_eq!(
20512 server.request_count("/api/worker/registrations/mock-worker"),
20513 1
20514 );
20515 }
20516
20517 #[tokio::test]
20518 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
20519 let server = MockWorkerServer::rejected_deregistration_protocol();
20520 let client = Client::builder(server.base_url())
20521 .timeout(Duration::from_secs(2))
20522 .build()
20523 .expect("client");
20524 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
20525
20526 let error = worker
20527 .run_until(async {})
20528 .await
20529 .expect_err("protocol rejection must fail shutdown");
20530 let Error::Protocol(failure) = error else {
20531 panic!("expected typed protocol failure");
20532 };
20533 assert_eq!(failure.reason, "unsupported_protocol_version");
20534 assert_eq!(
20535 failure.requested_version.as_deref(),
20536 Some(WORKER_PROTOCOL_VERSION)
20537 );
20538 assert_eq!(
20539 server.request_count("/api/worker/registrations/mock-worker"),
20540 1
20541 );
20542 }
20543
20544 #[tokio::test]
20545 async fn primary_poller_error_retains_deregistration_failure_context() {
20546 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
20547 let client = Client::builder(server.base_url())
20548 .timeout(Duration::from_secs(2))
20549 .build()
20550 .expect("client");
20551 let mut worker = Worker::new(client, "rust-workers")
20552 .worker_id("combined-failure")
20553 .poll_timeout(Duration::from_millis(10));
20554 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
20555 Ok(Value::Null)
20556 });
20557
20558 let error = worker
20559 .run()
20560 .await
20561 .expect_err("worker and cleanup must fail");
20562 let summary = error.to_string();
20563 assert!(summary.contains("authentication_failed"));
20564 assert!(summary.contains("worker cannot deregister"));
20565 let Error::WorkerShutdown {
20566 primary,
20567 deregistration,
20568 } = error
20569 else {
20570 panic!("expected combined worker shutdown error");
20571 };
20572 assert!(matches!(
20573 *primary,
20574 Error::Http {
20575 status: reqwest::StatusCode::UNAUTHORIZED,
20576 ..
20577 }
20578 ));
20579 assert!(matches!(
20580 *deregistration,
20581 Error::Http {
20582 status: reqwest::StatusCode::FORBIDDEN,
20583 ..
20584 }
20585 ));
20586 assert_eq!(
20587 server.request_count("/api/worker/registrations/mock-worker"),
20588 1
20589 );
20590 }
20591
20592 #[tokio::test]
20593 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
20594 let server = MockWorkerServer::start();
20595 let client = Client::builder(server.base_url())
20596 .timeout(Duration::from_secs(2))
20597 .build()
20598 .expect("client");
20599 let mut worker = Worker::new(client, "rust-workers")
20600 .worker_id("activity-only-worker")
20601 .poll_timeout(Duration::from_millis(10));
20602
20603 worker.register_activity(
20604 "activity.only",
20605 |_ctx, _args| async move { Ok(Value::Null) },
20606 );
20607
20608 worker.run_until(async {}).await.expect("run worker");
20609 }
20610
20611 #[tokio::test]
20612 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
20613 let server = MockWorkerServer::start();
20614 let client = Client::builder(server.base_url())
20615 .timeout(Duration::from_secs(2))
20616 .build()
20617 .expect("client");
20618 let mut worker = Worker::new(client, "rust-workers")
20619 .worker_id("workflow-only-worker")
20620 .poll_timeout(Duration::from_millis(10));
20621
20622 worker.register_workflow(
20623 "workflow.only",
20624 |_ctx, _input| async move { Ok(Value::Null) },
20625 );
20626
20627 worker.run_until(async {}).await.expect("run worker");
20628 }
20629
20630 #[tokio::test]
20631 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
20632 let server = MockWorkerServer::start();
20633 let client = Client::builder(server.base_url())
20634 .timeout(Duration::from_secs(2))
20635 .build()
20636 .expect("client");
20637 let observations = Arc::new(Mutex::new(Vec::new()));
20638 let observed = Arc::clone(&observations);
20639 let mut worker = Worker::new(client, "rust-workers")
20640 .worker_id("observed-heartbeat-worker")
20641 .poll_timeout(Duration::from_millis(10))
20642 .on_worker_heartbeat(move |observation| {
20643 observed
20644 .lock()
20645 .expect("heartbeat observations")
20646 .push(observation.clone());
20647 });
20648
20649 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
20650 Ok(Value::Null)
20651 });
20652 let acknowledged = Arc::clone(&observations);
20653 worker
20654 .run_until(async move {
20655 tokio::time::timeout(Duration::from_secs(2), async move {
20656 loop {
20657 if !acknowledged
20658 .lock()
20659 .expect("heartbeat observations")
20660 .is_empty()
20661 {
20662 break;
20663 }
20664 tokio::time::sleep(Duration::from_millis(1)).await;
20665 }
20666 })
20667 .await
20668 .expect("heartbeat acknowledgement within timeout");
20669 })
20670 .await
20671 .expect("run worker");
20672
20673 let observations = observations.lock().expect("heartbeat observations");
20674 let first = observations.first().expect("heartbeat acknowledgement");
20675 assert_eq!(first.worker_id, "observed-heartbeat-worker");
20676 assert_eq!(first.task_queue, "rust-workers");
20677 assert!(first.acknowledged_at_unix_millis > 0);
20678 assert_eq!(first.acknowledgement, json!({}));
20679 }
20680
20681 #[tokio::test]
20682 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
20683 let server = MockWorkerServer::delayed_heartbeat_worker();
20684 let client = Client::builder(server.base_url())
20685 .timeout(Duration::from_secs(3))
20686 .build()
20687 .expect("client");
20688 let observations = Arc::new(Mutex::new(Vec::new()));
20689 let observed = Arc::clone(&observations);
20690 let mut worker = Worker::new(client, "rust-snapshot-workers")
20691 .worker_id("rust-snapshot-worker")
20692 .poll_timeout(Duration::from_millis(10))
20693 .on_worker_heartbeat(move |observation| {
20694 observed
20695 .lock()
20696 .expect("heartbeat observations")
20697 .push(observation.clone());
20698 });
20699
20700 worker.register_workflow("snapshot", |ctx, _input| async move {
20701 ctx.wait_signal("finish").await?;
20702 Ok(json!({"status": "finished"}))
20703 });
20704 worker.register_query("snapshot", "current", |ctx, _args| async move {
20705 Ok(json!(ctx
20706 .signals("increment")
20707 .iter()
20708 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
20709 .sum::<i64>()))
20710 });
20711 worker.register_activity("cancel-aware", |_ctx, _args| async move {
20712 Ok(json!({"late": "completion"}))
20713 });
20714
20715 worker
20716 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
20717 .await
20718 .expect("delayed heartbeat must allow a clean worker shutdown");
20719
20720 let observations = observations.lock().expect("heartbeat observations");
20721 assert!(
20722 observations.len() >= 3,
20723 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
20724 );
20725 assert!(
20726 observations.windows(2).all(|pair| {
20727 pair[1].acknowledged_at_unix_millis
20728 .saturating_sub(pair[0].acknowledged_at_unix_millis)
20729 >= 850
20730 }),
20731 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
20732 );
20733 drop(observations);
20734
20735 let heartbeat_times = server.request_times("/api/worker/heartbeat");
20736 let delayed_request_at = *heartbeat_times
20737 .get(1)
20738 .expect("intentionally delayed heartbeat request");
20739 let delay_window_start = delayed_request_at + Duration::from_millis(100);
20740 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
20741 for path in [
20742 "/api/worker/workflow-tasks/poll",
20743 "/api/worker/activity-tasks/poll",
20744 "/api/worker/query-tasks/poll",
20745 ] {
20746 assert!(
20747 server
20748 .request_times(path)
20749 .iter()
20750 .any(|received_at| *received_at >= delay_window_start
20751 && *received_at <= delay_window_end),
20752 "{path} must keep polling while a heartbeat acknowledgement is delayed"
20753 );
20754 }
20755 assert!(
20756 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
20757 "workflow work must be settled"
20758 );
20759 assert!(
20760 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
20761 "activity work must be settled"
20762 );
20763 assert!(
20764 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
20765 "query work must be settled"
20766 );
20767 }
20768
20769 #[tokio::test]
20770 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
20771 let server = MockWorkerServer::heartbeat_retry_worker();
20772 let client = Client::builder(server.base_url())
20773 .timeout(Duration::from_secs(2))
20774 .build()
20775 .expect("client");
20776 let observations = Arc::new(Mutex::new(Vec::new()));
20777 let observed = Arc::clone(&observations);
20778 let worker = Worker::new(client, "rust-workers")
20779 .worker_id("heartbeat-retry-worker")
20780 .retry_policy(WorkerRetryPolicy {
20781 max_retries: 1,
20782 initial_backoff: Duration::from_millis(300),
20783 max_backoff: Duration::from_millis(300),
20784 })
20785 .on_worker_heartbeat(move |observation| {
20786 observed
20787 .lock()
20788 .expect("heartbeat observations")
20789 .push(observation.clone());
20790 });
20791
20792 worker
20793 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
20794 .await
20795 .expect("retryable heartbeat failure must remain bounded and recover");
20796
20797 let observations = observations.lock().expect("heartbeat observations");
20798 assert!(observations.len() >= 3, "heartbeat retry must recover");
20799 assert!(
20800 observations.windows(2).all(|pair| {
20801 pair[1]
20802 .acknowledged_at_unix_millis
20803 .saturating_sub(pair[0].acknowledged_at_unix_millis)
20804 >= 850
20805 }),
20806 "a successful retry must start a fresh advertised cadence: {observations:?}"
20807 );
20808 assert_eq!(
20809 server.request_count("/api/worker/heartbeat"),
20810 observations.len() + 1,
20811 "one retryable failure must add exactly one bounded request"
20812 );
20813 }
20814
20815 #[tokio::test]
20816 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
20817 let server = MockWorkerServer::waiting_query_worker();
20818 let client = Client::builder(server.base_url())
20819 .timeout(Duration::from_secs(2))
20820 .build()
20821 .expect("client");
20822 let observations = Arc::new(Mutex::new(Vec::new()));
20823 let observed = Arc::clone(&observations);
20824 let mut worker = Worker::new(client, "rust-snapshot-workers")
20825 .worker_id("rust-snapshot-worker")
20826 .poll_timeout(Duration::from_millis(10))
20827 .on_worker_heartbeat(move |observation| {
20828 observed
20829 .lock()
20830 .expect("heartbeat observations")
20831 .push(observation.clone());
20832 });
20833
20834 worker.register_workflow("snapshot", |ctx, _input| async move {
20835 ctx.wait_signal("finish").await?;
20836 Ok(json!({"status": "finished"}))
20837 });
20838 worker.register_query("snapshot", "current", |ctx, _args| async move {
20839 let current = ctx
20840 .signals("increment")
20841 .iter()
20842 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
20843 .sum::<i64>();
20844 Ok(json!(current))
20845 });
20846 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
20847
20848 worker
20849 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
20850 .await
20851 .expect("pending workflow and query poller must remain live until shutdown");
20852
20853 assert!(
20854 observations.lock().expect("heartbeat observations").len() >= 4,
20855 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
20856 );
20857 assert!(
20858 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
20859 "workflow polling must continue after empty replay acknowledgements"
20860 );
20861 assert!(
20862 server.request_count("/api/worker/query-tasks/poll") >= 2,
20863 "query polling must continue after serving the current query"
20864 );
20865 assert_eq!(
20866 server.request_body("/api/worker/register")["capabilities"],
20867 json!([
20868 CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
20869 MEMO_UPSERTS_CAPABILITY,
20870 TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
20871 QUERY_TASKS_CAPABILITY,
20872 WORKFLOW_UPDATES_CAPABILITY,
20873 MESSAGE_STREAMS_CAPABILITY
20874 ])
20875 );
20876 assert_eq!(
20877 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
20878 json!({
20879 "queries": ["current"],
20880 "query_contracts": [],
20881 "signals": [],
20882 "signal_contracts": [],
20883 "updates": ["replace"],
20884 "update_contracts": [],
20885 "update_validators": [],
20886 })
20887 );
20888
20889 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
20890 assert_eq!(
20891 opened["commands"],
20892 json!([{
20893 "type": "open_signal_wait",
20894 "signal_name": "finish",
20895 }])
20896 );
20897
20898 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
20899 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
20900 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
20901 let failure = server.request_body(&fail_path);
20902 assert_eq!(
20903 failure["failure"]["type"],
20904 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
20905 );
20906 assert_eq!(server.request_count(&completion_path), 0);
20907 }
20908
20909 let query_completion =
20910 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
20911 assert_eq!(query_completion["result"], json!(8));
20912
20913 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
20914 assert_eq!(
20915 server.request_count(terminal_path),
20916 1,
20917 "the matching signal must settle the workflow exactly once"
20918 );
20919 let terminal = server.request_body(terminal_path);
20920 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
20921 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
20922 assert_eq!(
20923 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
20924 .expect("terminal workflow result"),
20925 json!({"status": "finished"})
20926 );
20927 }
20928
20929 #[tokio::test]
20930 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
20931 let server = MockWorkerServer::transient_worker_failures();
20932 let client = Client::builder(server.base_url())
20933 .timeout(Duration::from_secs(2))
20934 .build()
20935 .expect("client");
20936 let mut worker = Worker::new(client, "rust-workers")
20937 .worker_id("retry-worker")
20938 .poll_timeout(Duration::from_millis(10))
20939 .retry_policy(WorkerRetryPolicy {
20940 max_retries: 2,
20941 initial_backoff: Duration::from_millis(1),
20942 max_backoff: Duration::from_millis(1),
20943 });
20944 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
20945 worker.register_activity(
20946 "counter.activity",
20947 |_ctx, _input| async move { Ok(Value::Null) },
20948 );
20949 worker.register_query(
20950 "counter",
20951 "current",
20952 |_ctx, _args| async move { Ok(json!(8)) },
20953 );
20954
20955 worker
20956 .run_until(tokio::time::sleep(Duration::from_millis(75)))
20957 .await
20958 .expect("transient failures must not stop the worker");
20959
20960 for path in [
20961 "/api/worker/heartbeat",
20962 "/api/worker/workflow-tasks/poll",
20963 "/api/worker/activity-tasks/poll",
20964 "/api/worker/query-tasks/poll",
20965 ] {
20966 assert!(
20967 server.request_count(path) >= 2,
20968 "{path} must continue after its transient failure"
20969 );
20970 }
20971 }
20972
20973 #[tokio::test]
20974 async fn worker_bounds_transport_retries() {
20975 let server = MockWorkerServer::unavailable_polls();
20976 let client = Client::builder(server.base_url())
20977 .timeout(Duration::from_secs(2))
20978 .build()
20979 .expect("client");
20980 let mut worker = Worker::new(client, "rust-workers")
20981 .worker_id("bounded-retry-worker")
20982 .poll_timeout(Duration::from_millis(10))
20983 .retry_policy(WorkerRetryPolicy {
20984 max_retries: 2,
20985 initial_backoff: Duration::from_millis(1),
20986 max_backoff: Duration::from_millis(1),
20987 });
20988 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
20989
20990 let error = worker.run().await.expect_err("retry bound must terminate");
20991 assert!(matches!(error, Error::Transport(_)));
20992 assert_eq!(
20993 server.request_count("/api/worker/workflow-tasks/poll"),
20994 3,
20995 "one initial request plus exactly two retries"
20996 );
20997 }
20998
20999 #[tokio::test]
21000 async fn worker_retry_policy_can_disable_poll_retries() {
21001 let server = MockWorkerServer::unavailable_polls();
21002 let client = Client::builder(server.base_url())
21003 .timeout(Duration::from_secs(2))
21004 .build()
21005 .expect("client");
21006 let mut worker = Worker::new(client, "rust-workers")
21007 .worker_id("no-retry-worker")
21008 .poll_timeout(Duration::from_millis(10))
21009 .retry_policy(WorkerRetryPolicy {
21010 max_retries: 0,
21011 initial_backoff: Duration::from_millis(1),
21012 max_backoff: Duration::from_millis(1),
21013 });
21014 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21015
21016 let error = worker
21017 .run_once()
21018 .await
21019 .expect_err("disabled retries must return the first transport failure");
21020 assert!(matches!(error, Error::Transport(_)));
21021 assert_eq!(
21022 server.request_count("/api/worker/workflow-tasks/poll"),
21023 1,
21024 "max_retries=0 must send only the initial request"
21025 );
21026 }
21027
21028 #[tokio::test]
21029 async fn worker_does_not_retry_authentication_failures() {
21030 let server = MockWorkerServer::unauthorized_polls();
21031 let client = Client::builder(server.base_url())
21032 .timeout(Duration::from_secs(2))
21033 .build()
21034 .expect("client");
21035 let mut worker = Worker::new(client, "rust-workers")
21036 .worker_id("unauthorized-worker")
21037 .poll_timeout(Duration::from_millis(10));
21038 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21039
21040 let error = worker
21041 .run()
21042 .await
21043 .expect_err("authentication must terminate");
21044 let Error::Http { status, body } = error else {
21045 panic!("expected stable HTTP authentication error");
21046 };
21047 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
21048 assert!(body.contains("authentication_failed"));
21049 assert_eq!(
21050 server.request_count("/api/worker/workflow-tasks/poll"),
21051 1,
21052 "authentication failures must not be retried"
21053 );
21054 }
21055
21056 #[derive(Clone, Debug)]
21057 struct CapturedRequest {
21058 method: String,
21059 path: String,
21060 authorization: Option<String>,
21061 namespace: Option<String>,
21062 worker_protocol: Option<String>,
21063 control_protocol: Option<String>,
21064 body: String,
21065 received_at: Instant,
21066 }
21067
21068 struct MockWorkerServer {
21069 addr: SocketAddr,
21070 stop: Arc<AtomicBool>,
21071 requests: Arc<Mutex<Vec<CapturedRequest>>>,
21072 thread: Option<thread::JoinHandle<()>>,
21073 }
21074
21075 #[derive(Clone, Copy, Default)]
21076 struct MockWorkerBehavior {
21077 reject_query_protocol: bool,
21078 reject_query_completion: bool,
21079 waiting_query_worker: bool,
21080 decline_registration: bool,
21081 complete_named_signal: bool,
21082 poll_failures_per_path: usize,
21083 heartbeat_failures: usize,
21084 heartbeat_failure_request: Option<usize>,
21085 delayed_heartbeat_request: Option<usize>,
21086 heartbeat_response_delay: Duration,
21087 concurrent_requests: bool,
21088 unauthorized_polls: bool,
21089 reject_registration: bool,
21090 reject_registration_protocol: bool,
21091 reject_deregistration: bool,
21092 reject_deregistration_protocol: bool,
21093 cancelled_activity: bool,
21094 draining_polls: bool,
21095 invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
21096 workflow_completion_status: Option<&'static str>,
21097 workflow_completion_body: Option<&'static str>,
21098 }
21099
21100 impl MockWorkerServer {
21101 fn start() -> Self {
21102 Self::start_with_behavior(MockWorkerBehavior::default())
21103 }
21104
21105 fn reject_query_protocol() -> Self {
21106 Self::start_with_behavior(MockWorkerBehavior {
21107 reject_query_protocol: true,
21108 ..MockWorkerBehavior::default()
21109 })
21110 }
21111
21112 fn reject_query_completion() -> Self {
21113 Self::start_with_behavior(MockWorkerBehavior {
21114 reject_query_completion: true,
21115 ..MockWorkerBehavior::default()
21116 })
21117 }
21118
21119 fn waiting_query_worker() -> Self {
21120 Self::start_with_behavior(MockWorkerBehavior {
21121 waiting_query_worker: true,
21122 complete_named_signal: true,
21123 ..MockWorkerBehavior::default()
21124 })
21125 }
21126
21127 fn transient_worker_failures() -> Self {
21128 Self::start_with_behavior(MockWorkerBehavior {
21129 poll_failures_per_path: 1,
21130 heartbeat_failures: 1,
21131 ..MockWorkerBehavior::default()
21132 })
21133 }
21134
21135 fn consecutive_poll_failures(count: usize) -> Self {
21136 Self::start_with_behavior(MockWorkerBehavior {
21137 poll_failures_per_path: count,
21138 ..MockWorkerBehavior::default()
21139 })
21140 }
21141
21142 fn delayed_heartbeat_worker() -> Self {
21143 Self::start_with_behavior(MockWorkerBehavior {
21144 waiting_query_worker: true,
21145 delayed_heartbeat_request: Some(2),
21146 heartbeat_response_delay: Duration::from_millis(1_500),
21147 concurrent_requests: true,
21148 cancelled_activity: true,
21149 ..MockWorkerBehavior::default()
21150 })
21151 }
21152
21153 fn heartbeat_retry_worker() -> Self {
21154 Self::start_with_behavior(MockWorkerBehavior {
21155 waiting_query_worker: true,
21156 heartbeat_failure_request: Some(2),
21157 concurrent_requests: true,
21158 ..MockWorkerBehavior::default()
21159 })
21160 }
21161
21162 fn unavailable_polls() -> Self {
21163 Self::start_with_behavior(MockWorkerBehavior {
21164 poll_failures_per_path: usize::MAX,
21165 ..MockWorkerBehavior::default()
21166 })
21167 }
21168
21169 fn unauthorized_polls() -> Self {
21170 Self::start_with_behavior(MockWorkerBehavior {
21171 unauthorized_polls: true,
21172 ..MockWorkerBehavior::default()
21173 })
21174 }
21175
21176 fn rejected_registration() -> Self {
21177 Self::start_with_behavior(MockWorkerBehavior {
21178 reject_registration: true,
21179 ..MockWorkerBehavior::default()
21180 })
21181 }
21182
21183 fn rejected_registration_protocol() -> Self {
21184 Self::start_with_behavior(MockWorkerBehavior {
21185 reject_registration_protocol: true,
21186 ..MockWorkerBehavior::default()
21187 })
21188 }
21189
21190 fn declined_registration() -> Self {
21191 Self::start_with_behavior(MockWorkerBehavior {
21192 decline_registration: true,
21193 ..MockWorkerBehavior::default()
21194 })
21195 }
21196
21197 fn rejected_deregistration() -> Self {
21198 Self::start_with_behavior(MockWorkerBehavior {
21199 reject_deregistration: true,
21200 ..MockWorkerBehavior::default()
21201 })
21202 }
21203
21204 fn rejected_deregistration_protocol() -> Self {
21205 Self::start_with_behavior(MockWorkerBehavior {
21206 reject_deregistration_protocol: true,
21207 ..MockWorkerBehavior::default()
21208 })
21209 }
21210
21211 fn unauthorized_polls_and_rejected_deregistration() -> Self {
21212 Self::start_with_behavior(MockWorkerBehavior {
21213 unauthorized_polls: true,
21214 reject_deregistration: true,
21215 ..MockWorkerBehavior::default()
21216 })
21217 }
21218
21219 fn cancelled_activity() -> Self {
21220 Self::start_with_behavior(MockWorkerBehavior {
21221 cancelled_activity: true,
21222 ..MockWorkerBehavior::default()
21223 })
21224 }
21225
21226 fn draining_polls() -> Self {
21227 Self::start_with_behavior(MockWorkerBehavior {
21228 draining_polls: true,
21229 ..MockWorkerBehavior::default()
21230 })
21231 }
21232
21233 fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
21234 Self::start_with_behavior(MockWorkerBehavior {
21235 invalid_task_payload_codec: Some(codec),
21236 ..MockWorkerBehavior::default()
21237 })
21238 }
21239
21240 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
21241 Self::start_with_behavior(MockWorkerBehavior {
21242 workflow_completion_status: Some(status),
21243 workflow_completion_body: Some(body),
21244 ..MockWorkerBehavior::default()
21245 })
21246 }
21247
21248 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
21249 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
21250 listener
21251 .set_nonblocking(true)
21252 .expect("configure mock listener");
21253 let addr = listener.local_addr().expect("mock server address");
21254 let stop = Arc::new(AtomicBool::new(false));
21255 let server_stop = Arc::clone(&stop);
21256 let requests = Arc::new(Mutex::new(Vec::new()));
21257 let server_requests = Arc::clone(&requests);
21258 let thread = thread::spawn(move || {
21259 let mut request_threads = Vec::new();
21260 while !server_stop.load(Ordering::SeqCst) {
21261 match listener.accept() {
21262 Ok((mut stream, _)) => {
21263 if behavior.concurrent_requests {
21264 let requests = Arc::clone(&server_requests);
21265 request_threads.push(thread::spawn(move || {
21266 handle_mock_worker_request(&mut stream, &requests, behavior)
21267 }));
21268 } else {
21269 handle_mock_worker_request(&mut stream, &server_requests, behavior);
21270 }
21271 }
21272 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
21273 let mut index = 0;
21274 while index < request_threads.len() {
21275 if request_threads[index].is_finished() {
21276 request_threads
21277 .swap_remove(index)
21278 .join()
21279 .expect("join mock request");
21280 } else {
21281 index += 1;
21282 }
21283 }
21284 thread::sleep(Duration::from_millis(5));
21285 }
21286 Err(_) => break,
21287 }
21288 }
21289 for request_thread in request_threads {
21290 request_thread.join().expect("join mock request");
21291 }
21292 });
21293
21294 Self {
21295 addr,
21296 stop,
21297 requests,
21298 thread: Some(thread),
21299 }
21300 }
21301
21302 fn base_url(&self) -> String {
21303 format!("http://{}", self.addr)
21304 }
21305
21306 fn worker_protocol_for(&self, path: &str) -> Option<String> {
21307 self.requests
21308 .lock()
21309 .expect("captured requests")
21310 .iter()
21311 .find(|request| request.path == path)
21312 .and_then(|request| request.worker_protocol.clone())
21313 }
21314
21315 fn control_protocol_for(&self, path: &str) -> Option<String> {
21316 self.requests
21317 .lock()
21318 .expect("captured requests")
21319 .iter()
21320 .find(|request| request.path == path)
21321 .and_then(|request| request.control_protocol.clone())
21322 }
21323
21324 fn method_for(&self, path: &str) -> Option<String> {
21325 self.requests
21326 .lock()
21327 .expect("captured requests")
21328 .iter()
21329 .find(|request| request.path == path)
21330 .map(|request| request.method.clone())
21331 }
21332
21333 fn authorization_for(&self, path: &str) -> Option<String> {
21334 self.requests
21335 .lock()
21336 .expect("captured requests")
21337 .iter()
21338 .find(|request| request.path == path)
21339 .and_then(|request| request.authorization.clone())
21340 }
21341
21342 fn namespace_for(&self, path: &str) -> Option<String> {
21343 self.requests
21344 .lock()
21345 .expect("captured requests")
21346 .iter()
21347 .find(|request| request.path == path)
21348 .and_then(|request| request.namespace.clone())
21349 }
21350
21351 fn request_count(&self, path: &str) -> usize {
21352 self.requests
21353 .lock()
21354 .expect("captured requests")
21355 .iter()
21356 .filter(|request| request.path == path)
21357 .count()
21358 }
21359
21360 fn captured_paths(&self) -> Vec<String> {
21361 self.requests
21362 .lock()
21363 .expect("captured requests")
21364 .iter()
21365 .map(|request| request.path.clone())
21366 .collect()
21367 }
21368
21369 fn request_times(&self, path: &str) -> Vec<Instant> {
21370 self.requests
21371 .lock()
21372 .expect("captured requests")
21373 .iter()
21374 .filter(|request| request.path == path)
21375 .map(|request| request.received_at)
21376 .collect()
21377 }
21378
21379 fn request_body(&self, path: &str) -> Value {
21380 let requests = self.requests.lock().expect("captured requests");
21381 let body = &requests
21382 .iter()
21383 .find(|request| request.path == path)
21384 .unwrap_or_else(|| panic!("missing request for {path}"))
21385 .body;
21386 serde_json::from_str(body).unwrap_or_else(|error| {
21387 panic!("invalid JSON request body for {path}: {error}: {body:?}")
21388 })
21389 }
21390
21391 fn request_bodies(&self, path: &str) -> Vec<Value> {
21392 self.requests
21393 .lock()
21394 .expect("captured requests")
21395 .iter()
21396 .filter(|request| request.path == path)
21397 .map(|request| {
21398 serde_json::from_str(&request.body).unwrap_or_else(|error| {
21399 panic!(
21400 "invalid JSON request body for {path}: {error}: {:?}",
21401 request.body
21402 )
21403 })
21404 })
21405 .collect()
21406 }
21407 }
21408
21409 impl Drop for MockWorkerServer {
21410 fn drop(&mut self) {
21411 self.stop.store(true, Ordering::SeqCst);
21412 let _ = TcpStream::connect(self.addr);
21413
21414 if let Some(thread) = self.thread.take() {
21415 thread.join().expect("join mock server");
21416 }
21417 }
21418 }
21419
21420 fn handle_mock_worker_request(
21421 stream: &mut TcpStream,
21422 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
21423 behavior: MockWorkerBehavior,
21424 ) {
21425 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
21426 let mut buffer = [0_u8; 8192];
21427 let mut request = Vec::new();
21428
21429 loop {
21430 match stream.read(&mut buffer) {
21431 Ok(0) => break,
21432 Ok(read) => {
21433 request.extend_from_slice(&buffer[..read]);
21434 if mock_request_is_complete(&request) {
21435 break;
21436 }
21437 }
21438 Err(error)
21439 if matches!(
21440 error.kind(),
21441 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
21442 ) =>
21443 {
21444 break;
21445 }
21446 Err(_) => return,
21447 }
21448 }
21449
21450 let request = String::from_utf8_lossy(&request);
21451 let body = request
21452 .split_once("\r\n\r\n")
21453 .map(|(_, body)| body)
21454 .unwrap_or_default();
21455 let path = request
21456 .lines()
21457 .next()
21458 .and_then(|line| line.split_whitespace().nth(1))
21459 .unwrap_or_default();
21460 let method = request
21461 .lines()
21462 .next()
21463 .and_then(|line| line.split_whitespace().next())
21464 .unwrap_or_default();
21465 let authorization = request.lines().find_map(|line| {
21466 let (name, value) = line.split_once(':')?;
21467 name.eq_ignore_ascii_case("Authorization")
21468 .then(|| value.trim().to_string())
21469 });
21470 let namespace = request.lines().find_map(|line| {
21471 let (name, value) = line.split_once(':')?;
21472 name.eq_ignore_ascii_case("X-Namespace")
21473 .then(|| value.trim().to_string())
21474 });
21475 let worker_protocol = request.lines().find_map(|line| {
21476 let (name, value) = line.split_once(':')?;
21477 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
21478 .then(|| value.trim().to_string())
21479 });
21480 let control_protocol = request.lines().find_map(|line| {
21481 let (name, value) = line.split_once(':')?;
21482 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
21483 .then(|| value.trim().to_string())
21484 });
21485 let request_number = {
21486 let mut requests = requests.lock().expect("captured requests");
21487 requests.push(CapturedRequest {
21488 method: method.to_string(),
21489 path: path.to_string(),
21490 authorization,
21491 namespace,
21492 worker_protocol: worker_protocol.clone(),
21493 control_protocol,
21494 body: body.to_string(),
21495 received_at: Instant::now(),
21496 });
21497 requests
21498 .iter()
21499 .filter(|request| request.path == path)
21500 .count()
21501 };
21502
21503 if path == "/api/worker/register" {
21504 if behavior.reject_registration_protocol {
21505 write_mock_response(
21506 stream,
21507 "400 Bad Request",
21508 r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
21509 );
21510 return;
21511 }
21512 if behavior.reject_registration {
21513 write_mock_response(
21514 stream,
21515 "503 Service Unavailable",
21516 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
21517 );
21518 return;
21519 }
21520 }
21521
21522 if path.starts_with("/api/worker/registrations/") {
21523 if behavior.reject_deregistration_protocol {
21524 write_mock_response(
21525 stream,
21526 "400 Bad Request",
21527 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.16","requested_version":"1.17"}"#,
21528 );
21529 } else if behavior.reject_deregistration {
21530 write_mock_response(
21531 stream,
21532 "403 Forbidden",
21533 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
21534 );
21535 } else {
21536 write_mock_response(
21537 stream,
21538 "200 OK",
21539 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
21540 );
21541 }
21542 return;
21543 }
21544
21545 let is_poll = matches!(
21546 path,
21547 "/api/worker/workflow-tasks/poll"
21548 | "/api/worker/activity-tasks/poll"
21549 | "/api/worker/query-tasks/poll"
21550 );
21551 if is_poll && request_number <= behavior.poll_failures_per_path {
21552 return;
21553 }
21554 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
21555 return;
21556 }
21557 if path == "/api/worker/heartbeat"
21558 && behavior.heartbeat_failure_request == Some(request_number)
21559 {
21560 return;
21561 }
21562 if path == "/api/worker/heartbeat"
21563 && behavior.delayed_heartbeat_request == Some(request_number)
21564 {
21565 thread::sleep(behavior.heartbeat_response_delay);
21566 }
21567 if behavior.unauthorized_polls && is_poll {
21568 write_mock_response(
21569 stream,
21570 "401 Unauthorized",
21571 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
21572 );
21573 return;
21574 }
21575 if behavior.draining_polls && is_poll {
21576 write_mock_response(
21577 stream,
21578 "409 Conflict",
21579 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
21580 );
21581 return;
21582 }
21583
21584 if let Some(codec_case) = behavior.invalid_task_payload_codec {
21585 if is_poll && request_number == 1 {
21586 let mut task = match path {
21587 "/api/worker/workflow-tasks/poll" => json!({
21588 "task_id": "codec-workflow",
21589 "workflow_type": "codec.workflow",
21590 "payload_codec": DEFAULT_CODEC,
21591 "workflow_task_attempt": 1,
21592 "lease_owner": "codec-worker"
21593 }),
21594 "/api/worker/activity-tasks/poll" => json!({
21595 "task_id": "codec-activity",
21596 "activity_attempt_id": "codec-activity-attempt",
21597 "activity_type": "codec.activity",
21598 "payload_codec": DEFAULT_CODEC,
21599 "attempt_number": 1,
21600 "lease_owner": "codec-worker"
21601 }),
21602 "/api/worker/query-tasks/poll" => json!({
21603 "query_task_id": "codec-query",
21604 "query_task_attempt": 1,
21605 "workflow_type": "codec.workflow",
21606 "query_name": "known",
21607 "payload_codec": DEFAULT_CODEC,
21608 "lease_owner": "codec-worker"
21609 }),
21610 _ => unreachable!("is_poll limits task codec probe paths"),
21611 };
21612 codec_case.apply(&mut task);
21613 write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
21614 return;
21615 }
21616
21617 if matches!(
21618 path,
21619 "/api/worker/workflow-tasks/codec-workflow/fail"
21620 | "/api/worker/activity-tasks/codec-activity/fail"
21621 | "/api/worker/query-tasks/codec-query/fail"
21622 ) {
21623 write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
21624 return;
21625 }
21626 }
21627
21628 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
21629 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
21630 let body = format!(
21631 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
21632 );
21633 write_mock_response(stream, "400 Bad Request", &body);
21634 return;
21635 }
21636
21637 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
21638 {
21639 write_mock_response(
21640 stream,
21641 "409 Conflict",
21642 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
21643 );
21644 return;
21645 }
21646
21647 if behavior.workflow_completion_status.is_some()
21648 && path == "/api/worker/workflow-tasks/poll"
21649 && request_number == 1
21650 {
21651 write_mock_response(
21652 stream,
21653 "200 OK",
21654 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
21655 );
21656 return;
21657 }
21658
21659 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
21660 if let (Some(status), Some(body)) = (
21661 behavior.workflow_completion_status,
21662 behavior.workflow_completion_body,
21663 ) {
21664 write_mock_response(stream, status, body);
21665 return;
21666 }
21667 }
21668
21669 if behavior.waiting_query_worker {
21670 if behavior.complete_named_signal
21671 && path == "/api/worker/workflow-tasks/poll"
21672 && request_number == 1
21673 {
21674 let body = json!({
21675 "task": {
21676 "task_id": "snapshot-open",
21677 "workflow_id": "snapshot-1",
21678 "run_id": "snapshot-run-1",
21679 "workflow_type": "snapshot",
21680 "payload_codec": DEFAULT_CODEC,
21681 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
21682 .expect("Avro workflow arguments"),
21683 "history_events": [],
21684 "workflow_task_attempt": 1,
21685 "lease_owner": "rust-snapshot-worker"
21686 }
21687 })
21688 .to_string();
21689 write_mock_response(stream, "200 OK", &body);
21690 return;
21691 }
21692
21693 let signal_request = request_number - usize::from(behavior.complete_named_signal);
21694 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
21695 if path == "/api/worker/workflow-tasks/poll"
21696 && signal_request >= 1
21697 && signal_request <= signal_request_limit
21698 {
21699 let finish = behavior.complete_named_signal && signal_request == 3;
21700 let amounts = if signal_request == 1 {
21701 vec![3]
21702 } else {
21703 vec![3, 5]
21704 };
21705 let task_id = if signal_request == 1 {
21706 "snapshot-wait-3"
21707 } else if finish {
21708 "snapshot-finish"
21709 } else {
21710 "snapshot-wait-5"
21711 };
21712 let mut history_events = std::iter::once(json!({
21713 "event_type": "SignalWaitOpened",
21714 "payload": {"sequence": 1, "signal_name": "finish"}
21715 }))
21716 .chain(amounts.iter().enumerate().map(|(index, amount)| {
21717 json!({
21718 "event_type": "SignalReceived",
21719 "payload": {
21720 "signal_id": format!("increment-{amount}"),
21721 "signal_name": "increment",
21722 "workflow_sequence": index + 2,
21723 "payload_codec": DEFAULT_CODEC,
21724 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
21725 .expect("Avro signal envelope")
21726 }
21727 })
21728 }))
21729 .collect::<Vec<_>>();
21730 let (resume_id, resume_name, resume_arguments) = if finish {
21731 history_events.push(json!({
21732 "event_type": "SignalReceived",
21733 "payload": {
21734 "signal_id": "finish",
21735 "signal_name": "finish",
21736 "workflow_sequence": 4,
21737 "payload_codec": DEFAULT_CODEC,
21738 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
21739 .expect("Avro finish signal envelope")
21740 }
21741 }));
21742 (
21743 "finish".to_string(),
21744 "finish".to_string(),
21745 encode_value_envelope(&json!([]), DEFAULT_CODEC)
21746 .expect("Avro finish resume signal"),
21747 )
21748 } else {
21749 let amount = amounts.last().expect("amount");
21750 (
21751 format!("increment-{amount}"),
21752 "increment".to_string(),
21753 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
21754 .expect("Avro increment resume signal"),
21755 )
21756 };
21757 let body = json!({
21758 "task": {
21759 "task_id": task_id,
21760 "workflow_id": "snapshot-1",
21761 "run_id": "snapshot-run-1",
21762 "workflow_type": "snapshot",
21763 "payload_codec": DEFAULT_CODEC,
21764 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
21765 .expect("Avro workflow arguments"),
21766 "history_events": history_events,
21767 "workflow_task_attempt": 1,
21768 "workflow_signal_id": resume_id,
21769 "signal_name": resume_name,
21770 "signal_arguments": resume_arguments,
21771 "lease_owner": "rust-snapshot-worker"
21772 }
21773 })
21774 .to_string();
21775 write_mock_response(stream, "200 OK", &body);
21776 return;
21777 }
21778
21779 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
21780 let history_events = [3, 5]
21781 .into_iter()
21782 .enumerate()
21783 .map(|(index, amount)| {
21784 json!({
21785 "event_type": "SignalReceived",
21786 "payload": {
21787 "signal_id": format!("increment-{amount}"),
21788 "signal_name": "increment",
21789 "workflow_sequence": index + 2,
21790 "payload_codec": DEFAULT_CODEC,
21791 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
21792 .expect("Avro query signal envelope")
21793 }
21794 })
21795 })
21796 .collect::<Vec<_>>();
21797 let body = json!({
21798 "task": {
21799 "query_task_id": "snapshot-current",
21800 "query_task_attempt": 1,
21801 "lease_owner": "rust-snapshot-worker",
21802 "workflow_id": "snapshot-1",
21803 "run_id": "snapshot-run-1",
21804 "workflow_type": "snapshot",
21805 "query_name": "current",
21806 "payload_codec": DEFAULT_CODEC,
21807 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
21808 .expect("Avro workflow arguments"),
21809 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
21810 .expect("Avro query arguments"),
21811 "history_events": history_events,
21812 "run_status": "waiting"
21813 }
21814 })
21815 .to_string();
21816 write_mock_response(stream, "200 OK", &body);
21817 return;
21818 }
21819
21820 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
21821 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
21822 {
21823 write_mock_response(
21824 stream,
21825 "200 OK",
21826 r#"{"outcome":"waiting_for_history","recorded":true}"#,
21827 );
21828 return;
21829 }
21830
21831 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
21832 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
21833 return;
21834 }
21835
21836 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
21837 write_mock_response(
21838 stream,
21839 "200 OK",
21840 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
21841 );
21842 return;
21843 }
21844
21845 if path == "/api/worker/query-tasks/snapshot-current/complete" {
21846 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
21847 return;
21848 }
21849 }
21850
21851 if matches!(
21852 path,
21853 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
21854 ) {
21855 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21856 .expect("typed mock result");
21857 let body = json!({
21858 "result": typed_fidelity_probe().into_json().expect("result projection"),
21859 "result_envelope": result,
21860 })
21861 .to_string();
21862 write_mock_response(stream, "200 OK", &body);
21863 return;
21864 }
21865
21866 if path == "/api/workflows/typed-1" {
21867 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21868 .expect("typed mock result");
21869 let body = json!({
21870 "workflow_id": "typed-1",
21871 "run_id": "run-typed-1",
21872 "workflow_type": "typed.echo",
21873 "status": "completed",
21874 "output": typed_fidelity_probe().into_json().expect("output projection"),
21875 "output_envelope": result,
21876 })
21877 .to_string();
21878 write_mock_response(stream, "200 OK", &body);
21879 return;
21880 }
21881
21882 let (status, body) = match path {
21883 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
21884 "/api/workflows" => (
21885 "201 Created",
21886 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
21887 ),
21888 "/api/worker/register" if behavior.decline_registration => (
21889 "200 OK",
21890 r#"{"worker_id":"declined-worker","registered":false}"#,
21891 ),
21892 "/api/worker/register" if behavior.waiting_query_worker => (
21893 "200 OK",
21894 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
21895 ),
21896 "/api/worker/register" => (
21897 "200 OK",
21898 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
21899 ),
21900 "/api/worker/heartbeat" => ("200 OK", "{}"),
21901 "/api/worker/activity-tasks/poll"
21902 if behavior.cancelled_activity && request_number == 1 =>
21903 {
21904 (
21905 "200 OK",
21906 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
21907 )
21908 }
21909 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
21910 ("200 OK", r#"{"task":null}"#)
21911 }
21912 "/api/worker/query-tasks/poll"
21913 if behavior.reject_query_completion && request_number == 1 =>
21914 {
21915 (
21916 "200 OK",
21917 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
21918 )
21919 }
21920 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
21921 "/api/worker/query-tasks/query-capture/complete"
21922 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
21923 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
21924 "200 OK",
21925 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
21926 ),
21927 "/api/worker/activity-tasks/activity-cancel/complete" => (
21928 "409 Conflict",
21929 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
21930 ),
21931 "/api/worker/activity-tasks/activity-typed/complete"
21932 | "/api/worker/activity-tasks/activity-typed/fail"
21933 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
21934 "/api/workflows/counter-1/query/current" => (
21935 "200 OK",
21936 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
21937 ),
21938 "/api/workflows/counter-1/query/missing" => (
21939 "404 Not Found",
21940 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
21941 ),
21942 "/api/workflows/wf-lifecycle/cancel" => (
21943 "200 OK",
21944 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
21945 ),
21946 "/api/workflows/wf-lifecycle/terminate" => (
21947 "200 OK",
21948 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
21949 ),
21950 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
21951 "200 OK",
21952 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
21953 ),
21954 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
21955 "200 OK",
21956 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
21957 ),
21958 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
21959 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
21960 "409 Conflict",
21961 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
21962 ),
21963 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
21964 "200 OK",
21965 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
21966 ),
21967 "/api/workflows/wf-cancelled" => (
21968 "200 OK",
21969 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
21970 ),
21971 "/api/workflows/wf-terminated" => (
21972 "200 OK",
21973 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
21974 ),
21975 "/api/workflows/wf-timed-out" => (
21976 "200 OK",
21977 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
21978 ),
21979 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
21980 "200 OK",
21981 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
21982 ),
21983 "/api/workflows/wf-selected" => (
21984 "200 OK",
21985 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
21986 ),
21987 "/api/workflows/wf-selected/runs/run-selected" => (
21988 "200 OK",
21989 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
21990 ),
21991 _ => ("404 Not Found", r#"{"message":"not found"}"#),
21992 };
21993 write_mock_response(stream, status, body);
21994 }
21995
21996 fn mock_request_is_complete(request: &[u8]) -> bool {
21997 let Some(header_end) = request
21998 .windows(4)
21999 .position(|window| window == b"\r\n\r\n")
22000 .map(|position| position + 4)
22001 else {
22002 return false;
22003 };
22004 let headers = String::from_utf8_lossy(&request[..header_end]);
22005 let content_length = headers.lines().find_map(|line| {
22006 let (name, value) = line.split_once(':')?;
22007 name.eq_ignore_ascii_case("content-length")
22008 .then(|| value.trim().parse::<usize>().ok())
22009 .flatten()
22010 });
22011
22012 request.len() >= header_end + content_length.unwrap_or(0)
22013 }
22014
22015 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
22016 let response = format!(
22017 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
22018 body.len()
22019 );
22020
22021 let _ = stream.write_all(response.as_bytes());
22022 let _ = stream.flush();
22023 }
22024}