[{"data":1,"prerenderedAt":2181},["ShallowReactive",2],{"navigation":3,"/operations/monitoring-and-debugging":879,"/operations/monitoring-and-debugging-surround":2176},[4,22,45,115,156,187,214,235,264,295,318,361,389,418,436,470,491,534,548,571,632,670,693,757,850],{"title":5,"path":6,"stem":7,"children":8,"icon":11},"Introduction","/introduction","01.introduction/1.index",[9,12,17],{"title":10,"path":6,"stem":7,"icon":11},"What is Orkestia?","i-lucide-sparkles",{"title":13,"path":14,"stem":15,"icon":16},"Core Philosophy","/introduction/core-philosophy","01.introduction/2.core-philosophy","i-lucide-shield-check",{"title":18,"path":19,"stem":20,"icon":21},"Key Benefits","/introduction/key-benefits","01.introduction/3.key-benefits","i-lucide-trophy",{"title":23,"path":24,"stem":25,"children":26,"icon":29},"Getting Started","/getting-started","02.getting-started/1.index",[27,30,35,40],{"title":28,"path":24,"stem":25,"icon":29},"Quick Start","i-lucide-rocket",{"title":31,"path":32,"stem":33,"icon":34},"Architecture Overview","/getting-started/architecture-overview","02.getting-started/2.architecture-overview","i-lucide-layout-dashboard",{"title":36,"path":37,"stem":38,"icon":39},"Concepts at a Glance","/getting-started/concepts","02.getting-started/3.concepts","i-lucide-list",{"title":41,"path":42,"stem":43,"icon":44},"Connect an AI Assistant","/getting-started/connect-an-ai-assistant","02.getting-started/4.connect-an-ai-assistant","i-lucide-bot",{"title":46,"path":47,"stem":48,"children":49,"icon":51},"Core Concepts","/concepts","03.concepts/01.index",[50,52,57,62,67,71,76,81,86,91,96,100,105,110],{"title":46,"path":47,"stem":48,"icon":51},"i-lucide-book-open",{"title":53,"path":54,"stem":55,"icon":56},"Workflows","/concepts/workflows","03.concepts/02.workflows","i-lucide-workflow",{"title":58,"path":59,"stem":60,"icon":61},"DGI — Dialog Generative Interface","/concepts/dgi","03.concepts/03.dgi","i-lucide-brain",{"title":63,"path":64,"stem":65,"icon":66},"Staff & AI Workforce Governance","/concepts/staff-governance","03.concepts/04.staff-governance","i-lucide-users-round",{"title":68,"path":69,"stem":70,"icon":44},"Agents — the execution substrate","/concepts/agents-platform","03.concepts/05.agents-platform",{"title":72,"path":73,"stem":74,"icon":75},"Runners & Execution Environments","/concepts/runners","03.concepts/06.runners","i-lucide-server-cog",{"title":77,"path":78,"stem":79,"icon":80},"Lumen Observability","/concepts/lumen","03.concepts/07.lumen","i-lucide-activity",{"title":82,"path":83,"stem":84,"icon":85},"Identity & Multi-Tenancy","/concepts/identity-multi-tenancy","03.concepts/08.identity-multi-tenancy","i-lucide-users",{"title":87,"path":88,"stem":89,"icon":90},"Billing, Pricing & Seats","/concepts/billing-and-seats","03.concepts/09.billing-and-seats","i-lucide-credit-card",{"title":92,"path":93,"stem":94,"icon":95},"App Data","/concepts/appdata","03.concepts/10.appdata","i-lucide-database",{"title":97,"path":98,"stem":99,"icon":61},"Engram","/concepts/engram","03.concepts/11.engram",{"title":101,"path":102,"stem":103,"icon":104},"DevKit","/concepts/devkit","03.concepts/12.devkit","i-lucide-terminal",{"title":106,"path":107,"stem":108,"icon":109},"App Host","/concepts/app-host","03.concepts/13.app-host","i-lucide-globe",{"title":111,"path":112,"stem":113,"icon":114},"Agent Exchange","/concepts/agent-exchange","03.concepts/14.agent-exchange","i-lucide-landmark",{"title":116,"path":117,"stem":118,"children":119,"icon":121},"Guides","/guides","04.guides/1.index",[120,122,127,132,137,142,147,151],{"title":116,"path":117,"stem":118,"icon":121},"i-lucide-compass",{"title":123,"path":124,"stem":125,"icon":126},"Building with DGI","/guides/building-with-dgi","04.guides/2.building-with-dgi","i-lucide-brain-circuit",{"title":128,"path":129,"stem":130,"icon":131},"Creating & Exposing Virtual Workflows","/guides/virtual-workflows","04.guides/4.virtual-workflows","i-lucide-blocks",{"title":133,"path":134,"stem":135,"icon":136},"Runner Management & Provisioning","/guides/runner-management","04.guides/5.runner-management","i-lucide-server",{"title":138,"path":139,"stem":140,"icon":141},"Security & Compliance","/guides/security-and-compliance","04.guides/7.security-and-compliance","i-lucide-shield",{"title":143,"path":144,"stem":145,"icon":146},"Tickets & Software Delivery","/guides/tickets-and-software-delivery","04.guides/8.tickets-and-software-delivery","i-lucide-ticket",{"title":148,"path":149,"stem":150},"Orkestia for AI-driven cloud infrastructure automation","/guides/cloud-automation-alternatives","04.guides/9.cloud-automation-alternatives",{"title":152,"path":153,"stem":154,"icon":155},"Typed decisions with TypeSafe","/guides/typed-decisions-with-typesafe","04.guides/9.typed-decisions-with-typesafe","i-lucide-git-branch",{"title":157,"path":158,"stem":159,"children":160,"icon":162},"Reference","/reference","05.reference/1.index",[161,163,168,173,177,182],{"title":157,"path":158,"stem":159,"icon":162},"i-lucide-library",{"title":164,"path":165,"stem":166,"icon":167},"Workflow Types & Registry","/reference/workflow-types-registry","05.reference/2.workflow-types-registry","i-lucide-list-tree",{"title":169,"path":170,"stem":171,"icon":172},"MCP Integration","/reference/mcp-integration","05.reference/3.mcp-integration","i-lucide-plug-zap",{"title":174,"path":175,"stem":176,"icon":104},"API & Tooling","/reference/api-tooling","05.reference/4.api-tooling",{"title":178,"path":179,"stem":180,"icon":181},"Integrations Catalog","/reference/integrations-catalog","05.reference/6.integrations-catalog","i-lucide-plug",{"title":183,"path":184,"stem":185,"icon":186},"Platform Services","/reference/platform-services","05.reference/7.platform-services","i-lucide-boxes",{"title":188,"path":189,"stem":190,"children":191,"icon":193},"Advanced Topics","/advanced","06.advanced/1.index",[192,194,199,204,209],{"title":188,"path":189,"stem":190,"icon":193},"i-lucide-flask-conical",{"title":195,"path":196,"stem":197,"icon":198},"Hybrid AI + Deterministic Execution","/advanced/hybrid-execution-model","06.advanced/2.hybrid-execution-model","i-lucide-git-merge",{"title":200,"path":201,"stem":202,"icon":203},"Drift Detection & Self-Healing","/advanced/drift-detection-self-healing","06.advanced/3.drift-detection-self-healing","i-lucide-heart-pulse",{"title":205,"path":206,"stem":207,"icon":208},"Governance & Approvals","/advanced/governance-and-approvals","06.advanced/4.governance-and-approvals","i-lucide-scale",{"title":210,"path":211,"stem":212,"icon":213},"Cost & Performance Optimization","/advanced/cost-and-performance","06.advanced/5.cost-and-performance","i-lucide-zap",{"title":215,"path":216,"stem":217,"children":218,"icon":220},"Operations","/operations","07.operations/1.index",[219,221,225,230],{"title":215,"path":216,"stem":217,"icon":220},"i-lucide-settings-2",{"title":222,"path":223,"stem":224,"icon":186},"Deployment Models","/operations/deployment-models","07.operations/2.deployment-models",{"title":226,"path":227,"stem":228,"icon":229},"Monitoring & Debugging","/operations/monitoring-and-debugging","07.operations/3.monitoring-and-debugging","i-lucide-bug",{"title":231,"path":232,"stem":233,"icon":234},"Best Practices","/operations/best-practices","07.operations/4.best-practices","i-lucide-check-check",{"title":236,"path":237,"stem":238,"children":239,"icon":241},"AWS Connections","/aws-connections","08.aws-connections/1.index",[240,242,245,250,255,260],{"title":236,"path":237,"stem":238,"icon":241},"i-simple-icons-amazonaws",{"title":23,"path":243,"stem":244,"icon":29},"/aws-connections/getting-started","08.aws-connections/2.getting-started",{"title":246,"path":247,"stem":248,"icon":249},"Setup Methods","/aws-connections/setup-methods","08.aws-connections/3.setup-methods","i-lucide-wrench",{"title":251,"path":252,"stem":253,"icon":254},"Managing Connections","/aws-connections/managing-connections","08.aws-connections/4.managing-connections","i-lucide-settings",{"title":256,"path":257,"stem":258,"icon":259},"Security Best Practices","/aws-connections/security-best-practices","08.aws-connections/5.security-best-practices","i-lucide-lock",{"title":261,"path":262,"stem":263,"icon":229},"Troubleshooting","/aws-connections/troubleshooting","08.aws-connections/6.troubleshooting",{"title":265,"path":266,"stem":267,"children":268,"icon":270},"Cloud Connections","/cloud-connections","09.cloud-connections/1.index",[269,271,276,281,286,291],{"title":265,"path":266,"stem":267,"icon":270},"i-lucide-cable",{"title":272,"path":273,"stem":274,"icon":275},"Google Cloud (GCP)","/cloud-connections/gcp","09.cloud-connections/2.gcp","i-simple-icons-googlecloud",{"title":277,"path":278,"stem":279,"icon":280},"Microsoft Azure","/cloud-connections/azure","09.cloud-connections/3.azure","i-simple-icons-microsoftazure",{"title":282,"path":283,"stem":284,"icon":285},"Magalu Cloud","/cloud-connections/magalu","09.cloud-connections/4.magalu","i-lucide-cloud",{"title":287,"path":288,"stem":289,"icon":290},"Kubernetes","/cloud-connections/kubernetes","09.cloud-connections/5.kubernetes","i-simple-icons-kubernetes",{"title":292,"path":293,"stem":294,"icon":208},"TypeSafe","/cloud-connections/typesafe","09.cloud-connections/6.typesafe",{"title":296,"path":297,"stem":298,"children":299,"icon":136},"DNS Providers","/dns-providers","10.dns-providers/1.index",[300,302,305,308,311,315],{"title":301,"path":297,"stem":298,"icon":136},"DNS Provider Connections",{"title":23,"path":303,"stem":304,"icon":29},"/dns-providers/getting-started","10.dns-providers/2.getting-started",{"title":246,"path":306,"stem":307,"icon":249},"/dns-providers/setup-methods","10.dns-providers/3.setup-methods",{"title":251,"path":309,"stem":310,"icon":254},"/dns-providers/managing-connections","10.dns-providers/4.managing-connections",{"title":312,"path":313,"stem":314,"icon":39},"Zones and Records","/dns-providers/zones-and-records","10.dns-providers/5.zones-and-records",{"title":261,"path":316,"stem":317,"icon":229},"/dns-providers/troubleshooting","10.dns-providers/6.troubleshooting",{"title":319,"path":320,"stem":321,"children":322,"icon":29},"Cloud Deploy","/cloud-deploy","11.cloud-deploy/01.index",[323,324,327,332,336,341,345,349,354,358],{"title":319,"path":320,"stem":321,"icon":29},{"title":23,"path":325,"stem":326,"icon":29},"/cloud-deploy/getting-started","11.cloud-deploy/02.getting-started",{"title":328,"path":329,"stem":330,"icon":331},"Creating a Site","/cloud-deploy/creating-a-site","11.cloud-deploy/03.creating-a-site","i-lucide-plus-circle",{"title":333,"path":334,"stem":335,"icon":34},"Site Overview and Deployments","/cloud-deploy/site-overview","11.cloud-deploy/04.site-overview",{"title":337,"path":338,"stem":339,"icon":340},"Deployment Progress and Success","/cloud-deploy/deployment-progress","11.cloud-deploy/05.deployment-progress","i-lucide-loader",{"title":342,"path":343,"stem":344,"icon":254},"Settings","/cloud-deploy/settings","11.cloud-deploy/06.settings",{"title":346,"path":347,"stem":348,"icon":109},"Custom Domains","/cloud-deploy/custom-domains","11.cloud-deploy/07.custom-domains",{"title":350,"path":351,"stem":352,"icon":353},"Resources, Releases, and Danger Zone","/cloud-deploy/resources-releases","11.cloud-deploy/08.resources-releases","i-lucide-package",{"title":355,"path":356,"stem":357,"icon":141},"Admin Dashboard","/cloud-deploy/admin-dashboard","11.cloud-deploy/09.admin-dashboard",{"title":261,"path":359,"stem":360,"icon":229},"/cloud-deploy/troubleshooting","11.cloud-deploy/10.troubleshooting",{"title":362,"path":363,"stem":364,"children":365,"icon":368},"User Onboarding","/user-onboarding","12.user-onboarding/1.index",[366,369,372,377,382,386],{"title":367,"path":363,"stem":364,"icon":368},"User Creation and Onboarding","i-lucide-user-plus",{"title":23,"path":370,"stem":371,"icon":29},"/user-onboarding/getting-started","12.user-onboarding/2.getting-started",{"title":373,"path":374,"stem":375,"icon":376},"Creating Your Organization","/user-onboarding/onboarding","12.user-onboarding/3.onboarding","i-lucide-building-2",{"title":378,"path":379,"stem":380,"icon":381},"Invitations","/user-onboarding/invitations","12.user-onboarding/4.invitations","i-lucide-mail",{"title":383,"path":384,"stem":385,"icon":254},"Managing Your Account","/user-onboarding/managing-account","12.user-onboarding/5.managing-account",{"title":261,"path":387,"stem":388,"icon":229},"/user-onboarding/troubleshooting","12.user-onboarding/6.troubleshooting",{"title":342,"path":390,"stem":391,"children":392,"icon":394},"/settings","13.settings/1.index",[393,395,400,404,409,413],{"title":342,"path":390,"stem":391,"icon":394},"i-lucide-sliders-horizontal",{"title":396,"path":397,"stem":398,"icon":399},"General Settings","/settings/general","13.settings/2.general","i-lucide-user",{"title":401,"path":402,"stem":403,"icon":85},"Members Settings","/settings/members","13.settings/3.members",{"title":405,"path":406,"stem":407,"icon":408},"Notifications Settings","/settings/notifications","13.settings/4.notifications","i-lucide-bell",{"title":410,"path":411,"stem":412,"icon":141},"Security Settings","/settings/security","13.settings/5.security",{"title":414,"path":415,"stem":416,"icon":417},"Signing Keys","/settings/keys","13.settings/6.keys","i-lucide-key-round",{"title":419,"path":420,"stem":421,"children":422,"icon":29},"App Enablement","/app-enablement","14.app-enablement/1.index",[423,424,428,432],{"title":419,"path":420,"stem":421,"icon":29},{"title":425,"path":426,"stem":427,"icon":417},"Sign in with Orkestia","/app-enablement/sign-in-with-orkestia","14.app-enablement/2.sign-in-with-orkestia",{"title":429,"path":430,"stem":431,"icon":95},"End-user data","/app-enablement/end-user-data","14.app-enablement/3.end-user-data",{"title":433,"path":434,"stem":435,"icon":131},"Compositions — use, invoke, share","/app-enablement/compositions","14.app-enablement/4.compositions",{"title":437,"path":438,"stem":439,"children":440,"icon":80},"Lumen","/lumen","15.lumen/1.index",[441,442,447,452,457,461,465],{"title":437,"path":438,"stem":439,"icon":80},{"title":443,"path":444,"stem":445,"icon":446},"Enable Lumen","/lumen/enable","15.lumen/2.enable","i-lucide-power",{"title":448,"path":449,"stem":450,"icon":451},"Send data","/lumen/send-data","15.lumen/3.send-data","i-lucide-upload",{"title":453,"path":454,"stem":455,"icon":456},"Collector","/lumen/collector","15.lumen/4.collector","i-lucide-container",{"title":458,"path":459,"stem":460,"icon":34},"Use Lumen","/lumen/observe","15.lumen/5.observe",{"title":462,"path":463,"stem":464,"icon":44},"Lumen MCP","/lumen/mcp","15.lumen/6.mcp",{"title":466,"path":467,"stem":468,"icon":469},"Query API","/lumen/query-api","15.lumen/7.query-api","i-lucide-search",{"title":471,"path":472,"stem":473,"children":474,"icon":476},"SDKs","/sdks","16.sdks/1.index",[475,477,482,487],{"title":471,"path":472,"stem":473,"icon":476},"i-lucide-code",{"title":478,"path":479,"stem":480,"icon":481},"Workflows SDK — Node / TypeScript","/sdks/workflows-nodejs","16.sdks/2.workflows-nodejs","i-lucide-file-ts",{"title":483,"path":484,"stem":485,"icon":486},"Workflows SDK — Python","/sdks/workflows-python","16.sdks/3.workflows-python","i-simple-icons-python",{"title":488,"path":489,"stem":490,"icon":417},"Auth SDK — Sign in with Orkestia","/sdks/auth","16.sdks/4.auth",{"title":92,"path":492,"stem":493,"children":494,"icon":95},"/appdata","17.appdata/1.index",[495,496,501,506,510,515,520,525,530],{"title":92,"path":492,"stem":493,"icon":95},{"title":497,"path":498,"stem":499,"icon":500},"Declare structures","/appdata/declare","17.appdata/2.declare","i-lucide-table",{"title":502,"path":503,"stem":504,"icon":505},"Records & Data API","/appdata/data-api","17.appdata/3.data-api","i-lucide-rows-3",{"title":507,"path":508,"stem":509,"icon":376},"Ownership & workspaces","/appdata/ownership","17.appdata/4.ownership",{"title":511,"path":512,"stem":513,"icon":514},"Expose App Data to end-users","/appdata/expose","17.appdata/5.expose","i-lucide-app-window",{"title":516,"path":517,"stem":518,"icon":519},"PostgREST HTTP","/appdata/postgrest","17.appdata/6.postgrest","i-lucide-unplug",{"title":521,"path":522,"stem":523,"icon":524},"Ordered append","/appdata/append","17.appdata/7.append","i-lucide-list-ordered",{"title":526,"path":527,"stem":528,"icon":529},"Databases and instances","/appdata/instances","17.appdata/8.instances","i-lucide-hard-drive",{"title":531,"path":532,"stem":533,"icon":104},"Query console and SQL","/appdata/query","17.appdata/9.query",{"title":97,"path":535,"stem":536,"children":537,"icon":61},"/engram","18.engram/1.index",[538,539,543],{"title":97,"path":535,"stem":536,"icon":61},{"title":540,"path":541,"stem":542,"icon":155},"Write & recall","/engram/write-recall","18.engram/2.write-recall",{"title":544,"path":545,"stem":546,"icon":547},"Field & feed","/engram/agent-memory","18.engram/3.agent-memory","i-lucide-radio",{"title":101,"path":549,"stem":550,"children":551,"icon":104},"/devkit","19.devkit/1.index",[552,553,558,562,567],{"title":101,"path":549,"stem":550,"icon":104},{"title":554,"path":555,"stem":556,"icon":557},"Install DevKit","/devkit/install","19.devkit/2.install","i-lucide-download",{"title":559,"path":560,"stem":561,"icon":519},"Hook redirect","/devkit/hooks","19.devkit/3.hooks",{"title":563,"path":564,"stem":565,"icon":566},"Local coding runner","/devkit/local-runner","19.devkit/4.local-runner","i-lucide-cpu",{"title":568,"path":569,"stem":570,"icon":131},"Compositions from DevKit","/devkit/compositions","19.devkit/5.compositions",{"title":572,"path":573,"stem":574,"children":575,"icon":44},"Staff & Agents","/staff-and-agents","20.staff-and-agents/01.index",[576,577,582,586,590,594,598,602,606,611,615,620,625,629],{"title":572,"path":573,"stem":574,"icon":44},{"title":578,"path":579,"stem":580,"icon":581},"Prerequisites","/staff-and-agents/prerequisites","20.staff-and-agents/02.prerequisites","i-lucide-list-checks",{"title":583,"path":584,"stem":585,"icon":368},"Hire an actor","/staff-and-agents/hire-an-actor","20.staff-and-agents/03.hire-an-actor",{"title":587,"path":588,"stem":589,"icon":34},"Console","/staff-and-agents/console","20.staff-and-agents/04.console",{"title":591,"path":592,"stem":593,"icon":11},"Configs, skills, and MCP","/staff-and-agents/configs-skills-mcp","20.staff-and-agents/05.configs-skills-mcp",{"title":595,"path":596,"stem":597,"icon":136},"Agent runner groups","/staff-and-agents/runner-groups","20.staff-and-agents/06.runner-groups",{"title":599,"path":600,"stem":601,"icon":16},"Governance","/staff-and-agents/governance","20.staff-and-agents/07.governance",{"title":603,"path":604,"stem":605,"icon":417},"Identity and tokens","/staff-and-agents/identity","20.staff-and-agents/08.identity",{"title":607,"path":608,"stem":609,"icon":610},"Memory and cost","/staff-and-agents/memory-and-cost","20.staff-and-agents/09.memory-and-cost","i-lucide-wallet",{"title":612,"path":613,"stem":614,"icon":155},"Coding agents","/staff-and-agents/coding-agents","20.staff-and-agents/10.coding-agents",{"title":616,"path":617,"stem":618,"icon":619},"Wire a repository for coding agents","/staff-and-agents/wire-a-repository","20.staff-and-agents/11.wire-a-repository","i-lucide-link",{"title":621,"path":622,"stem":623,"icon":624},"Run a ticket end to end","/staff-and-agents/run-a-ticket","20.staff-and-agents/12.run-a-ticket","i-lucide-play",{"title":626,"path":627,"stem":628,"icon":66},"Build a product team of actors","/staff-and-agents/build-a-product-team","20.staff-and-agents/13.build-a-product-team",{"title":261,"path":630,"stem":631,"icon":249},"/staff-and-agents/troubleshooting","20.staff-and-agents/14.troubleshooting",{"title":106,"path":633,"stem":634,"children":635,"icon":109},"/app-host","21.app-host/1.index",[636,637,641,644,649,653,658,662,666],{"title":106,"path":633,"stem":634,"icon":109},{"title":638,"path":639,"stem":640,"icon":514},"Your app and site","/app-host/your-app","21.app-host/2.your-app",{"title":92,"path":642,"stem":643,"icon":95},"/app-host/app-data","21.app-host/3.app-data",{"title":645,"path":646,"stem":647,"icon":648},"Website and process","/app-host/hosting","21.app-host/4.hosting","i-lucide-cloud-upload",{"title":650,"path":651,"stem":652,"icon":547},"Buzz","/app-host/buzz","21.app-host/5.buzz",{"title":654,"path":655,"stem":656,"icon":657},"Files","/app-host/files","21.app-host/6.files","i-lucide-folder",{"title":659,"path":660,"stem":661,"icon":109},"Your own domain","/app-host/your-domain","21.app-host/7.your-domain",{"title":663,"path":664,"stem":665,"icon":229},"Troubleshooting App Host","/app-host/troubleshooting","21.app-host/8.troubleshooting",{"title":667,"path":668,"stem":669,"icon":417},"Signing keys","/app-host/signing-keys","21.app-host/9.signing-keys",{"title":111,"path":671,"stem":672,"children":673,"icon":114},"/agent-exchange","22.agent-exchange/01.index",[674,675,678,682,686,690],{"title":111,"path":671,"stem":672,"icon":114},{"title":583,"path":676,"stem":677,"icon":368},"/agent-exchange/hire","22.agent-exchange/02.hire",{"title":679,"path":680,"stem":681,"icon":208},"List an actor","/agent-exchange/list","22.agent-exchange/03.list",{"title":683,"path":684,"stem":685,"icon":104},"Invoke & leases","/agent-exchange/invoke","22.agent-exchange/04.invoke",{"title":687,"path":688,"stem":689,"icon":141},"Settlement & trust","/agent-exchange/settlement","22.agent-exchange/05.settlement",{"title":53,"path":691,"stem":692,"icon":56},"/agent-exchange/workflows","22.agent-exchange/06.workflows",{"title":694,"path":695,"stem":696,"children":697,"icon":75},"Runners","/runners","23.runners/1.index",[698,700,704,709,713,717,721,726,730,734,738,741,745,749,753],{"title":699,"path":695,"stem":696,"icon":75},"Runner groups",{"title":701,"path":702,"stem":703,"icon":275},"Cloud Run","/runners/cloud-run","23.runners/10.cloud-run",{"title":705,"path":706,"stem":707,"icon":708},"DigitalOcean App Job","/runners/do-app-job","23.runners/11.do-app-job","i-simple-icons-digitalocean",{"title":710,"path":711,"stem":712,"icon":708},"DigitalOcean Droplet","/runners/do-droplet","23.runners/12.do-droplet",{"title":714,"path":715,"stem":716,"icon":285},"Magalu Cloud VM","/runners/magalu-vm","23.runners/13.magalu-vm",{"title":101,"path":718,"stem":719,"icon":720},"/runners/devkit","23.runners/14.devkit","i-lucide-laptop",{"title":722,"path":723,"stem":724,"icon":725},"Purposes & integrations","/runners/purposes","23.runners/16.purposes","i-lucide-layers",{"title":727,"path":728,"stem":729,"icon":241},"Fargate","/runners/fargate","23.runners/2.fargate",{"title":731,"path":732,"stem":733,"icon":241},"EC2 Auto Scaling","/runners/ec2-auto-scaling","23.runners/3.ec2-auto-scaling",{"title":735,"path":736,"stem":737,"icon":241},"EC2 VM","/runners/ec2-vm","23.runners/4.ec2-vm",{"title":287,"path":739,"stem":740,"icon":290},"/runners/kubernetes","23.runners/5.kubernetes",{"title":742,"path":743,"stem":744,"icon":280},"Azure Container Apps","/runners/azure-container-apps","23.runners/6.azure-container-apps",{"title":746,"path":747,"stem":748,"icon":280},"Azure VMSS","/runners/azure-vmss","23.runners/7.azure-vmss",{"title":750,"path":751,"stem":752,"icon":280},"Azure VM","/runners/azure-vm","23.runners/8.azure-vm",{"title":754,"path":755,"stem":756,"icon":275},"GCE","/runners/gce","23.runners/9.gce",{"title":758,"path":759,"stem":760,"children":761,"icon":763},"Chat","/chat","24.chat/01.index",[762,764,768,773,777,782,787,791,796,800,805,810,814,819,824,828,832,836,840,845],{"title":758,"path":759,"stem":760,"icon":763},"i-lucide-messages-square",{"title":765,"path":766,"stem":767,"icon":446},"Enable and publish","/chat/enable-and-publish","24.chat/02.enable-and-publish",{"title":769,"path":770,"stem":771,"icon":772},"Theme and customization","/chat/theme-and-customization","24.chat/03.theme-and-customization","i-lucide-palette",{"title":774,"path":775,"stem":776,"icon":85},"Members and moderation","/chat/members-and-moderation","24.chat/04.members-and-moderation",{"title":778,"path":779,"stem":780,"icon":781},"Channels","/chat/channels","24.chat/05.channels","i-lucide-hash",{"title":783,"path":784,"stem":785,"icon":786},"Using the chat","/chat/using-the-chat","24.chat/06.using-the-chat","i-lucide-message-circle",{"title":788,"path":789,"stem":790,"icon":44},"Actors in chat","/chat/actors-in-chat","24.chat/07.actors-in-chat",{"title":792,"path":793,"stem":794,"icon":795},"Internal support actor","/chat/internal-support-actor","24.chat/08.internal-support-actor","i-lucide-life-buoy",{"title":797,"path":798,"stem":799,"icon":104},"API and console","/chat/api-and-console","24.chat/09.api-and-console",{"title":801,"path":802,"stem":803,"icon":804},"Limits","/chat/limits","24.chat/10.limits","i-lucide-triangle-alert",{"title":806,"path":807,"stem":808,"icon":809},"Structured chat with DGI","/chat/structured-chat","24.chat/11.structured-chat","i-lucide-layout-list",{"title":811,"path":812,"stem":813,"icon":109},"Option A: hosted chat with DGI","/chat/option-a-hosted-chat","24.chat/12.option-a-hosted-chat",{"title":815,"path":816,"stem":817,"icon":818},"Option B: embed the chat component","/chat/option-b-embed-component","24.chat/13.option-b-embed-component","i-lucide-component",{"title":820,"path":821,"stem":822,"icon":823},"Option C: custom client (wire contract)","/chat/option-c-custom-client","24.chat/14.option-c-custom-client","i-lucide-braces",{"title":825,"path":826,"stem":827,"icon":394},"Responder configuration reference","/chat/responder-reference","24.chat/15.responder-reference",{"title":829,"path":830,"stem":831,"icon":547},"Cards, live updates and proactive posts","/chat/cards-live-and-proactive","24.chat/16.cards-live-and-proactive",{"title":833,"path":834,"stem":835,"icon":16},"Structured chat security model","/chat/structured-chat-security","24.chat/17.structured-chat-security",{"title":837,"path":838,"stem":839,"icon":181},"Option D: any app or API (dgi.chat)","/chat/option-d-chat-api","24.chat/18.option-d-chat-api",{"title":841,"path":842,"stem":843,"icon":844},"Card catalog","/chat/card-catalog","24.chat/19.card-catalog","i-lucide-layout-grid",{"title":846,"path":847,"stem":848,"icon":849},"Living Surfaces","/chat/living-surfaces","24.chat/20.living-surfaces","i-lucide-sprout",{"title":851,"path":852,"stem":853,"children":854,"icon":61},"DGI","/dgi","25.dgi/1.index",[855,857,862,866,870,874],{"title":856,"path":852,"stem":853,"icon":61},"What is DGI",{"title":858,"path":859,"stem":860,"icon":861},"How DGI works","/dgi/how-it-works","25.dgi/2.how-it-works","i-lucide-cog",{"title":863,"path":864,"stem":865,"icon":844},"Interfaces","/dgi/interfaces","25.dgi/3.interfaces",{"title":867,"path":868,"stem":869,"icon":29},"Quickstart","/dgi/quickstart","25.dgi/4.quickstart",{"title":871,"path":872,"stem":873,"icon":16},"Trust and safety","/dgi/trust-and-safety","25.dgi/5.trust-and-safety",{"title":875,"path":876,"stem":877,"icon":878},"FAQ","/dgi/faq","25.dgi/6.faq","i-lucide-circle-help",{"id":880,"title":226,"body":881,"description":2169,"extension":2170,"links":2171,"meta":2172,"navigation":2173,"path":227,"seo":2174,"stem":228,"__hash__":2175},"docs/07.operations/3.monitoring-and-debugging.md",{"type":882,"value":883,"toc":2147},"minimark",[884,906,909,942,956,961,964,969,1072,1076,1079,1114,1132,1135,1141,1161,1171,1182,1186,1200,1245,1263,1266,1322,1326,1337,1342,1361,1372,1376,1382,1398,1407,1411,1433,1494,1504,1508,1515,1531,1537,1591,1595,1631,1647,1651,1678,1682,1688,1757,1837,1864,1881,1885,1896,1900,1907,1943,1947,1955,1961,1965,2095,2099,2134,2143],[885,886,887,888,892,893,896,897,900,901,905],"p",{},"When a workflow misbehaves in production, you need a fast, repeatable path from \"something is wrong\" to \"here is the failing step, here is why, here is the fix.\" This page is that runbook. It assumes you already understand the ",[889,890,891],"a",{"href":54},"workflow engine",", ",[889,894,895],{"href":73},"runners",", and ",[889,898,899],{"href":78},"Lumen observability"," at the concept level — here we focus on the ",[902,903,904],"em",{},"operational moves",".",[885,907,908],{},"The two instruments you reach for, in order:",[910,911,912,933],"card-group",{},[913,914,916],"card",{"icon":104,"title":915,"to":170},"The workflow MCP / SDK",[885,917,918,919,892,923,892,926,892,929,932],{},"Run-state tooling — ",[920,921,922],"code",{},"watch_workflow",[920,924,925],{},"get_workflow_history",[920,927,928],{},"list_stuck_workflows",[920,930,931],{},"retry_workflow",". This is the engine's own view of what each run is doing right now.",[913,934,935],{"icon":80,"title":437,"to":78},[885,936,937,938,941],{},"Workflow-native observability — logs, error groups, traces, spans, and triage memory, all attributed by workflow, actor, and state. This is ",[902,939,940],{},"why"," a run did what the engine reports.",[943,944,945],"note",{},[885,946,947,948,952,953,955],{},"The engine answers ",[949,950,951],"strong",{},"what happened"," (which state, which transition, which error string); Lumen answers ",[949,954,940],{}," (the trace, the grouped error, the previous resolution). Most debugging sessions bounce between the two. Start with the engine for the failing step, jump to Lumen for the root cause.",[957,958,960],"h2",{"id":959},"the-debugging-decision-tree","The debugging decision tree",[885,962,963],{},"Use this to pick your first move. Each leaf links to the matching section below.",[965,966],"dag-diagram",{":edges":967,":nodes":968},"[{\"from\":\"S\",\"to\":\"A\"},{\"from\":\"S\",\"to\":\"B\"},{\"from\":\"S\",\"to\":\"C\"},{\"from\":\"S\",\"to\":\"D\"},{\"from\":\"S\",\"to\":\"E\"},{\"from\":\"A\",\"to\":\"A1\"},{\"from\":\"B\",\"to\":\"B1\"},{\"from\":\"C\",\"to\":\"C1\"},{\"from\":\"D\",\"to\":\"D1\"},{\"from\":\"E\",\"to\":\"E1\"}]","[{\"id\":\"S\",\"label\":\"What's wrong?\",\"kind\":\"start\"},{\"id\":\"A\",\"label\":\"A run is still going\",\"sub\":\"and I want to follow it\"},{\"id\":\"B\",\"label\":\"A run failed\",\"sub\":\"and ended\"},{\"id\":\"C\",\"label\":\"A run is stuck\",\"sub\":\"not progressing\"},{\"id\":\"D\",\"label\":\"Many runs are slow\",\"sub\":\"or failing at once\"},{\"id\":\"E\",\"label\":\"Runs depending on a\",\"sub\":\"connection/runner fail\"},{\"id\":\"A1\",\"label\":\"watch_workflow → live transitions\",\"kind\":\"engine\"},{\"id\":\"B1\",\"label\":\"get_workflow_history → find failed state\",\"sub\":\"then Lumen error group + trace\",\"kind\":\"engine\"},{\"id\":\"C1\",\"label\":\"list_stuck_workflows → retry_workflow\",\"sub\":\"force_terminate as last resort\",\"kind\":\"engine\"},{\"id\":\"D1\",\"label\":\"Lumen dashboard → top error groups\",\"sub\":\"latency/cost pulse\",\"kind\":\"data\"},{\"id\":\"E1\",\"label\":\"Check connection / runner health\",\"sub\":\"+ SLA Health policies\",\"kind\":\"cloud\"}]",[970,971,972,988],"table",{},[973,974,975],"thead",{},[976,977,978,982,985],"tr",{},[979,980,981],"th",{},"Symptom",[979,983,984],{},"First move",[979,986,987],{},"Section",[989,990,991,1007,1026,1044,1058],"tbody",{},[976,992,993,997,1001],{},[994,995,996],"td",{},"Run in progress, want to follow it",[994,998,999],{},[920,1000,922],{},[994,1002,1003],{},[889,1004,1006],{"href":1005},"#watch-a-live-run","Watch a live run",[976,1008,1009,1015,1020],{},[994,1010,1011,1012],{},"Run ended in ",[920,1013,1014],{},"FAILED",[994,1016,1017,1019],{},[920,1018,925],{},", then Lumen trace",[994,1021,1022],{},[889,1023,1025],{"href":1024},"#read-history-and-debug-a-failure","Read history & debug a failure",[976,1027,1028,1031,1038],{},[994,1029,1030],{},"Run hasn't moved in a while",[994,1032,1033,1035,1036],{},[920,1034,928],{}," → ",[920,1037,931],{},[994,1039,1040],{},[889,1041,1043],{"href":1042},"#find-and-recover-stuck-workflows","Find & recover stuck workflows",[976,1045,1046,1049,1052],{},[994,1047,1048],{},"Broad spike of failures/latency",[994,1050,1051],{},"Lumen dashboard → error groups",[994,1053,1054],{},[889,1055,1057],{"href":1056},"#debug-with-lumen-traces-and-logs","Debug with Lumen traces & logs",[976,1059,1060,1063,1066],{},[994,1061,1062],{},"Failures tied to a cloud/runner",[994,1064,1065],{},"Connection + runner + SLA health",[994,1067,1068],{},[889,1069,1071],{"href":1070},"#check-runner-and-sla-health","Check runner & SLA health",[957,1073,1075],{"id":1074},"identify-yourself-first","Identify yourself first",[885,1077,1078],{},"Every run is scoped to your organization, resolved server-side from your token. Before any operation, confirm who you are — this is the mandatory first step for the workflow MCP and surfaces the org your run-state queries will be scoped to.",[1080,1081,1086],"pre",{"className":1082,"code":1083,"language":1084,"meta":1085,"style":1085},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","# via the MCP tool surface (whoami)\norkestia whoami\n# → { organization_uuid: \"...\", scopes: [\"workflow:read\", \"workflow:write\", ...] }\n","bash","",[920,1087,1088,1097,1108],{"__ignoreMap":1085},[1089,1090,1093],"span",{"class":1091,"line":1092},"line",1,[1089,1094,1096],{"class":1095},"sHwdD","# via the MCP tool surface (whoami)\n",[1089,1098,1100,1104],{"class":1091,"line":1099},2,[1089,1101,1103],{"class":1102},"sBMFI","orkestia",[1089,1105,1107],{"class":1106},"sfazB"," whoami\n",[1089,1109,1111],{"class":1091,"line":1110},3,[1089,1112,1113],{"class":1095},"# → { organization_uuid: \"...\", scopes: [\"workflow:read\", \"workflow:write\", ...] }\n",[1115,1116,1117],"tip",{},[885,1118,1119,1120,1123,1124,1127,1128,1131],{},"Never pass ",[920,1121,1122],{},"organization_uuid"," in ",[920,1125,1126],{},"initial_data",". The engine derives it from your token and scopes every list/watch/retry call automatically. If ",[920,1129,1130],{},"whoami"," shows the wrong org or missing scopes, fix auth before debugging — you may simply be looking at the wrong tenant's runs.",[957,1133,1006],{"id":1134},"watch-a-live-run",[885,1136,1137,1138,1140],{},"When a run is in progress and you want to follow its state machine in real time, ",[920,1139,922],{}," streams transitions as they happen. This is the cheapest way to confirm a long-running DAG is advancing rather than wedged.",[1080,1142,1144],{"className":1082,"code":1143,"language":1084,"meta":1085,"style":1085},"watch_workflow(workflow_id=\"wf_01J...\")\n",[920,1145,1146],{"__ignoreMap":1085},[1089,1147,1148,1151,1154,1157],{"class":1091,"line":1092},[1089,1149,1150],{"class":1102},"watch_workflow(workflow_id",[1089,1152,1153],{"class":1106},"=",[1089,1155,1156],{"class":1102},"\"wf_01J...\"",[1089,1158,1160],{"class":1159},"sTEyZ",")\n",[885,1162,1163,1164,1167,1168,905],{},"You'll see each transition as the engine commits it — state name, the transition that fired, and any emitted output. For a ",[889,1165,1166],{"href":129},"virtual workflow / composition"," you'll see the constituent steps advance in dependency order. If the stream goes quiet for longer than the step's expected duration, the run is a stuck-workflow candidate — jump to ",[889,1169,1170],{"href":1042},"recovery",[943,1172,1173],{},[885,1174,1175,1176,1178,1179,905],{},"Watching is read-only and safe to leave running. It reflects committed engine state (the engine uses Postgres advisory locks and an event-sourced log), so what you see is the durable truth, not an optimistic UI guess. A pod restart or HPA scale event will not lose the run — ",[920,1177,922],{}," re-attaches to the same ",[920,1180,1181],{},"workflow_id",[957,1183,1185],{"id":1184},"read-history-and-debug-a-failure","Read history and debug a failure",[885,1187,1188,1189,1192,1193,1195,1196,1199],{},"When a run has ",[902,1190,1191],{},"ended"," in a failure, ",[920,1194,925],{}," gives you the full ordered transition log — every state the run entered, the transition that moved it, timestamps, and the error captured at the failing state. Pair it with ",[920,1197,1198],{},"get_workflow_status"," for the one-line current verdict.",[1080,1201,1203],{"className":1082,"code":1202,"language":1084,"meta":1085,"style":1085},"get_workflow_status(workflow_id=\"wf_01J...\")\n# → { state: \"FAILED\", failed_at: \"...\", error: \"...\" }\n\nget_workflow_history(workflow_id=\"wf_01J...\")\n# → ordered list of { state, transition, entered_at, output | error }\n",[920,1204,1205,1216,1221,1227,1239],{"__ignoreMap":1085},[1089,1206,1207,1210,1212,1214],{"class":1091,"line":1092},[1089,1208,1209],{"class":1102},"get_workflow_status(workflow_id",[1089,1211,1153],{"class":1106},[1089,1213,1156],{"class":1102},[1089,1215,1160],{"class":1159},[1089,1217,1218],{"class":1091,"line":1099},[1089,1219,1220],{"class":1095},"# → { state: \"FAILED\", failed_at: \"...\", error: \"...\" }\n",[1089,1222,1223],{"class":1091,"line":1110},[1089,1224,1226],{"emptyLinePlaceholder":1225},true,"\n",[1089,1228,1230,1233,1235,1237],{"class":1091,"line":1229},4,[1089,1231,1232],{"class":1102},"get_workflow_history(workflow_id",[1089,1234,1153],{"class":1106},[1089,1236,1156],{"class":1102},[1089,1238,1160],{"class":1159},[1089,1240,1242],{"class":1091,"line":1241},5,[1089,1243,1244],{"class":1095},"# → ordered list of { state, transition, entered_at, output | error }\n",[885,1246,1247,1248,1251,1252,1254,1255,1258,1259,1262],{},"Read the history bottom-up: the ",[949,1249,1250],{},"last"," non-terminal state before ",[920,1253,1014],{}," is where the work actually broke. Atomic workflows follow the 3-state pattern — ",[920,1256,1257],{},"PENDING → COMPLETED | FAILED"," — so a failure is unambiguous about which operation failed. For a DAG, the history tells you which node failed and which upstream nodes had already completed (so you know what ",[902,1260,1261],{},"not"," to redo).",[885,1264,1265],{},"Once you have the failing step and its error string, cross to Lumen for the root cause.",[1080,1267,1271],{"className":1268,"code":1269,"language":1270,"meta":1085,"style":1085},"language-mermaid shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","sequenceDiagram\n  participant You\n  participant Engine as Workflow engine\n  participant Lumen\n  You->>Engine: get_workflow_history(workflow_id)\n  Engine-->>You: failing state + error string\n  You->>Lumen: search logs by workflow_id / trace_id\n  Lumen-->>You: grouped error + trace + prior resolution\n  You->>Engine: retry_workflow(workflow_id)\n","mermaid",[920,1272,1273,1278,1283,1288,1293,1298,1304,1310,1316],{"__ignoreMap":1085},[1089,1274,1275],{"class":1091,"line":1092},[1089,1276,1277],{},"sequenceDiagram\n",[1089,1279,1280],{"class":1091,"line":1099},[1089,1281,1282],{},"  participant You\n",[1089,1284,1285],{"class":1091,"line":1110},[1089,1286,1287],{},"  participant Engine as Workflow engine\n",[1089,1289,1290],{"class":1091,"line":1229},[1089,1291,1292],{},"  participant Lumen\n",[1089,1294,1295],{"class":1091,"line":1241},[1089,1296,1297],{},"  You->>Engine: get_workflow_history(workflow_id)\n",[1089,1299,1301],{"class":1091,"line":1300},6,[1089,1302,1303],{},"  Engine-->>You: failing state + error string\n",[1089,1305,1307],{"class":1091,"line":1306},7,[1089,1308,1309],{},"  You->>Lumen: search logs by workflow_id / trace_id\n",[1089,1311,1313],{"class":1091,"line":1312},8,[1089,1314,1315],{},"  Lumen-->>You: grouped error + trace + prior resolution\n",[1089,1317,1319],{"class":1091,"line":1318},9,[1089,1320,1321],{},"  You->>Engine: retry_workflow(workflow_id)\n",[957,1323,1325],{"id":1324},"find-and-recover-stuck-workflows","Find and recover stuck workflows",[885,1327,1328,1329,1332,1333,1336],{},"A ",[902,1330,1331],{},"stuck"," workflow is one that has stopped progressing without reaching a terminal state — typically an ",[920,1334,1335],{},"auto_advance"," step that never fired, a step waiting on a dependency that will never arrive, or a worker that died mid-step. The engine surfaces these directly.",[1338,1339,1341],"h3",{"id":1340},"step-1-list-them","Step 1 — List them",[1080,1343,1345],{"className":1082,"code":1344,"language":1084,"meta":1085,"style":1085},"list_stuck_workflows()\n# → [ { workflow_id, workflow_type, current_state, stuck_since, ... }, ... ]\n",[920,1346,1347,1356],{"__ignoreMap":1085},[1089,1348,1349,1352],{"class":1091,"line":1092},[1089,1350,928],{"class":1351},"s2Zo4",[1089,1353,1355],{"class":1354},"sMK4o","()\n",[1089,1357,1358],{"class":1091,"line":1099},[1089,1359,1360],{"class":1095},"# → [ { workflow_id, workflow_type, current_state, stuck_since, ... }, ... ]\n",[885,1362,1363,1364,1367,1368,1371],{},"This returns runs the engine considers stalled for your org. Triage by ",[920,1365,1366],{},"workflow_type"," and ",[920,1369,1370],{},"stuck_since"," — a cluster of the same type stuck at the same state points to a systemic cause (bad input shape, a downstream connection outage) rather than one unlucky run.",[1338,1373,1375],{"id":1374},"step-2-retry","Step 2 — Retry",[885,1377,1378,1379,1381],{},"For the common case — a transient failure (rate limit, brief connection blip, a runner that was reaped mid-job) — retry resumes the run from where it failed rather than starting over. Because the engine is event-sourced, retry replays from the last committed state, so already-completed steps are ",[949,1380,1261],{}," repeated.",[1080,1383,1385],{"className":1082,"code":1384,"language":1084,"meta":1085,"style":1085},"retry_workflow(workflow_id=\"wf_01J...\")\n",[920,1386,1387],{"__ignoreMap":1085},[1089,1388,1389,1392,1394,1396],{"class":1091,"line":1092},[1089,1390,1391],{"class":1102},"retry_workflow(workflow_id",[1089,1393,1153],{"class":1106},[1089,1395,1156],{"class":1102},[1089,1397,1160],{"class":1159},[1115,1399,1400],{},[885,1401,1402,1403,1406],{},"Before retrying ",[902,1404,1405],{},"en masse",", fix the underlying cause. Retrying a run whose input is malformed, or whose dependency is still down, just re-queues the same failure. Confirm the root cause in Lumen first (next section), then retry.",[1338,1408,1410],{"id":1409},"step-2b-resolve-a-remediation-gate","Step 2b — Resolve a remediation gate",[885,1412,1413,1414,1417,1418,1421,1422,1425,1426,1429,1430,1432],{},"Not every non-terminal run is stuck. A DAG step that fails on a ",[949,1415,1416],{},"fixable precondition"," parks the run in ",[920,1419,1420],{},"remediation_pending"," instead of compensating — the run is ",[902,1423,1424],{},"waiting for you",". Its ",[920,1427,1428],{},"state_data.remediation"," envelope (read it via ",[920,1431,1198],{},") names exactly what to fix, typically a fix workflow and its inputs. Apply the fix, then:",[1080,1434,1436],{"className":1082,"code":1435,"language":1084,"meta":1085,"style":1085},"resolve_workflow(workflow_id=\"wf_01J...\", resolution=\"remediated\")\n# the engine re-runs ONLY the failed step; completed steps keep their outputs\nresolve_workflow(workflow_id=\"wf_01J...\", resolution=\"denied\")\n# no fix — deferred compensation runs and the run fails terminally\n",[920,1437,1438,1463,1468,1489],{"__ignoreMap":1085},[1089,1439,1440,1443,1445,1447,1450,1453,1456,1459,1461],{"class":1091,"line":1092},[1089,1441,1442],{"class":1102},"resolve_workflow(workflow_id",[1089,1444,1153],{"class":1106},[1089,1446,1156],{"class":1102},[1089,1448,1449],{"class":1102},",",[1089,1451,1452],{"class":1106}," resolution=",[1089,1454,1455],{"class":1354},"\"",[1089,1457,1458],{"class":1106},"remediated",[1089,1460,1455],{"class":1354},[1089,1462,1160],{"class":1159},[1089,1464,1465],{"class":1091,"line":1099},[1089,1466,1467],{"class":1095},"# the engine re-runs ONLY the failed step; completed steps keep their outputs\n",[1089,1469,1470,1472,1474,1476,1478,1480,1482,1485,1487],{"class":1091,"line":1110},[1089,1471,1442],{"class":1102},[1089,1473,1153],{"class":1106},[1089,1475,1156],{"class":1102},[1089,1477,1449],{"class":1102},[1089,1479,1452],{"class":1106},[1089,1481,1455],{"class":1354},[1089,1483,1484],{"class":1106},"denied",[1089,1486,1455],{"class":1354},[1089,1488,1160],{"class":1159},[1089,1490,1491],{"class":1091,"line":1229},[1089,1492,1493],{"class":1095},"# no fix — deferred compensation runs and the run fails terminally\n",[885,1495,1496,1497,1500,1501,905],{},"Find gated runs with ",[920,1498,1499],{},"list_workflows"," filtered to ",[920,1502,1503],{},"state_name=\"remediation_pending\"",[1338,1505,1507],{"id":1506},"step-3-force-terminate-last-resort","Step 3 — Force-terminate (last resort)",[885,1509,1510,1511,1514],{},"If a run is genuinely unrecoverable — corrupt input, a step waiting on something that no longer exists, or a duplicate you need to clear — ",[920,1512,1513],{},"force_terminate_workflow"," drives it to a terminal state so it stops consuming queue and lock resources.",[1080,1516,1518],{"className":1082,"code":1517,"language":1084,"meta":1085,"style":1085},"force_terminate_workflow(workflow_id=\"wf_01J...\")\n",[920,1519,1520],{"__ignoreMap":1085},[1089,1521,1522,1525,1527,1529],{"class":1091,"line":1092},[1089,1523,1524],{"class":1102},"force_terminate_workflow(workflow_id",[1089,1526,1153],{"class":1106},[1089,1528,1156],{"class":1102},[1089,1530,1160],{"class":1159},[1532,1533,1534],"warning",{},[885,1535,1536],{},"Force-termination is destructive: it ends the run without completing remaining work and without compensating already-completed steps. Use it only after retry has failed and you've confirmed the run can't be salvaged. For workflows with side effects in your cloud, verify what already executed (via history) before terminating, so you know what state your account is left in.",[970,1538,1539,1552],{},[973,1540,1541],{},[976,1542,1543,1546,1549],{},[979,1544,1545],{},"Tool",[979,1547,1548],{},"Use when",[979,1550,1551],{},"Reversible?",[989,1553,1554,1566,1579],{},[976,1555,1556,1560,1563],{},[994,1557,1558],{},[920,1559,931],{},[994,1561,1562],{},"Transient failure; want to resume from the failed step",[994,1564,1565],{},"Yes — re-runnable",[976,1567,1568,1573,1576],{},[994,1569,1570],{},[920,1571,1572],{},"resolve_workflow",[994,1574,1575],{},"Run parked on the remediation gate; fix applied (or denied)",[994,1577,1578],{},"remediated resumes; denied compensates + fails",[976,1580,1581,1585,1588],{},[994,1582,1583],{},[920,1584,1513],{},[994,1586,1587],{},"Unrecoverable run; need to free resources",[994,1589,1590],{},"No — terminal",[957,1592,1594],{"id":1593},"debug-with-lumen-traces-and-logs","Debug with Lumen traces and logs",[885,1596,1597,1599,1600,1603,1604,1607,1608,892,1610,892,1613,892,1616,1619,1620,1623,1624,892,1626,1628,1629,905],{},[889,1598,437],{"href":78}," is a ",[949,1601,1602],{},"separate API"," (",[920,1605,1606],{},"https://lumen-api.orkestia.dev","). Filter on fields you put on ingest (",[920,1609,1181],{},[920,1611,1612],{},"trace_id",[920,1614,1615],{},"project",[920,1617,1618],{},"level",", …) — they are query params on ",[920,1621,1622],{},"GET /api/logs",", not reconstructed later. Contract: ",[889,1625,448],{"href":449},[889,1627,466],{"href":467},". App mapping: ",[889,1630,458],{"href":459},[1532,1632,1633],{},[885,1634,1635,1636,1639,1640,1642,1643,1646],{},"Lumen is ",[949,1637,1638],{},"alpha",". The self-serve onboarding/control plane (org provisioning, plan, retention) ships, but standalone packaging is still maturing, and trace fidelity depends on workflows emitting ",[920,1641,1612],{},"/",[920,1644,1645],{},"span_id",". Where those are missing, Lumen shows pseudo-traces grouped by workflow/error/fingerprint rather than true distributed spans.",[1338,1648,1650],{"id":1649},"the-triage-path","The triage path",[910,1652,1653,1659,1665,1672],{},[913,1654,1656],{"icon":34,"title":1655,"to":459},"1. Dashboard",[885,1657,1658],{},"Start at the project dashboard: top error groups, event feed, log pulse, and latency/cost signals. A broad spike shows here first.",[913,1660,1662],{"icon":725,"title":1661,"to":459},"2. Error groups",[885,1663,1664],{},"Failures are fingerprinted and grouped. Open the group to see occurrence count, severity, affected workflows, and whether it's new, regressed, or known.",[913,1666,1669],{"icon":1667,"title":1668,"to":459},"i-lucide-git-fork","3. Traces & spans",[885,1670,1671],{},"Drill into the trace timeline for the failing run — slow spans, error spans, and latency stats reveal whether the failure is logic, a dependency, or a timeout.",[913,1673,1675],{"icon":61,"title":1674,"to":459},"4. Triage memory",[885,1676,1677],{},"Lumen surfaces previous resolutions for similar fingerprints. If this error has been fixed before, you reuse the fix instead of re-diagnosing.",[1338,1679,1681],{"id":1680},"connecting-the-engine-to-lumen","Connecting the engine to Lumen",[885,1683,1684,1685,1687],{},"The link between the two instruments is identity. Carry these from ",[920,1686,925],{}," into Lumen's log filters:",[970,1689,1690,1703],{},[973,1691,1692],{},[976,1693,1694,1697,1700],{},[979,1695,1696],{},"From the engine",[979,1698,1699],{},"Filter in Lumen",[979,1701,1702],{},"Gets you",[989,1704,1705,1717,1730,1745],{},[976,1706,1707,1711,1714],{},[994,1708,1709],{},[920,1710,1181],{},[994,1712,1713],{},"log search by workflow",[994,1715,1716],{},"Every log line that run emitted",[976,1718,1719,1724,1727],{},[994,1720,1721,1723],{},[920,1722,1612],{}," (if emitted)",[994,1725,1726],{},"trace view",[994,1728,1729],{},"The end-to-end span timeline",[976,1731,1732,1739,1742],{},[994,1733,1734,1735,1738],{},"failing ",[920,1736,1737],{},"state"," name",[994,1740,1741],{},"log filter by state + error fingerprint",[994,1743,1744],{},"The grouped error and its history",[976,1746,1747,1751,1754],{},[994,1748,1749],{},[920,1750,1366],{},[994,1752,1753],{},"error groups by type",[994,1755,1756],{},"Whether this is one run or a fleet-wide pattern",[1080,1758,1760],{"className":1082,"code":1759,"language":1084,"meta":1085,"style":1085},"curl -sG \"https://lumen-api.orkestia.dev/api/logs\" \\\n  --data-urlencode \"workflow_id=\u003Cuuid-from-the-run>\" \\\n  --data-urlencode \"level=ERROR\" \\\n  --data-urlencode \"processed=false\" \\\n  -H \"Authorization: Bearer $ORKESTIA_TOKEN\"\n",[920,1761,1762,1781,1795,1808,1821],{"__ignoreMap":1085},[1089,1763,1764,1767,1770,1773,1776,1778],{"class":1091,"line":1092},[1089,1765,1766],{"class":1102},"curl",[1089,1768,1769],{"class":1106}," -sG",[1089,1771,1772],{"class":1354}," \"",[1089,1774,1775],{"class":1106},"https://lumen-api.orkestia.dev/api/logs",[1089,1777,1455],{"class":1354},[1089,1779,1780],{"class":1159}," \\\n",[1089,1782,1783,1786,1788,1791,1793],{"class":1091,"line":1099},[1089,1784,1785],{"class":1106},"  --data-urlencode",[1089,1787,1772],{"class":1354},[1089,1789,1790],{"class":1106},"workflow_id=\u003Cuuid-from-the-run>",[1089,1792,1455],{"class":1354},[1089,1794,1780],{"class":1159},[1089,1796,1797,1799,1801,1804,1806],{"class":1091,"line":1110},[1089,1798,1785],{"class":1106},[1089,1800,1772],{"class":1354},[1089,1802,1803],{"class":1106},"level=ERROR",[1089,1805,1455],{"class":1354},[1089,1807,1780],{"class":1159},[1089,1809,1810,1812,1814,1817,1819],{"class":1091,"line":1229},[1089,1811,1785],{"class":1106},[1089,1813,1772],{"class":1354},[1089,1815,1816],{"class":1106},"processed=false",[1089,1818,1455],{"class":1354},[1089,1820,1780],{"class":1159},[1089,1822,1823,1826,1828,1831,1834],{"class":1091,"line":1241},[1089,1824,1825],{"class":1106},"  -H",[1089,1827,1772],{"class":1354},[1089,1829,1830],{"class":1106},"Authorization: Bearer ",[1089,1832,1833],{"class":1159},"$ORKESTIA_TOKEN",[1089,1835,1836],{"class":1354},"\"\n",[885,1838,1839,1840,1603,1843,1642,1846,1849,1850,1853,1854,1603,1856,1859,1860,1863],{},"Default window is the last ",[949,1841,1842],{},"24h",[920,1844,1845],{},"from",[920,1847,1848],{},"to"," to widen, max 90 days). ",[920,1851,1852],{},"limit"," max 500. Agents use ",[889,1855,462],{"href":463},[920,1857,1858],{},"mcp-lumen.orkestia.dev",") — not the workflow MCP. ",[920,1861,1862],{},"lumk_"," read keys are GET-only; resolve/rules need an org token.",[1115,1865,1866],{},[885,1867,1868,1869,1872,1873,1876,1877,1880],{},"Drop/sample ",[949,1870,1871],{},"ingest rules"," can hide signal before it reaches the dashboard, and over-broad ",[949,1874,1875],{},"fingerprint rules"," can merge unrelated failures. If a failure you can see in engine history has ",[902,1878,1879],{},"no"," matching Lumen logs, check the project's ingest rules before assuming the instrumentation is broken.",[957,1882,1884],{"id":1883},"check-runner-and-sla-health","Check runner and SLA health",[885,1886,1887,1888,1891,1892,1895],{},"When failures cluster around a particular cloud, region, or CI pipeline, the problem is often ",[902,1889,1890],{},"below"," the workflow — a degraded ",[889,1893,1894],{"href":73},"runner group"," or a drifted resource, not the workflow logic itself.",[1338,1897,1899],{"id":1898},"runner-health","Runner health",[885,1901,1902,1903,1906],{},"Runner compute lives in ",[949,1904,1905],{},"your"," cloud; Orkestia is the control plane that provisions, registers, scales, and reaps it. When runner-dependent workflows fail:",[1908,1909,1910,1929,1936],"ul",{},[1911,1912,1913,1914,1917,1918,1921,1922,1925,1926,905],"li",{},"Confirm the runner group is ",[920,1915,1916],{},"active"," (not stuck in ",[920,1919,1920],{},"provisioning"," or silently ",[920,1923,1924],{},"archived",") — see ",[889,1927,1928],{"href":134},"Runner management",[1911,1930,1931,1932,1935],{},"Check whether ",[889,1933,1934],{"href":201},"drift detection"," has flagged the group; a runner that GitHub no longer sees, or compute that was deleted out-of-band in your account, manifests as workflows that can't acquire a runner.",[1911,1937,1938,1939,1942],{},"Verify the underlying ",[889,1940,1941],{"href":237},"cloud connection"," still grants the trust Orkestia needs to mint registration tokens and scale.",[1338,1944,1946],{"id":1945},"sla-health","SLA health",[943,1948,1949],{},[885,1950,1951,1954],{},[949,1952,1953],{},"SLA Health (Verifiable Health)"," is a preview surface. It defines policies for uptime, latency, and error-rate over evaluation windows (5m, 10m, 1h, monthly) and turns observed health into shareable, tamper-evident reports. The backing metrics-ingestion and proof path are on the roadmap, so treat it as a preview of that experience rather than a production health source today.",[885,1956,1957,1958,1960],{},"For live health signals, rely on ",[949,1959,437],{}," (latency/error pulse, slow-span p95/p99, error-group counts), which is the production observability surface.",[957,1962,1964],{"id":1963},"quick-reference","Quick reference",[970,1966,1967,1979],{},[973,1968,1969],{},[976,1970,1971,1974,1976],{},[979,1972,1973],{},"Goal",[979,1975,1545],{},[979,1977,1978],{},"Notes",[989,1980,1981,1993,2005,2017,2029,2043,2055,2067,2079],{},[976,1982,1983,1986,1990],{},[994,1984,1985],{},"Confirm identity & org",[994,1987,1988],{},[920,1989,1130],{},[994,1991,1992],{},"Mandatory first step",[976,1994,1995,1998,2002],{},[994,1996,1997],{},"Follow a live run",[994,1999,2000],{},[920,2001,922],{},[994,2003,2004],{},"Read-only, re-attachable",[976,2006,2007,2010,2014],{},[994,2008,2009],{},"One-line verdict",[994,2011,2012],{},[920,2013,1198],{},[994,2015,2016],{},"Current state + error",[976,2018,2019,2022,2026],{},[994,2020,2021],{},"Full transition log",[994,2023,2024],{},[920,2025,925],{},[994,2027,2028],{},"Read bottom-up to the failing step",[976,2030,2031,2034,2038],{},[994,2032,2033],{},"Find stalled runs",[994,2035,2036],{},[920,2037,928],{},[994,2039,2040,2041],{},"Triage by type + ",[920,2042,1370],{},[976,2044,2045,2048,2052],{},[994,2046,2047],{},"Resume a failed run",[994,2049,2050],{},[920,2051,931],{},[994,2053,2054],{},"Replays from last committed state",[976,2056,2057,2060,2064],{},[994,2058,2059],{},"Resume a gated run",[994,2061,2062],{},[920,2063,1572],{},[994,2065,2066],{},"After fixing the named precondition",[976,2068,2069,2072,2076],{},[994,2070,2071],{},"Kill an unrecoverable run",[994,2073,2074],{},[920,2075,1513],{},[994,2077,2078],{},"Destructive, no compensation",[976,2080,2081,2084,2087],{},[994,2082,2083],{},"Root-cause a failure",[994,2085,2086],{},"Lumen logs / error groups / traces",[994,2088,2089,2090,2092,2093],{},"Filter by ",[920,2091,1181],{}," / ",[920,2094,1612],{},[957,2096,2098],{"id":2097},"where-to-go-next","Where to go next",[910,2100,2101,2107,2112,2117,2123],{},[913,2102,2104],{"icon":80,"title":2103,"to":78},"Lumen concepts",[885,2105,2106],{},"How workflow-native observability is structured — logs, error groups, traces, triage memory.",[913,2108,2109],{"icon":469,"title":466,"to":467},[885,2110,2111],{},"Filters, mutations, rule JSON.",[913,2113,2114],{"icon":34,"title":458,"to":459},[885,2115,2116],{},"App routes mapped to the same endpoints.",[913,2118,2120],{"icon":249,"title":2119,"to":201},"Drift detection & self-healing",[885,2121,2122],{},"How Orkestia detects and repairs resources that drift out of declared state.",[913,2124,2126],{"icon":181,"title":2125,"to":170},"MCP integration",[885,2127,2128,2129,892,2131,2133],{},"The run-state tool surface (",[920,2130,922],{},[920,2132,931],{},", …) and how agents call it.",[885,2135,2136,2137,905],{},"For per-workflow detail — exact inputs, states, and recovery semantics of any individual workflow type — consult the live catalog at ",[889,2138,2142],{"href":2139,"rel":2140},"https://reference.orkestia.dev",[2141],"nofollow","reference.orkestia.dev",[2144,2145,2146],"style",{},"html pre.shiki code .sHwdD, html code.shiki .sHwdD{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#546E7A;--shiki-default-font-style:italic;--shiki-dark:#676E95;--shiki-dark-font-style:italic}html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}html pre.shiki code .s2Zo4, html code.shiki .s2Zo4{--shiki-light:#6182B8;--shiki-default:#82AAFF;--shiki-dark:#82AAFF}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}",{"title":1085,"searchDepth":1092,"depth":1099,"links":2148},[2149,2150,2151,2152,2153,2159,2163,2167,2168],{"id":959,"depth":1099,"text":960},{"id":1074,"depth":1099,"text":1075},{"id":1134,"depth":1099,"text":1006},{"id":1184,"depth":1099,"text":1185},{"id":1324,"depth":1099,"text":1325,"children":2154},[2155,2156,2157,2158],{"id":1340,"depth":1110,"text":1341},{"id":1374,"depth":1110,"text":1375},{"id":1409,"depth":1110,"text":1410},{"id":1506,"depth":1110,"text":1507},{"id":1593,"depth":1099,"text":1594,"children":2160},[2161,2162],{"id":1649,"depth":1110,"text":1650},{"id":1680,"depth":1110,"text":1681},{"id":1883,"depth":1099,"text":1884,"children":2164},[2165,2166],{"id":1898,"depth":1110,"text":1899},{"id":1945,"depth":1110,"text":1946},{"id":1963,"depth":1099,"text":1964},{"id":2097,"depth":1099,"text":2098},"An operational runbook for watching runs, reading history, recovering stuck workflows, and tracing failures with Lumen","md",null,{},{"icon":229},{"title":226,"description":2169},"cDuRbCMGIAI4So4c_QXJMYdLnBcqqpCRIC5ksYn9v9Q",[2177,2179],{"title":222,"path":223,"stem":224,"description":2178,"icon":186,"children":-1},"How Orkestia splits a managed control plane from execution that runs entirely inside your own cloud accounts",{"title":231,"path":232,"stem":233,"description":2180,"icon":234,"children":-1},"A checklist-driven field guide for running Orkestia in production — idempotent workflows, least-privilege roles, governed AI actors, and right-sized runners",1790354046224]