[ci] Fix nextjs lazy disocvery impacting e2e tests, disable experimental DurableAgent tests (#1400)

Author: VaguelySerious

.changeset/strong-dryers-begin.md

@@ -0,0 +1,6 @@
+---
+"@workflow/core": patch
+"@workflow/next": patch
+---
+
+Seed lazy workflow file discovery in NextJS. Require workflow definitions to be in manifest for Vercel environments.

package.json

@@ -33,7 +33,7 @@
     "test": "turbo test",
     "clean": "turbo clean",
     "typecheck": "turbo typecheck",
-    "test:e2e": "vitest run packages/core/e2e/e2e.test.ts packages/core/e2e/e2e-agent.test.ts",
+    "test:e2e": "vitest run packages/core/e2e/e2e.test.ts",
     "test:e2e:nextjs-webpack:staged": "node scripts/test-staged-nextjs-webpack.mjs",
     "test:docs": "pnpm --filter @workflow/docs-typecheck test:docs",
     "bench": "vitest bench packages/core/e2e/bench.bench.ts",

packages/core/e2e/utils.ts

@@ -388,9 +388,23 @@ export async function getWorkflowMetadata(
     await sleep(manifestRetryIntervalMs);
   }
 
-  // Deferred discovery can lag behind manifest publication in staged/out-of-
-  // monorepo tests. Fall back to the deterministic workflow ID format used by
-  // the transform so tests can continue exercising runtime behavior.
+  // For Vercel deployments, the workflow must be in the manifest. A missing
+  // workflow means the deployment's build didn't include it, so a fallback ID
+  // would just create a run that never executes and times out silently.
+  if (!isLocalDeployment()) {
+    const availableWorkflows = Object.entries(manifest.workflows)
+      .flatMap(([file, fns]) => Object.keys(fns).map((fn) => `${file}:${fn}`))
+      .join(', ');
+    throw new Error(
+      `Workflow "${workflowFn}" not found in manifest for "${workflowFile}" ` +
+        `after ${manifestRetryTimeoutMs}ms. The deployment may not include this workflow. ` +
+        `Available workflows: ${availableWorkflows || '(none)'}`
+    );
+  }
+
+  // For local development, fall back to the deterministic workflow ID format
+  // used by the transform. Deferred discovery can lag behind manifest
+  // publication in staged/out-of-monorepo tests.
   const fallbackWorkflowId = getFallbackWorkflowId(workflowFile, workflowFn);
   console.warn(
     `Workflow "${workflowFn}" not found in manifest for "${workflowFile}" after ${manifestRetryTimeoutMs}ms; ` +

packages/next/src/builder-deferred.ts

@@ -661,9 +661,54 @@ export async function getNextBuilderDeferred() {
       }
 
       await this.loadWorkflowsCache();
+
+      // The cache only contains files discovered in previous builds. On the
+      // first build after adding new workflow files, those files won't be in
+      // the cache. In production builds (`watch: false`), the loader's socket
+      // notifications arrive too late — `scheduleDeferredRebuild()` is a no-op
+      // so newly discovered files never trigger a rebuild.
+      //
+      // To fix this, eagerly scan the dirs for files with workflow/step
+      // directives and seed the discovered sets before the build runs.
+      await this.seedDiscoveredFilesFromDirs();
+
       this.cacheInitialized = true;
     }
 
+    /**
+     * Scans the configured dirs for files containing `'use workflow'` or
+     * `'use step'` directives and adds them to the discovered sets. This
+     * ensures new workflow files are included in the very first production
+     * build, without waiting for the loader's async socket notifications.
+     */
+    private async seedDiscoveredFilesFromDirs(): Promise<void> {
+      // Use the base builder's file scanning (which searches all TS/JS files
+      // in dirs, not just entrypoints). Call the base class version directly
+      // to bypass the deferred builder's entrypoint-only filter.
+      const allInputFiles = await this.getAllDirFiles();
+
+      await Promise.all(
+        allInputFiles.map(async (filePath) => {
+          try {
+            const source = await readFile(filePath, 'utf-8');
+            const patterns = detectWorkflowPatterns(source);
+
+            if (patterns.hasUseWorkflow) {
+              this.discoveredWorkflowFiles.add(filePath);
+            }
+            if (patterns.hasUseStep) {
+              this.discoveredStepFiles.add(filePath);
+            }
+            if (patterns.hasSerde && !isWorkflowSdkFile(filePath)) {
+              this.discoveredSerdeFiles.add(filePath);
+            }
+          } catch {
+            // File may have been deleted between glob and read — skip it.
+          }
+        })
+      );
+    }
+
     private getDistDir(): string {
       return (this.config as { distDir?: string }).distDir || '.next';
     }
@@ -1000,6 +1045,15 @@ export async function getNextBuilderDeferred() {
       );
     }
 
+    /**
+     * Returns ALL files from the configured dirs without filtering to
+     * entrypoints. Used by `seedDiscoveredFilesFromDirs` to scan for
+     * workflow/step directives on first build.
+     */
+    private async getAllDirFiles(): Promise<string[]> {
+      return super.getInputFiles();
+    }
+
     protected async getInputFiles(): Promise<string[]> {
       const inputFiles = await super.getInputFiles();
       return inputFiles.filter((item) => {

packages/next/src/index.ts

@@ -86,8 +86,15 @@ export function withWorkflow(
           const NextBuilder = await getNextBuilder(nextVersion);
           return new NextBuilder({
             watch: shouldWatch,
-            // discover workflows from pages/app entries
-            dirs: ['pages', 'app', 'src/pages', 'src/app'],
+            // discover workflows from pages/app entries and common workflow dirs
+            dirs: [
+              'pages',
+              'app',
+              'src/pages',
+              'src/app',
+              'workflows',
+              'src/workflows',
+            ],
             workingDir: process.cwd(),
             distDir: nextConfig.distDir || '.next',
             buildTarget: 'next',