From 1476a49140e2ddbc93c817437a5bd87c02062d91 Mon Sep 17 00:00:00 2001 From: Ash Wu Date: Mon, 9 Feb 2026 21:12:17 +0800 Subject: [PATCH] [build-tools] Add eas/report_maestro_test_results build function Signed-off-by: Ash Wu --- packages/build-tools/package.json | 1 + .../build-tools/src/customBuildContext.ts | 3 + .../build-tools/src/steps/easFunctions.ts | 3 + .../__tests__/internalMaestroTest.test.ts | 98 +++ .../__tests__/maestroResultParser.test.ts | 605 ++++++++++++++++++ .../reportMaestroTestResults.test.ts | 279 ++++++++ .../steps/functions/internalMaestroTest.ts | 8 + .../steps/functions/maestroResultParser.ts | 282 ++++++++ .../functions/reportMaestroTestResults.ts | 119 ++++ 9 files changed, 1398 insertions(+) create mode 100644 packages/build-tools/src/steps/functions/__tests__/internalMaestroTest.test.ts create mode 100644 packages/build-tools/src/steps/functions/__tests__/maestroResultParser.test.ts create mode 100644 packages/build-tools/src/steps/functions/__tests__/reportMaestroTestResults.test.ts create mode 100644 packages/build-tools/src/steps/functions/maestroResultParser.ts create mode 100644 packages/build-tools/src/steps/functions/reportMaestroTestResults.ts diff --git a/packages/build-tools/package.json b/packages/build-tools/package.json index a9cc9d13e6..a6a8ec62d1 100644 --- a/packages/build-tools/package.json +++ b/packages/build-tools/package.json @@ -52,6 +52,7 @@ "@google-cloud/storage": "^7.11.2", "@urql/core": "^6.0.1", "fast-glob": "^3.3.2", + "fast-xml-parser": "5.3.5", "fs-extra": "^11.2.0", "gql.tada": "^1.8.13", "joi": "^17.13.1", diff --git a/packages/build-tools/src/customBuildContext.ts b/packages/build-tools/src/customBuildContext.ts index 4fa9450b6d..d4dd843185 100644 --- a/packages/build-tools/src/customBuildContext.ts +++ b/packages/build-tools/src/customBuildContext.ts @@ -10,6 +10,7 @@ import { } from '@expo/eas-build-job'; import { bunyan } from '@expo/logger'; import { BuildRuntimePlatform, ExternalBuildContextProvider } from '@expo/steps'; +import { Client } from '@urql/core'; import assert from 'assert'; import path from 'path'; @@ -53,6 +54,7 @@ export class CustomBuildContext implements ExternalBuild public readonly startTime: Date; public readonly logger: bunyan; + public readonly graphqlClient: Client; public readonly runtimeApi: BuilderRuntimeApi; public job: TJob; public metadata?: Metadata; @@ -65,6 +67,7 @@ export class CustomBuildContext implements ExternalBuild this.metadata = buildCtx.metadata; this.logger = buildCtx.logger.child({ phase: BuildPhase.CUSTOM }); + this.graphqlClient = buildCtx.graphqlClient; this.projectSourceDirectory = path.join(buildCtx.workingdir, 'temporary-custom-build'); this.projectTargetDirectory = path.join(buildCtx.workingdir, 'build'); this.defaultWorkingDirectory = buildCtx.getReactNativeProjectDirectory(); diff --git a/packages/build-tools/src/steps/easFunctions.ts b/packages/build-tools/src/steps/easFunctions.ts index 88629aca12..41be6a99c8 100644 --- a/packages/build-tools/src/steps/easFunctions.ts +++ b/packages/build-tools/src/steps/easFunctions.ts @@ -20,6 +20,7 @@ import { createInstallPodsBuildFunction } from './functions/installPods'; import { createInternalEasMaestroTestFunction } from './functions/internalMaestroTest'; import { createPrebuildBuildFunction } from './functions/prebuild'; import { createRepackBuildFunction } from './functions/repack'; +import { createReportMaestroTestResultsFunction } from './functions/reportMaestroTestResults'; import { resolveAppleTeamIdFromCredentialsFunction } from './functions/resolveAppleTeamIdFromCredentials'; import { createResolveBuildConfigBuildFunction } from './functions/resolveBuildConfig'; import { @@ -79,6 +80,8 @@ export function getEasFunctions(ctx: CustomBuildContext): BuildFunction[] { createUploadToAscBuildFunction(), createInternalEasMaestroTestFunction(ctx), + + createReportMaestroTestResultsFunction(ctx), ]; if (ctx.hasBuildJob()) { diff --git a/packages/build-tools/src/steps/functions/__tests__/internalMaestroTest.test.ts b/packages/build-tools/src/steps/functions/__tests__/internalMaestroTest.test.ts new file mode 100644 index 0000000000..c0fe9c90c5 --- /dev/null +++ b/packages/build-tools/src/steps/functions/__tests__/internalMaestroTest.test.ts @@ -0,0 +1,98 @@ +import { spawnAsync } from '@expo/steps'; +import { vol } from 'memfs'; +import os from 'os'; + +import { createGlobalContextMock } from '../../../__tests__/utils/context'; +import { createMockLogger } from '../../../__tests__/utils/logger'; +import { IosSimulatorUtils } from '../../../utils/IosSimulatorUtils'; +import { findMaestroPathsFlowsToExecuteAsync } from '../../../utils/findMaestroPathsFlowsToExecuteAsync'; +import { createInternalEasMaestroTestFunction } from '../internalMaestroTest'; + +jest.mock('@expo/steps', () => ({ + ...jest.requireActual('@expo/steps'), + spawnAsync: jest.fn(), +})); + +jest.mock('../../../utils/IosSimulatorUtils'); +jest.mock('../../../utils/AndroidEmulatorUtils'); +jest.mock('../../../utils/findMaestroPathsFlowsToExecuteAsync'); + +const mockedSpawnAsync = jest.mocked(spawnAsync); +const mockedIosUtils = jest.mocked(IosSimulatorUtils); +const mockedFindFlows = jest.mocked(findMaestroPathsFlowsToExecuteAsync); + +describe(createInternalEasMaestroTestFunction, () => { + const mockUploadArtifact = jest.fn(); + + beforeEach(() => { + vol.mkdirSync(os.tmpdir(), { recursive: true }); + + mockedSpawnAsync.mockResolvedValue(undefined as any); + mockUploadArtifact.mockResolvedValue({ artifactId: null }); + + mockedIosUtils.getAvailableDevicesAsync.mockResolvedValue([ + { name: 'iPhone 15', udid: 'test-udid-123' } as any, + ]); + mockedIosUtils.cloneAsync.mockResolvedValue(undefined as any); + mockedIosUtils.startAsync.mockResolvedValue({ udid: 'cloned-udid' } as any); + mockedIosUtils.waitForReadyAsync.mockResolvedValue(undefined as any); + mockedIosUtils.collectLogsAsync.mockResolvedValue({ outputPath: '/tmp/logs.txt' } as any); + mockedIosUtils.deleteAsync.mockResolvedValue(undefined as any); + + mockedFindFlows.mockResolvedValue(['/project/.maestro/home.yml']); + }); + + function createStep(overrides?: { callInputs?: Record }) { + const ctx = { + runtimeApi: { uploadArtifact: mockUploadArtifact }, + }; + const fn = createInternalEasMaestroTestFunction(ctx as any); + return fn.createBuildStepFromFunctionCall( + createGlobalContextMock({ + logger: createMockLogger(), + }), + { + callInputs: { + platform: 'ios', + flow_paths: JSON.stringify(['.maestro']), + ...overrides?.callInputs, + }, + } + ); + } + + it('sets junit_report_directory output when output_format is junit', async () => { + const step = createStep({ + callInputs: { output_format: 'junit' }, + }); + await step.executeAsync(); + + const junitDir = step.getOutputValueByName('junit_report_directory'); + expect(junitDir).toMatch(/maestro-reports-/); + }); + + it('does not set junit_report_directory output when output_format is not junit', async () => { + const step = createStep(); + await step.executeAsync(); + + const junitDir = step.getOutputValueByName('junit_report_directory'); + expect(junitDir).toBeUndefined(); + }); + + it('does not set junit_report_directory when function throws before reaching output code', async () => { + mockedIosUtils.getAvailableDevicesAsync.mockResolvedValue([]); + + const step = createStep({ + callInputs: { output_format: 'junit' }, + }); + + try { + await step.executeAsync(); + } catch { + // Expected - no booted device found + } + + const junitDir = step.getOutputValueByName('junit_report_directory'); + expect(junitDir).toBeUndefined(); + }); +}); diff --git a/packages/build-tools/src/steps/functions/__tests__/maestroResultParser.test.ts b/packages/build-tools/src/steps/functions/__tests__/maestroResultParser.test.ts new file mode 100644 index 0000000000..d767f42e72 --- /dev/null +++ b/packages/build-tools/src/steps/functions/__tests__/maestroResultParser.test.ts @@ -0,0 +1,605 @@ +import { vol } from 'memfs'; + +import { + parseFlowMetadata, + parseFlowTags, + parseJUnitTestCases, + parseMaestroResults, +} from '../maestroResultParser'; + +describe(parseFlowMetadata, () => { + it('parses valid ai-*.json', async () => { + vol.fromJSON({ + '/tests/2026-01-28_055409/ai-home.json': JSON.stringify({ + flow_name: 'home', + flow_file_path: '/Users/expo/workingdir/build/.maestro/home.yml', + }), + }); + + const result = await parseFlowMetadata('/tests/2026-01-28_055409/ai-home.json'); + expect(result).toEqual({ + flowName: 'home', + flowFilePath: '/Users/expo/workingdir/build/.maestro/home.yml', + }); + }); + + it('returns null when flow_name is missing', async () => { + vol.fromJSON({ + '/tests/2026-01-28_055409/ai-home.json': JSON.stringify({ + flow_file_path: '/Users/expo/workingdir/build/.maestro/home.yml', + }), + }); + + const result = await parseFlowMetadata('/tests/2026-01-28_055409/ai-home.json'); + expect(result).toBeNull(); + }); + + it('returns null when flow_file_path is missing', async () => { + vol.fromJSON({ + '/tests/2026-01-28_055409/ai-home.json': JSON.stringify({ + flow_name: 'home', + }), + }); + + const result = await parseFlowMetadata('/tests/2026-01-28_055409/ai-home.json'); + expect(result).toBeNull(); + }); + + it('returns null for invalid JSON', async () => { + vol.fromJSON({ + '/tests/2026-01-28_055409/ai-home.json': 'not json', + }); + + const result = await parseFlowMetadata('/tests/2026-01-28_055409/ai-home.json'); + expect(result).toBeNull(); + }); +}); + +describe(parseFlowTags, () => { + it('extracts tags from flow YAML config section', async () => { + vol.fromJSON({ + '/flows/home.yml': [ + 'appId: com.example.app', + 'tags:', + ' - e2e', + ' - smoke', + '---', + '- launchApp', + ].join('\n'), + }); + + const tags = await parseFlowTags('/flows/home.yml'); + expect(tags).toEqual(['e2e', 'smoke']); + }); + + it('returns empty array when no tags field', async () => { + vol.fromJSON({ + '/flows/home.yml': ['appId: com.example.app', '---', '- launchApp'].join('\n'), + }); + + const tags = await parseFlowTags('/flows/home.yml'); + expect(tags).toEqual([]); + }); + + it('returns empty array when file does not exist', async () => { + const tags = await parseFlowTags('/nonexistent/home.yml'); + expect(tags).toEqual([]); + }); + + it('filters out non-string tags', async () => { + vol.fromJSON({ + '/flows/home.yml': [ + 'appId: com.example.app', + 'tags:', + ' - e2e', + ' - 123', + '---', + '- launchApp', + ].join('\n'), + }); + + const tags = await parseFlowTags('/flows/home.yml'); + expect(tags).toEqual(['e2e']); + }); +}); + +describe(parseMaestroResults, () => { + it('parses JUnit results and enriches with ai-*.json metadata', async () => { + vol.fromJSON({ + // JUnit XML (primary data) + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + ' Tap failed', + ' ', + ' ', + '', + ].join('\n'), + // Debug output (for flow_file_path) + '/tests/2026-01-28_055409/ai-home.json': JSON.stringify({ + flow_name: 'home', + flow_file_path: '/root/project/.maestro/home.yml', + }), + '/tests/2026-01-28_055409/ai-login.json': JSON.stringify({ + flow_name: 'login', + flow_file_path: '/root/project/.maestro/login.yml', + }), + }); + + const results = await parseMaestroResults('/junit', '/tests', '/root/project'); + expect(results).toHaveLength(2); + expect(results).toEqual( + expect.arrayContaining([ + { + name: 'home', + path: '.maestro/home.yml', + status: 'passed', + errorMessage: null, + duration: 10500, + retryCount: 0, + tags: [], + properties: {}, + }, + { + name: 'login', + path: '.maestro/login.yml', + status: 'failed', + errorMessage: 'Tap failed', + duration: 5000, + retryCount: 0, + tags: [], + properties: {}, + }, + ]) + ); + }); + + it('calculates retryCount from timestamp directory occurrences', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + '', + ].join('\n'), + // Two timestamp dirs = 1 retry + '/tests/2026-01-28_055409/ai-home.json': JSON.stringify({ + flow_name: 'home', + flow_file_path: '/root/project/.maestro/home.yml', + }), + '/tests/2026-01-28_055420/ai-home.json': JSON.stringify({ + flow_name: 'home', + flow_file_path: '/root/project/.maestro/home.yml', + }), + }); + + const results = await parseMaestroResults('/junit', '/tests', '/root/project'); + expect(results[0].retryCount).toBe(1); + }); + + it('uses flow name as fallback path when ai-*.json not found', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + '', + ].join('\n'), + // No ai-*.json files + }); + + const results = await parseMaestroResults('/junit', '/tests', '/root/project'); + expect(results[0].path).toBe('home'); + expect(results[0].retryCount).toBe(0); + }); + + it('returns empty array when no JUnit files found', async () => { + vol.fromJSON({ + '/junit/.gitkeep': '', + '/tests/2026-01-28_055409/ai-home.json': JSON.stringify({ + flow_name: 'home', + flow_file_path: '/root/project/.maestro/home.yml', + }), + }); + + const results = await parseMaestroResults('/junit', '/tests', '/root/project'); + expect(results).toEqual([]); + }); + + it('extracts tags from flow YAML and properties from JUnit', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + '', + ].join('\n'), + '/tests/2026-01-28_055409/ai-home.json': JSON.stringify({ + flow_name: 'home', + flow_file_path: '/root/project/.maestro/home.yml', + }), + '/root/project/.maestro/home.yml': [ + 'appId: com.example.app', + 'tags:', + ' - e2e', + ' - smoke', + '---', + '- launchApp', + ].join('\n'), + }); + + const results = await parseMaestroResults('/junit', '/tests', '/root/project'); + expect(results[0].tags).toEqual(['e2e', 'smoke']); + expect(results[0].properties).toEqual({ env: 'staging' }); + }); + + it('handles reuse_devices=true (junit_report_directory == tests_directory)', async () => { + vol.fromJSON({ + // Same directory for both JUnit and debug output + '/maestro-tests/android-maestro-junit.xml': [ + '', + '', + ' ', + ' ', + ' ', + '', + ].join('\n'), + '/maestro-tests/2026-01-28_055409/ai-home.json': JSON.stringify({ + flow_name: 'home', + flow_file_path: '/root/project/.maestro/home.yml', + }), + }); + + const results = await parseMaestroResults('/maestro-tests', '/maestro-tests', '/root/project'); + expect(results).toEqual([ + expect.objectContaining({ + name: 'home', + path: '.maestro/home.yml', + status: 'passed', + }), + ]); + }); + + it('handles reuse_devices=false (separate junit_report_directory)', async () => { + vol.fromJSON({ + // JUnit in temp dir (per-flow files) + '/tmp/maestro-reports-abc123/junit-report-flow-1.xml': [ + '', + '', + ' ', + ' ', + ' ', + '', + ].join('\n'), + '/tmp/maestro-reports-abc123/junit-report-flow-2.xml': [ + '', + '', + ' ', + ' ', + ' Failed', + ' ', + ' ', + '', + ].join('\n'), + // Debug output in default location + '/tests/2026-01-28_055409/ai-home.json': JSON.stringify({ + flow_name: 'home', + flow_file_path: '/root/project/.maestro/home.yml', + }), + '/tests/2026-01-28_055409/ai-login.json': JSON.stringify({ + flow_name: 'login', + flow_file_path: '/root/project/.maestro/login.yml', + }), + }); + + const results = await parseMaestroResults( + '/tmp/maestro-reports-abc123', + '/tests', + '/root/project' + ); + expect(results).toHaveLength(2); + expect(results).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'home', path: '.maestro/home.yml', status: 'passed' }), + expect.objectContaining({ name: 'login', path: '.maestro/login.yml', status: 'failed' }), + ]) + ); + }); + + it('uses raw path when flow_file_path is outside project_root', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + '', + ].join('\n'), + '/tests/2026-01-28_055409/ai-home.json': JSON.stringify({ + flow_name: 'home', + flow_file_path: '/somewhere/else/.maestro/home.yml', + }), + }); + + const results = await parseMaestroResults('/junit', '/tests', '/root/project'); + expect(results[0].path).toBe('/somewhere/else/.maestro/home.yml'); + }); + + it('filters out non-timestamp directories when scanning debug output', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + '', + ].join('\n'), + '/tests/2026-01-28_055409/ai-home.json': JSON.stringify({ + flow_name: 'home', + flow_file_path: '/root/project/.maestro/home.yml', + }), + '/tests/not-a-timestamp/ai-home.json': JSON.stringify({ + flow_name: 'home', + flow_file_path: '/root/project/.maestro/home.yml', + }), + }); + + const results = await parseMaestroResults('/junit', '/tests', '/root/project'); + // Only 1 occurrence (not-a-timestamp dir should be ignored) + expect(results[0].retryCount).toBe(0); + }); +}); + +describe(parseJUnitTestCases, () => { + it('parses a single testcase with SUCCESS status', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + ' ', + '', + ].join('\n'), + }); + + const results = await parseJUnitTestCases('/junit'); + expect(results).toEqual([ + { + name: 'home', + status: 'passed', + duration: 10500, + errorMessage: null, + properties: {}, + }, + ]); + }); + + it('parses a failed testcase with failure message', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' Element not visible', + ' ', + ' ', + '', + ].join('\n'), + }); + + const results = await parseJUnitTestCases('/junit'); + expect(results).toEqual([ + { + name: 'login', + status: 'failed', + duration: 5000, + errorMessage: 'Element not visible', + properties: {}, + }, + ]); + }); + + it('extracts properties from testcase', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + '', + ].join('\n'), + }); + + const results = await parseJUnitTestCases('/junit'); + expect(results[0].properties).toEqual({ env: 'staging', priority: 'high' }); + }); + + it('parses multiple testcases across multiple testsuites (shards)', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + ' ', + ' ', + ' Tap failed', + ' ', + ' ', + '', + ].join('\n'), + }); + + const results = await parseJUnitTestCases('/junit'); + expect(results).toHaveLength(2); + expect(results).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'home', status: 'passed' }), + expect.objectContaining({ name: 'login', status: 'failed', errorMessage: 'Tap failed' }), + ]) + ); + }); + + it('parses multiple JUnit XML files in the same directory', async () => { + vol.fromJSON({ + '/junit/junit-report-flow-1.xml': [ + '', + '', + ' ', + ' ', + ' ', + '', + ].join('\n'), + '/junit/junit-report-flow-2.xml': [ + '', + '', + ' ', + ' ', + ' Tap failed', + ' ', + ' ', + '', + ].join('\n'), + }); + + const results = await parseJUnitTestCases('/junit'); + expect(results).toHaveLength(2); + }); + + it('handles missing time attribute (defaults to 0)', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + '', + ].join('\n'), + }); + + const results = await parseJUnitTestCases('/junit'); + expect(results[0].duration).toBe(0); + }); + + it('handles invalid time attribute (defaults to 0)', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + '', + ].join('\n'), + }); + + const results = await parseJUnitTestCases('/junit'); + expect(results[0].duration).toBe(0); + }); + + it('uses @_status attribute for pass/fail (not presence)', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + '', + ].join('\n'), + }); + + const results = await parseJUnitTestCases('/junit'); + expect(results[0].status).toBe('failed'); + expect(results[0].errorMessage).toBeNull(); + }); + + it('extracts error message from element', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' Runtime exception occurred', + ' ', + ' ', + '', + ].join('\n'), + }); + + const results = await parseJUnitTestCases('/junit'); + expect(results[0].status).toBe('failed'); + expect(results[0].errorMessage).toBe('Runtime exception occurred'); + }); + + it('returns empty array when directory does not exist', async () => { + const results = await parseJUnitTestCases('/nonexistent'); + expect(results).toEqual([]); + }); + + it('returns empty array when no XML files found', async () => { + vol.fromJSON({ '/junit/.gitkeep': '' }); + const results = await parseJUnitTestCases('/junit'); + expect(results).toEqual([]); + }); + + it('skips invalid XML files gracefully', async () => { + vol.fromJSON({ + '/junit/bad.xml': 'not xml at all', + '/junit/good.xml': [ + '', + '', + ' ', + ' ', + ' ', + '', + ].join('\n'), + }); + + const results = await parseJUnitTestCases('/junit'); + expect(results).toEqual([expect.objectContaining({ name: 'home', status: 'passed' })]); + }); + + it('handles testcase with no properties element', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + '', + ].join('\n'), + }); + + const results = await parseJUnitTestCases('/junit'); + expect(results[0].properties).toEqual({}); + }); +}); diff --git a/packages/build-tools/src/steps/functions/__tests__/reportMaestroTestResults.test.ts b/packages/build-tools/src/steps/functions/__tests__/reportMaestroTestResults.test.ts new file mode 100644 index 0000000000..78b2bf2eab --- /dev/null +++ b/packages/build-tools/src/steps/functions/__tests__/reportMaestroTestResults.test.ts @@ -0,0 +1,279 @@ +import { Client } from '@urql/core'; +import { vol } from 'memfs'; + +import { createGlobalContextMock } from '../../../__tests__/utils/context'; +import { createMockLogger } from '../../../__tests__/utils/logger'; +import { createReportMaestroTestResultsFunction } from '../reportMaestroTestResults'; + +const JUNIT_PASS = [ + '', + '', + ' ', + ' ', + ' ', + '', +].join('\n'); + +const JUNIT_FAIL = [ + '', + '', + ' ', + ' ', + ' Tap failed', + ' ', + ' ', + '', +].join('\n'); + +const FLOW_AI = JSON.stringify({ + flow_name: 'home', + flow_file_path: '/root/project/.maestro/home.yml', +}); + +describe(createReportMaestroTestResultsFunction, () => { + let mockMutationFn: jest.Mock; + let mockGraphqlClient: Client; + + beforeEach(() => { + mockMutationFn = jest.fn(); + mockGraphqlClient = { + mutation: jest.fn().mockReturnValue({ + toPromise: mockMutationFn, + }), + } as unknown as Client; + }); + + function createStep(overrides?: { + callInputs?: Record; + staticContextContent?: Record; + env?: Record; + }) { + const ctx = { graphqlClient: mockGraphqlClient }; + const fn = createReportMaestroTestResultsFunction(ctx as any); + const globalCtx = createGlobalContextMock({ + logger: createMockLogger(), + projectTargetDirectory: '/root/project', + staticContextContent: { + expoApiServerURL: 'https://api.expo.test', + job: { secrets: { robotAccessToken: 'test-token' } }, + ...overrides?.staticContextContent, + }, + }); + globalCtx.updateEnv(overrides?.env ?? { __WORKFLOW_JOB_ID: 'job-uuid-123' }); + return fn.createBuildStepFromFunctionCall(globalCtx, { + callInputs: { + junit_report_directory: '/junit', + tests_directory: '/tests', + ...overrides?.callInputs, + }, + }); + } + + it('parses JUnit results and calls GraphQL mutation', async () => { + vol.fromJSON({ + '/junit/report.xml': JUNIT_PASS, + '/tests/2026-01-28_055409/ai-home.json': FLOW_AI, + }); + + mockMutationFn.mockResolvedValue({ + data: { + workflowDeviceTestCaseResult: { + createWorkflowDeviceTestCaseResults: [{ id: 'id-1' }], + }, + }, + }); + + await createStep().executeAsync(); + + expect(mockGraphqlClient.mutation).toHaveBeenCalledTimes(1); + const [, variables] = (mockGraphqlClient.mutation as jest.Mock).mock.calls[0]; + expect(variables.input.workflowJobId).toBe('job-uuid-123'); + expect(variables.input.testCaseResults).toEqual([ + expect.objectContaining({ + name: 'home', + path: '.maestro/home.yml', + status: 'PASSED', + errorMessage: null, + duration: 10000, + retryCount: 0, + properties: [], + }), + ]); + }); + + it('sends tags from flow YAML and properties from JUnit', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + '', + ].join('\n'), + '/tests/2026-01-28_055409/ai-home.json': FLOW_AI, + '/root/project/.maestro/home.yml': [ + 'appId: com.example.app', + 'tags:', + ' - e2e', + '---', + '- launchApp', + ].join('\n'), + }); + + mockMutationFn.mockResolvedValue({ + data: { + workflowDeviceTestCaseResult: { + createWorkflowDeviceTestCaseResults: [{ id: 'id-1' }], + }, + }, + }); + + await createStep().executeAsync(); + + const [, variables] = (mockGraphqlClient.mutation as jest.Mock).mock.calls[0]; + expect(variables.input.testCaseResults[0].tags).toEqual(['e2e']); + expect(variables.input.testCaseResults[0].properties).toEqual([ + { name: 'testCaseId', value: 'TC-001' }, + { name: 'priority', value: 'high' }, + ]); + }); + + it('reports failed flow with correct status and errorMessage', async () => { + vol.fromJSON({ + '/junit/report.xml': JUNIT_FAIL, + '/tests/2026-01-28_055409/ai-home.json': FLOW_AI, + }); + + mockMutationFn.mockResolvedValue({ + data: { + workflowDeviceTestCaseResult: { + createWorkflowDeviceTestCaseResults: [{ id: 'id-1' }], + }, + }, + }); + + await createStep().executeAsync(); + + const [, variables] = (mockGraphqlClient.mutation as jest.Mock).mock.calls[0]; + expect(variables.input.testCaseResults[0]).toEqual( + expect.objectContaining({ + status: 'FAILED', + errorMessage: 'Tap failed', + duration: 5000, + }) + ); + }); + + // robotAccessToken guard removed — Generic.JobZ requires robotAccessToken (z.string(), + // not optional), so it's always present in workflow jobs. graphqlClient is already + // initialized with the token in BuildContext constructor. + + it('skips report when junit_report_directory is empty string (upstream step failed)', async () => { + const step = createStep({ + callInputs: { junit_report_directory: '' }, + }); + await step.executeAsync(); + expect(mockGraphqlClient.mutation).not.toHaveBeenCalled(); + }); + + it('skips report when __WORKFLOW_JOB_ID env is not set', async () => { + vol.fromJSON({ + '/junit/report.xml': JUNIT_PASS, + '/tests/2026-01-28_055409/ai-home.json': FLOW_AI, + }); + + const step = createStep({ env: {} }); + await step.executeAsync(); + expect(mockGraphqlClient.mutation).not.toHaveBeenCalled(); + }); + + it('skips report when no JUnit files found', async () => { + const step = createStep(); + await step.executeAsync(); + expect(mockGraphqlClient.mutation).not.toHaveBeenCalled(); + }); + + it('does not throw when GraphQL returns error', async () => { + vol.fromJSON({ + '/junit/report.xml': JUNIT_PASS, + '/tests/2026-01-28_055409/ai-home.json': FLOW_AI, + }); + + mockMutationFn.mockResolvedValue({ + error: { message: 'Something went wrong' }, + }); + + await createStep().executeAsync(); + }); + + it('does not throw when mutation rejects', async () => { + vol.fromJSON({ + '/junit/report.xml': JUNIT_PASS, + '/tests/2026-01-28_055409/ai-home.json': FLOW_AI, + }); + + mockMutationFn.mockRejectedValue(new Error('Network error')); + + await createStep().executeAsync(); + }); + + it('skips report when duplicate testcase names found', async () => { + vol.fromJSON({ + '/junit/report.xml': [ + '', + '', + ' ', + ' ', + ' ', + ' ', + '', + ].join('\n'), + '/tests/2026-01-28_055409/ai-login.json': JSON.stringify({ + flow_name: 'login', + flow_file_path: '/root/project/.maestro/login.yml', + }), + }); + + await createStep().executeAsync(); + expect(mockGraphqlClient.mutation).not.toHaveBeenCalled(); + }); + + it('uses default directories when inputs are not provided', async () => { + vol.fromJSON({ + '/home/expo/.maestro/tests/report.xml': JUNIT_PASS, + '/home/expo/.maestro/tests/2026-01-28_055409/ai-home.json': FLOW_AI, + }); + + const ctx = { graphqlClient: mockGraphqlClient }; + const fn = createReportMaestroTestResultsFunction(ctx as any); + const globalCtx = createGlobalContextMock({ + logger: createMockLogger(), + projectTargetDirectory: '/root/project', + staticContextContent: { + expoApiServerURL: 'https://api.expo.test', + job: { secrets: { robotAccessToken: 'test-token' } }, + }, + }); + globalCtx.updateEnv({ __WORKFLOW_JOB_ID: 'job-uuid-123', HOME: '/home/expo' }); + + mockMutationFn.mockResolvedValue({ + data: { + workflowDeviceTestCaseResult: { + createWorkflowDeviceTestCaseResults: [{ id: 'id-1' }], + }, + }, + }); + + const step = fn.createBuildStepFromFunctionCall(globalCtx, { + callInputs: {}, + }); + await step.executeAsync(); + expect(mockGraphqlClient.mutation).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/build-tools/src/steps/functions/internalMaestroTest.ts b/packages/build-tools/src/steps/functions/internalMaestroTest.ts index 48cac43fc9..045be6b206 100644 --- a/packages/build-tools/src/steps/functions/internalMaestroTest.ts +++ b/packages/build-tools/src/steps/functions/internalMaestroTest.ts @@ -81,6 +81,10 @@ export function createInternalEasMaestroTestFunction(ctx: CustomBuildContext): B id: 'test_reports_artifact_id', required: false, }), + BuildStepOutput.createProvider({ + id: 'junit_report_directory', + required: false, + }), ], fn: async (stepCtx, { inputs: _inputs, env, outputs }) => { // inputs come in form of { value: unknown }. Here we parse them into a typed and validated object. @@ -348,6 +352,10 @@ export function createInternalEasMaestroTestFunction(ctx: CustomBuildContext): B } } + if (output_format === 'junit') { + outputs.junit_report_directory.set(maestroReportsDir); + } + const generatedDeviceLogs = await fs.promises.readdir(deviceLogsDir); if (generatedDeviceLogs.length === 0) { stepCtx.logger.warn('No device logs were successfully collected.'); diff --git a/packages/build-tools/src/steps/functions/maestroResultParser.ts b/packages/build-tools/src/steps/functions/maestroResultParser.ts new file mode 100644 index 0000000000..b93032fd84 --- /dev/null +++ b/packages/build-tools/src/steps/functions/maestroResultParser.ts @@ -0,0 +1,282 @@ +import { XMLParser } from 'fast-xml-parser'; +import fs from 'fs/promises'; +import path from 'path'; +import YAML from 'yaml'; +import { z } from 'zod'; + +export interface FlowMetadata { + flowName: string; + flowFilePath: string; +} + +export interface MaestroFlowResult { + name: string; + path: string; + status: 'passed' | 'failed'; + errorMessage: string | null; + duration: number; // milliseconds + retryCount: number; + tags: string[]; + properties: Record; +} + +const TIMESTAMP_DIR_PATTERN = /^\d{4}-\d{2}-\d{2}_\d{6}$/; + +export function extractFlowKey(filename: string, prefix: string): string | null { + const match = filename.match(new RegExp(`^${prefix}-(.+)\\.json$`)); + return match?.[1] ?? null; +} + +export interface JUnitTestCaseResult { + name: string; + status: 'passed' | 'failed'; + duration: number; // milliseconds + errorMessage: string | null; + properties: Record; +} + +const xmlParser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '@_', + // Ensure single-element arrays are always arrays + isArray: name => ['testsuite', 'testcase', 'property'].includes(name), +}); + +export async function parseJUnitTestCases(junitDirectory: string): Promise { + let entries: string[]; + try { + entries = await fs.readdir(junitDirectory); + } catch { + return []; + } + + const xmlFiles = entries.filter(f => f.endsWith('.xml')); + if (xmlFiles.length === 0) { + return []; + } + + const results: JUnitTestCaseResult[] = []; + + for (const xmlFile of xmlFiles) { + try { + const content = await fs.readFile(path.join(junitDirectory, xmlFile), 'utf-8'); + const parsed = xmlParser.parse(content); + + const testsuites = parsed?.testsuites?.testsuite; + if (!Array.isArray(testsuites)) { + continue; + } + + for (const suite of testsuites) { + const testcases = suite?.testcase; + if (!Array.isArray(testcases)) { + continue; + } + + for (const tc of testcases) { + const name = tc['@_name']; + if (!name) { + continue; + } + + const timeStr = tc['@_time']; + const timeSeconds = timeStr ? parseFloat(timeStr) : 0; + const duration = Number.isFinite(timeSeconds) ? Math.round(timeSeconds * 1000) : 0; + + // Use @_status as primary indicator (more robust than checking presence) + const status: 'passed' | 'failed' = tc['@_status'] === 'SUCCESS' ? 'passed' : 'failed'; + // Extract error message from or elements + const failureText = + tc.failure != null + ? typeof tc.failure === 'string' + ? tc.failure + : (tc.failure?.['#text'] ?? null) + : null; + const errorText = + tc.error != null + ? typeof tc.error === 'string' + ? tc.error + : (tc.error?.['#text'] ?? null) + : null; + const errorMessage: string | null = failureText ?? errorText ?? null; + + // Extract properties + const rawProperties: { '@_name': string; '@_value': string }[] = + tc.properties?.property ?? []; + const properties: Record = {}; + + for (const prop of rawProperties) { + const propName = prop['@_name']; + const value = prop['@_value']; + if (typeof propName !== 'string' || typeof value !== 'string') { + continue; + } + properties[propName] = value; + } + + results.push({ name, status, duration, errorMessage, properties }); + } + } + } catch { + // Skip malformed XML files + continue; + } + } + + return results; +} + +const FlowMetadataFileSchema = z.object({ + flow_name: z.string(), + flow_file_path: z.string(), +}); + +/** + * Parses an `ai-*.json` file produced by Maestro's TestDebugReporter. + * + * The file contains: + * - `flow_name`: derived from the YAML `config.name` field if present, otherwise + * the flow filename without extension. + * See: https://github.com/mobile-dev-inc/Maestro/blob/c0e95fd/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt#L70 + * - `flow_file_path`: absolute path to the original flow YAML file. + * - `outputs`: screenshot defect data (unused here). + * + * Filename format: `ai-(flowName).json` where `/` in flowName is replaced with `_`. + * See: https://github.com/mobile-dev-inc/Maestro/blob/c0e95fd/maestro-cli/src/main/java/maestro/cli/report/TestDebugReporter.kt#L67 + */ +export async function parseFlowMetadata(filePath: string): Promise { + try { + const content = await fs.readFile(filePath, 'utf-8'); + const data = JSON.parse(content); + const parsed = FlowMetadataFileSchema.parse(data); + return { + flowName: parsed.flow_name, + flowFilePath: parsed.flow_file_path, + }; + } catch { + return null; + } +} + +/** + * Reads tags from a Maestro flow YAML file's config section. + * Flow files are structured as: config (object) + `---` + commands (array). + */ +export async function parseFlowTags(flowFilePath: string): Promise { + try { + const content = await fs.readFile(flowFilePath, 'utf-8'); + const docs = YAML.parseAllDocuments(content); + if (docs.length === 0) { + return []; + } + const metadata = docs[0].toJSON(); + if (metadata && Array.isArray(metadata.tags)) { + return metadata.tags.filter((t: unknown): t is string => typeof t === 'string'); + } + return []; + } catch { + return []; + } +} + +export async function parseMaestroResults( + junitDirectory: string, + testsDirectory: string, + projectRoot: string +): Promise { + // 1. Parse JUnit XML files (primary source) + const junitResults = await parseJUnitTestCases(junitDirectory); + if (junitResults.length === 0) { + return []; + } + + // 2. Parse ai-*.json from debug output for flow_file_path + retryCount + const flowPathMap = new Map(); // flowName → flowFilePath + const flowOccurrences = new Map(); // flowName → count + + let entries: string[]; + try { + entries = await fs.readdir(testsDirectory); + } catch { + entries = []; + } + + const timestampDirs = entries.filter(name => TIMESTAMP_DIR_PATTERN.test(name)).sort(); + + for (const dir of timestampDirs) { + const dirPath = path.join(testsDirectory, dir); + let files: string[]; + try { + files = await fs.readdir(dirPath); + } catch { + continue; + } + + for (const file of files) { + const flowKey = extractFlowKey(file, 'ai'); + if (!flowKey) { + continue; + } + + const metadata = await parseFlowMetadata(path.join(dirPath, file)); + if (!metadata) { + continue; + } + + // Track latest path (last timestamp dir wins) + flowPathMap.set(metadata.flowName, metadata.flowFilePath); + + // Count occurrences for retryCount + flowOccurrences.set(metadata.flowName, (flowOccurrences.get(metadata.flowName) ?? 0) + 1); + } + } + + // 3. Merge: JUnit results + ai-*.json metadata + const results: MaestroFlowResult[] = []; + + for (const junit of junitResults) { + const flowFilePath = flowPathMap.get(junit.name); + const relativePath = flowFilePath + ? await relativizePathAsync(flowFilePath, projectRoot) + : junit.name; // fallback: use flow name if ai-*.json not found + + const occurrences = flowOccurrences.get(junit.name) ?? 0; + const retryCount = Math.max(0, occurrences - 1); + const tags = flowFilePath ? await parseFlowTags(flowFilePath) : []; + + results.push({ + name: junit.name, + path: relativePath, + status: junit.status, + errorMessage: junit.errorMessage, + duration: junit.duration, + retryCount, + tags, + properties: junit.properties, + }); + } + + return results; +} + +async function relativizePathAsync(flowFilePath: string, projectRoot: string): Promise { + if (!path.isAbsolute(flowFilePath)) { + return flowFilePath; + } + + // Resolve symlinks (e.g., /tmp -> /private/tmp on macOS) for consistent comparison + let resolvedRoot = projectRoot; + let resolvedFlow = flowFilePath; + try { + resolvedRoot = await fs.realpath(projectRoot); + } catch {} + try { + resolvedFlow = await fs.realpath(flowFilePath); + } catch {} + + const relative = path.relative(resolvedRoot, resolvedFlow); + if (relative.startsWith('..')) { + return flowFilePath; + } + return relative; +} diff --git a/packages/build-tools/src/steps/functions/reportMaestroTestResults.ts b/packages/build-tools/src/steps/functions/reportMaestroTestResults.ts new file mode 100644 index 0000000000..47b222187a --- /dev/null +++ b/packages/build-tools/src/steps/functions/reportMaestroTestResults.ts @@ -0,0 +1,119 @@ +import { BuildFunction, BuildStepInput, BuildStepInputValueTypeName } from '@expo/steps'; +import { graphql } from 'gql.tada'; + +import { MaestroFlowResult, parseMaestroResults } from './maestroResultParser'; +import { CustomBuildContext } from '../../customBuildContext'; + +const CREATE_MUTATION = graphql(` + mutation CreateWorkflowDeviceTestCaseResults($input: CreateWorkflowDeviceTestCaseResultsInput!) { + workflowDeviceTestCaseResult { + createWorkflowDeviceTestCaseResults(input: $input) { + id + } + } + } +`); + +const FLOW_STATUS_TO_TEST_CASE_RESULT_STATUS: Record = { + passed: 'PASSED', + failed: 'FAILED', +} satisfies Record; + +export function createReportMaestroTestResultsFunction(ctx: CustomBuildContext): BuildFunction { + return new BuildFunction({ + namespace: 'eas', + id: 'report_maestro_test_results', + name: 'Report Maestro Test Results', + __metricsId: 'eas/report_maestro_test_results', + inputProviders: [ + BuildStepInput.createProvider({ + id: 'junit_report_directory', + required: false, + allowedValueTypeName: BuildStepInputValueTypeName.STRING, + defaultValue: '${ eas.env.HOME }/.maestro/tests', + }), + BuildStepInput.createProvider({ + id: 'tests_directory', + required: false, + allowedValueTypeName: BuildStepInputValueTypeName.STRING, + defaultValue: '${ eas.env.HOME }/.maestro/tests', + }), + ], + fn: async (stepsCtx, { inputs }) => { + const { logger } = stepsCtx; + const workflowJobId = stepsCtx.global.env.__WORKFLOW_JOB_ID; + if (!workflowJobId) { + return; + } + const junitDirectory = (inputs.junit_report_directory.value as string | undefined) ?? ''; + if (!junitDirectory) { + logger.info('No JUnit directory provided, skipping test results report'); + return; + } + const testsDirectory = inputs.tests_directory.value as string; + + const flowResults = await parseMaestroResults( + junitDirectory, + testsDirectory, + stepsCtx.workingDirectory + ); + if (flowResults.length === 0) { + logger.info('No maestro test results found, skipping report'); + return; + } + + // Maestro allows overriding flow names via config, so different flow files can share + // the same name. JUnit XML only contains names (not file paths), making it impossible + // to map duplicates back to their original flow files. Skip and let the user fix it. + const names = flowResults.map(r => r.name); + const duplicates = names.filter((n, i) => names.indexOf(n) !== i); + if (duplicates.length > 0) { + logger.error( + `Duplicate test case names found in JUnit output: ${[...new Set(duplicates)].join( + ', ' + )}. Skipping report. Ensure each Maestro flow has a unique name.` + ); + return; + } + + try { + const testCaseResults = flowResults.flatMap(f => { + const status = FLOW_STATUS_TO_TEST_CASE_RESULT_STATUS[f.status]; + if (!status) { + return []; + } + return [ + { + name: f.name, + path: f.path, + status, + errorMessage: f.errorMessage, + duration: f.duration, + retryCount: f.retryCount, + tags: f.tags, + properties: Object.entries(f.properties).map(([name, value]) => ({ name, value })), + }, + ]; + }); + + const result = await ctx.graphqlClient + .mutation(CREATE_MUTATION, { + input: { + workflowJobId, + testCaseResults, + }, + }) + .toPromise(); + + if (result.error) { + logger.error({ error: result.error }, 'GraphQL error creating test case results'); + return; + } + + logger.info(`Reported ${testCaseResults.length} test case result(s).`); + } catch (error) { + logger.error({ err: error }, 'Failed to create test case results'); + } + }, + }); +}