From dbc5e7d07c45a0f0cde05f1088ddc003c3c99e7f Mon Sep 17 00:00:00 2001 From: Giovanni Mascellani <gmascellani@codeweavers.com> Date: Tue, 26 Sep 2023 11:20:39 +0200 Subject: [PATCH] ci: Run cross tests on Windows. A driver program is introduced to coordinate test running on Windows, similarly to what "make test" does on Linux and macOS. --- .gitignore | 1 + .gitlab-ci.yml | 2 + Makefile.am | 27 +++++- gitlab/README | 10 +- gitlab/build-crosstest | 5 +- gitlab/test.yml | 35 +++++++ tests/driver.c | 204 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 278 insertions(+), 6 deletions(-) create mode 100644 gitlab/test.yml create mode 100644 tests/driver.c diff --git a/.gitignore b/.gitignore index 841220ee6b..5c35199a5b 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ vkd3d-*.tar.xz *.tab.c *.tab.h *.trs +*.txt *.yy.c *~ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8cc59d7649..276c654ef4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,9 @@ stages: - image - build + - test include: - local: "/gitlab/image.yml" - local: "/gitlab/build.yml" + - local: "/gitlab/test.yml" diff --git a/Makefile.am b/Makefile.am index 2821ddc6ff..4e3a43bb89 100644 --- a/Makefile.am +++ b/Makefile.am @@ -480,6 +480,9 @@ shader_runner_cross_sources = \ $(srcdir)/tests/shader_runner_d3d11.c \ $(srcdir)/tests/shader_runner_d3d12.c +driver_cross_sources = \ + $(srcdir)/tests/driver.c + if HAVE_CROSSTARGET32 CROSS32_CC = @CROSSCC32@ CROSS32_DLLTOOL = @CROSSTARGET32@-dlltool @@ -495,6 +498,7 @@ endif CROSS32_FILES = $(CROSS32_EXEFILES) if BUILD_TESTS CROSS32_FILES += tests/shader_runner.cross32.exe +CROSS32_FILES += tests/driver.cross32.exe endif CLEANFILES += $(CROSS32_IMPLIBS) $(CROSS32_FILES) @@ -517,6 +521,11 @@ tests/shader_runner.cross32.exe: $(shader_runner_cross_sources) $(CROSS32_IMPLIB $(CROSS32_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(shader_runner_cross_sources) $(CROSS32_IMPLIBS) -ldxgi -lgdi32 -ld3dcompiler_47 && \ $(am__mv) $$depbase.Tpo $$depbase.Po +tests/driver.cross32.exe: $(driver_cross_sources) + $(AM_V_CCLD)depbase=`echo $@ | sed 's![^/]*$$!$(DEPDIR)/&!;s!\.exe$$!!'`; \ + $(CROSS32_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(driver_cross_sources) && \ + $(am__mv) $$depbase.Tpo $$depbase.Po + else crosstest32: endif @@ -536,6 +545,7 @@ endif CROSS64_FILES = $(CROSS64_EXEFILES) if BUILD_TESTS CROSS64_FILES += tests/shader_runner.cross64.exe +CROSS64_FILES += tests/driver.cross64.exe endif CLEANFILES += $(CROSS64_IMPLIBS) $(CROSS64_FILES) @@ -558,12 +568,25 @@ tests/shader_runner.cross64.exe: $(shader_runner_cross_sources) $(CROSS64_IMPLIB $(CROSS64_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(shader_runner_cross_sources) $(CROSS64_IMPLIBS) -ldxgi -lgdi32 -ld3dcompiler_47 && \ $(am__mv) $$depbase.Tpo $$depbase.Po +tests/driver.cross64.exe: $(driver_cross_sources) + $(AM_V_CCLD)depbase=`echo $@ | sed 's![^/]*$$!$(DEPDIR)/&!;s!\.exe$$!!'`; \ + $(CROSS64_CC) $(CROSS_CFLAGS) -MT $@ -MD -MP -MF $$depbase.Tpo -o $@ $(driver_cross_sources) && \ + $(am__mv) $$depbase.Tpo $$depbase.Po + else crosstest64: endif -.PHONY: crosstest crosstest32 crosstest64 -crosstest: crosstest32 crosstest64 +tests/crosstests.txt: FORCE + $(AM_V_GEN) for i in $(vkd3d_cross_tests) ; do echo $$i ; done > $@ + +crosstest-lists: tests/crosstests.txt +CLEANFILES += tests/crosstests.txt + +FORCE: + +.PHONY: crosstest crosstest32 crosstest64 crosstest-lists FORCE +crosstest: crosstest32 crosstest64 crosstest-lists if BUILD_DOC @DX_RULES@ diff --git a/gitlab/README b/gitlab/README index 04fd611190..310912be29 100644 --- a/gitlab/README +++ b/gitlab/README @@ -19,9 +19,10 @@ MoltenVK as the Vulkan driver. The llvmpipe and macOS jobs are currently allowed to fail. Additionally, MinGW is used to build PE binaries for both vkd3d and -its crosstests, for both 32 and 64 bit. These builds are not currently -tested (but the pipeline still fails if the compilation is not -successful). +its crosstests, for both 32 and 64 bit. The PE crosstests are executed +on Windows 10 to check that behavior imposed by the tests corresponds +to Microsoft's D3D12 implementation. The rendering backend is +currently Window's WARP software implementation. The testing logs are available as CI artifacts, as well as the PE modules built by the crosstest and MinGW jobs. @@ -58,3 +59,6 @@ environment for running the tests. All the software required to compile and run the tests will therefore have to be installed directly on the host system. Complete instructions to setup the macOS are currently not available. + +Finally, a runner tagged with `win10-21h2' must be available and +submit jobs to a Windows 10 virtual machine. diff --git a/gitlab/build-crosstest b/gitlab/build-crosstest index 4a1341a09b..da59bc8da5 100755 --- a/gitlab/build-crosstest +++ b/gitlab/build-crosstest @@ -19,6 +19,9 @@ cd build touch ../pipeline_failed mkdir -p ../artifacts/$COMMIT -rsync -Rr config.log tests/*.exe ../artifacts/$COMMIT +rsync -Rr config.log tests/*.txt tests/*.exe ../artifacts/$COMMIT + +# Make the driver easily available to the Windows CI job +cp tests/driver.cross64.exe ../artifacts git reset --hard diff --git a/gitlab/test.yml b/gitlab/test.yml new file mode 100644 index 0000000000..97bebba531 --- /dev/null +++ b/gitlab/test.yml @@ -0,0 +1,35 @@ +test-win-64: + stage: test + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + interruptible: true + needs: + - job: build-crosstest + tags: + - win10-21h2 + script: + - ./artifacts/driver.cross64.exe + variables: + TEST_ARCH: "64" + artifacts: + when: always + paths: + - artifacts + +test-win-32: + stage: test + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + interruptible: true + needs: + - job: build-crosstest + tags: + - win10-21h2 + script: + - ./artifacts/driver.cross64.exe + variables: + TEST_ARCH: "32" + artifacts: + when: always + paths: + - artifacts diff --git a/tests/driver.c b/tests/driver.c new file mode 100644 index 0000000000..f6da34054e --- /dev/null +++ b/tests/driver.c @@ -0,0 +1,204 @@ +/* + * Copyright 2023 Giovanni Mascellani for CodeWeavers + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + */ + +#include <stdio.h> +#include <stdbool.h> +#include <windows.h> + +#define TIMEOUT_MS (10 * 1000) +#define MAX_TIMEOUT_COUNT 3 + +enum program_result +{ + PROGRAM_RESULT_SUCCESS, + PROGRAM_RESULT_TIMEOUT, + PROGRAM_RESULT_FAILURE, +}; + +static enum program_result run_program(const char *cmdline, const char *log_filename) +{ + enum program_result ret = PROGRAM_RESULT_SUCCESS; + HANDLE log = INVALID_HANDLE_VALUE; + SECURITY_ATTRIBUTES attrs = {0}; + PROCESS_INFORMATION info = {0}; + DWORD exit_code, wait_result; + STARTUPINFOA startup = {0}; + char cmdline2[1024]; + + strcpy(cmdline2, cmdline); + + attrs.nLength = sizeof(attrs); + attrs.bInheritHandle = TRUE; + + log = CreateFileA(log_filename, GENERIC_WRITE, 0, &attrs, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (log == INVALID_HANDLE_VALUE) + { + fprintf(stderr, "Cannot create log file %s, last error %ld.\n", log_filename, GetLastError()); + ret = PROGRAM_RESULT_FAILURE; + goto out; + } + + startup.cb = sizeof(startup); + startup.dwFlags = STARTF_USESTDHANDLES; + startup.hStdInput = INVALID_HANDLE_VALUE; + startup.hStdOutput = log; + startup.hStdError = log; + + if (!CreateProcessA(NULL, cmdline2, NULL, NULL, TRUE, 0, NULL, NULL, &startup, &info)) + { + fprintf(stderr, "Cannot create process %s, last error %ld.\n", cmdline2, GetLastError()); + ret = PROGRAM_RESULT_FAILURE; + goto out; + } + + wait_result = WaitForSingleObject(info.hProcess, TIMEOUT_MS); + if (wait_result == WAIT_TIMEOUT) + { + fprintf(stderr, "Process timed out, terminating it.\n"); + ret = PROGRAM_RESULT_TIMEOUT; + + if (!TerminateProcess(info.hProcess, 1)) + { + fprintf(stderr, "Cannot terminate process, last error %ld.\n", GetLastError()); + goto out; + } + + wait_result = WaitForSingleObject(info.hProcess, INFINITE); + } + + if (wait_result != WAIT_OBJECT_0) + { + fprintf(stderr, "Cannot wait for process termination, last error %ld.\n", GetLastError()); + ret = PROGRAM_RESULT_FAILURE; + goto out; + } + + if (!GetExitCodeProcess(info.hProcess, &exit_code)) + { + fprintf(stderr, "Cannot retrive the process exit code, last error %ld.\n", GetLastError()); + ret = PROGRAM_RESULT_FAILURE; + goto out; + } + + ret = exit_code == 0 ? PROGRAM_RESULT_SUCCESS : PROGRAM_RESULT_FAILURE; + + printf("%s: %s\n", ret == PROGRAM_RESULT_SUCCESS ? "PASS" : "FAIL", cmdline); + +out: + if (info.hProcess && !CloseHandle(info.hProcess)) + fprintf(stderr, "Cannot close process, last error %ld.\n", GetLastError()); + if (info.hThread && !CloseHandle(info.hThread)) + fprintf(stderr, "Cannot close thread, last error %ld.\n", GetLastError()); + if (log != INVALID_HANDLE_VALUE && !CloseHandle(log)) + fprintf(stderr, "Cannot close log file, last error %ld.\n", GetLastError()); + + return ret; +} + +static bool run_tests_for_directory(const char *commit_dir) +{ + char cmdline[1024], log_filename[1024], list_filename[1024], line[1024]; + unsigned int success_count = 0, test_count = 0, timeout_count = 0; + const char *test_arch = getenv("TEST_ARCH"); + enum program_result result; + FILE *list_file; + bool ret = true; + + if (!test_arch) + test_arch = "64"; + + printf("Building %s\n", commit_dir); + printf("---\n"); + + sprintf(list_filename, "artifacts/%s/tests/crosstests.txt", commit_dir); + list_file = fopen(list_filename, "r"); + + if (!list_file) + { + fprintf(stderr, "Cannot open list file %s, errno %d.\n", list_filename, errno); + ret = false; + } + else + { + while (fgets(line, sizeof(line), list_file) && timeout_count < MAX_TIMEOUT_COUNT) + { + size_t len = strlen(line); + + if (line[len - 1] == '\n') + line[len - 1] = '\0'; + + sprintf(cmdline, "artifacts/%s/%s.cross%s.exe", commit_dir, line, test_arch); + sprintf(log_filename, "artifacts/%s/%s.log", commit_dir, line); + ++test_count; + result = run_program(cmdline, log_filename); + success_count += result == PROGRAM_RESULT_SUCCESS; + timeout_count += result == PROGRAM_RESULT_TIMEOUT; + } + + fclose(list_file); + } + + if (timeout_count >= MAX_TIMEOUT_COUNT) + fprintf(stderr, "Too many timeouts, aborting tests.\n"); + + printf("=======\n"); + printf("Summary\n"); + printf("=======\n"); + + printf("# TOTAL: %u\n", test_count); + printf("# PASS: %u\n", success_count); + printf("# FAIL: %u\n", test_count - success_count); + + if (test_count != success_count) + ret = false; + + return ret; +} + +int wmain(void) +{ + WIN32_FIND_DATAA find_data; + HANDLE find_handle; + bool ret = true; + + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX); + + find_handle = FindFirstFileA("artifacts/*-*", &find_data); + if (find_handle == INVALID_HANDLE_VALUE) + { + fprintf(stderr, "Cannot list commits, last error %ld.\n", GetLastError()); + ret = false; + } + else + { + do + { + ret &= run_tests_for_directory(find_data.cFileName); + } while (FindNextFileA(find_handle, &find_data)); + + if (GetLastError() != ERROR_NO_MORE_FILES) + { + fprintf(stderr, "Cannot list tests, last error %ld.\n", GetLastError()); + ret = false; + } + + FindClose(find_handle); + } + + return !ret; +} -- GitLab