I’m working on a program that imports memory from an external process into my application. The external process utilizes a Vulkan layer that creates a handle for the memory, which is then duplicated into my application outside the Vulkan layer. Although I can read the memory associated with that handle from my application, I’m unable to allocate any memory, regardless of my efforts. I consistently encounter the error VK_ERROR_OUT_OF_DEVICE_MEMORY. I’m not entirely sure why this is happening, as I don’t believe I’ve made any major mistakes with my handle. Below is a screenshot of both processes with the same handle (I believe).
However, if I map the memory from my application, I’m able to read it (the external process).
Successfully connected to the named pipe.
Read successful.
Handle value from the pipe: 0000000000000014
First 24 bytes of the section: 81 7a db 3b 81 7a db 3b 81 7a db 3b 81 7a db 3b 81 7a db 3b 81 7a db 3b
Memory Requirements - Size: 15728640, Alignment: 1024, allocInfo.memoryTypeIndex: 1, MemoryTypeBits: 3
vkAllocateMemory failed with error code: -2
VK_ERROR_OUT_OF_DEVICE_MEMORY
failed to allocate depth texture image memory!
I duplicate the handle inside of my Vulcan layer for the target application that I want to import the external memory from, and then I use pipes to notify that external application and pass the handle ID to it.
This is the code running on the client for context that imports the memory:
// create the depth texture image
if (vkCreateImage(device, &imageInfo, nullptr, &depthImage) != VK_SUCCESS) {
throw std::runtime_error("failed to create depth texture image!");
}
if (externalHandle == NULL) {
// import external memory
HANDLE hPipe = CreateFileA(
"\\\\.\\pipe\\vkpipe",
GENERIC_READ,
0,
nullptr,
OPEN_EXISTING,
0,
nullptr);
if (hPipe != INVALID_HANDLE_VALUE) {
std::cout << "Successfully connected to the named pipe." << std::endl;
// read the handle value from the pipe
DWORD bytesRead = 0;
BOOL result = ReadFile(
hPipe,
&externalHandle,
sizeof(HANDLE),
&bytesRead,
nullptr);
if (result && bytesRead == sizeof(HANDLE)) {
std::cout << "Read successful." << std::endl;
}
else {
std::cout << "Failed to read handle value from the pipe." << std::endl;
}
// print the value of the handle
std::cout << "Handle value from the pipe: " << externalHandle << std::endl;
// close the pipe handle
CloseHandle(hPipe);
}
else {
std::cout << "Failed to connect to the named pipe." << std::endl;
}
}
// map the memory
void* mappedData = MapViewOfFile(externalHandle, FILE_MAP_READ, 0, 0, 0);
if (mappedData == nullptr) {
DWORD err = GetLastError();
std::cerr << "failed to map view of file! GetLastError: " << err << std::endl;
throw std::runtime_error("failed to map view of file!");
}
unsigned char* byteData = static_cast<unsigned char*>(mappedData);
std::cout << "First 24 bytes of the section: ";
for (uint32_t i = 0; i < 24; i++) {
std::cout << std::hex << static_cast<int>(byteData[i]) << " ";
}
std::cout << std::dec << std::endl;
// wait for input
// std::cin.get();
VkImportMemoryWin32HandleInfoKHR importInfo = {};
importInfo.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_WIN32_HANDLE_INFO_KHR;
importInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
importInfo.handle = externalHandle;
VkExternalMemoryImageCreateInfo externalInfo{};
externalInfo.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
externalInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
// get the memory requirements for the image
VkMemoryRequirements memRequirements;
vkGetImageMemoryRequirements(device, depthImage, &memRequirements);
// Memory allocation info
VkMemoryAllocateInfo allocInfo = {};
allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
allocInfo.allocationSize = memRequirements.size;
allocInfo.memoryTypeIndex = findMemoryType(memRequirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
allocInfo.pNext = &importInfo;
// allocInfo.pNext = nullptr;
// print memory requirements
std::cout << "Memory Requirements - Size: " << memRequirements.size
<< ", Alignment: " << memRequirements.alignment
<< ", allocInfo.memoryTypeIndex: " << allocInfo.memoryTypeIndex
<< ", MemoryTypeBits: " << memRequirements.memoryTypeBits << std::endl;
VkResult result = vkAllocateMemory(device, &allocInfo, nullptr, &depthImageMemory);
if (result != VK_SUCCESS) {
std::cerr << "vkAllocateMemory failed with error code: " << result << std::endl;
if (result == -2) {
std::cerr << "VK_ERROR_OUT_OF_DEVICE_MEMORY" << std::endl;
}
throw std::runtime_error("failed to allocate depth texture image memory!");
}
This is the code responsible for exporting the handle and duplicating it in the target process (the Vulkan layer):
// existing code above this point
// export the buffer handle
if (g_stagingBufferMemory != VK_NULL_HANDLE)
{
VkMemoryGetWin32HandleInfoKHR handleInfo = {};
handleInfo.sType = VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR;
handleInfo.memory = g_stagingBufferMemory;
handleInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
auto GetMemoryWin32HandleKHR = device_dispatch[GetDeviceKey(g_Device)].GetMemoryWin32HandleKHR;
VkResult handleRes = GetMemoryWin32HandleKHR(g_Device, &handleInfo, &handle);
if (handleRes == VK_SUCCESS)
{
std::cout << "Staging buffer handle: " << handle << std::endl;
// print g_imageSize
std::cout << "g_imageSize: " << g_imageSize << std::endl;
}
else
{
std::cout << "Failed to get staging buffer handle. VkResult: " << handleRes << std::endl;
}
}
}
// find the target process
if (!processFound)
{
const wchar_t* targetModuleName = L"client.exe";
targetPid = GetProcessIdByName(targetModuleName);
if (targetPid == 0) {
// std::cout << "could not find process!" << std::endl;
} else {
std::cout << "Found process: " << targetPid << std::endl;
processFound = true;
}
}
if (handle != nullptr && processFound && !duplicatedHandleSet)
{
// duplicate the handle to the target process
HANDLE duplicatedHandle = NULL;
// open handle to target process
HANDLE targetPidHandle = OpenProcess(PROCESS_DUP_HANDLE, false, targetPid);
std::cout << "Target PID handle: " << targetPidHandle << std::endl;
// duplicate the handle
DWORD parentPid = GetCurrentProcessId();
std::cout << "Parent PID: " << parentPid << std::endl;
HANDLE parentProcessHandle = GetCurrentProcess();
BOOL dupRes = DuplicateHandle(
parentProcessHandle,
handle,
targetPidHandle,
&duplicatedHandle,
0,
FALSE,
DUPLICATE_SAME_ACCESS
);
if (!dupRes)
{
std::cout << "Failed to duplicate handle. Error: " << GetLastError() << std::endl;
}
else
{
std::cout << "Duplicated handle: " << duplicatedHandle << " for " << handle << std::endl;
g_duplicatedHandle = duplicatedHandle;
duplicatedHandleSet = true;
}
}
Any help is appreciated, and thanks in advance.