Merge pull request #1437 from elicn/dev-win

Windows bug fixes

qiling/os/utils.py

@@ -153,7 +153,10 @@ def __dup(iterator: Iterator[T], out: List[T]) -> Iterator[T]:
 
         va_list = __dup(va_args, orig_args)
 
-        read_string = self.read_wstring if wstring else self.read_cstring
+        read_str = {
+            False: self.read_cstring,
+            True:  self.read_wstring
+        }
 
         def __repl(m: re.Match) -> str:
             """Convert printf format string tokens into Python's.
@@ -187,17 +190,21 @@ def __repl(m: re.Match) -> str:
                 typ = m['type']
                 arg = next(va_list)
 
-                if typ in 'sS':
+                if typ == 's':
                     typ = 's'
-                    arg = read_string(arg)
+                    arg = read_str[wstring](arg)
+
+                elif typ == 'S':
+                    typ = 's'
+                    arg = read_str[not wstring](arg)
 
                 elif typ == 'Z':
                     # note: ANSI_STRING and UNICODE_STRING have identical layout
                     ucstr_struct = make_unicode_string(self.ql.arch.bits)
 
                     with ucstr_struct.ref(self.ql.mem, arg) as ucstr_obj:
                         typ = 's'
-                        arg = read_string(ucstr_obj.Buffer)
+                        arg = read_str[wstring](ucstr_obj.Buffer)
 
                 elif typ == 'p':
                     pound = '#'

qiling/os/windows/const.py

@@ -786,3 +786,10 @@
 TIME_ZONE_ID_UNKNOWN  = 0
 TIME_ZONE_ID_STANDARD = 1
 TIME_ZONE_ID_DAYLIGHT = 2
+
+# heap management flags
+HEAP_NO_SERIALIZE = 0x00000001
+HEAP_GENERATE_EXCEPTIONS = 0x00000004
+HEAP_ZERO_MEMORY = 0x00000008
+HEAP_REALLOC_IN_PLACE_ONLY = 0x00000010
+HEAP_CREATE_ENABLE_EXECUTE = 0x00040000

qiling/os/windows/dlls/kernel32/fileapi.py

@@ -3,18 +3,20 @@
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 #
 
+from contextlib import contextmanager
 import ntpath
 import os
 
 from shutil import copyfile
 from datetime import datetime
+from typing import IO, Optional
 
 from qiling import Qiling
 from qiling.exception import QlErrorNotImplemented
 from qiling.os.windows.api import *
 from qiling.os.windows.const import *
 from qiling.os.windows.fncc import *
-from qiling.os.windows.handle import Handle
+from qiling.os.windows.handle import Handle, HandleManager
 from qiling.os.windows.structs import FILETIME, make_win32_find_data
 
 # DWORD GetFileType(
@@ -604,14 +606,124 @@ def hook_GetFileSize(ql: Qiling, address: int, params):
         ql.os.last_error = ERROR_INVALID_HANDLE
         return -1 # INVALID_FILE_SIZE
 
+
+class FileMapping:
+    pass
+
+
+class FileMappingMem(FileMapping):
+    # mapping backed my page file, for which we simply use memory. no need to do anything really
+    pass
+
+
+class FileMappingFile(FileMapping):
+    def __init__(self, fobj: IO) -> None:
+        self._fobj = fobj
+
+        self._read_hook = None
+        self._write_hook = None
+
+    def map_view(self, ql: Qiling, fbase: int, lbound: int, ubound: int) -> None:
+        def __read_mapview(ql: Qiling, access: int, addr: int, size: int, _) -> None:
+            """Fetch the corresponding file part into memory.
+            """
+
+            data = self.read(fbase + (addr - lbound), size)
+
+            # FIXME: that triggers the write hook, and may be problematic for read-only ranges
+            ql.mem.write(addr, data)
+
+        def __write_mapview(ql: Qiling, access: int, addr: int, size: int, value: int) -> None:
+            """Write data back to the corresponding file part.
+            """
+
+            pack = {
+                1: ql.pack8,
+                2: ql.pack16,
+                4: ql.pack32,
+                8: ql.pack64
+            }[size]
+
+            self.write(fbase + (addr - lbound), pack(value))
+
+        self._read_hook = ql.hook_mem_read(__read_mapview, begin=lbound, end=ubound)
+        self._write_hook = ql.hook_mem_write(__write_mapview, begin=lbound, end=ubound)
+
+    def unmap_view(self) -> None:
+        if self._read_hook:
+            self._read_hook.remove()
+
+        if self._write_hook:
+            self._write_hook.remove()
+
+    @contextmanager
+    def __seek_temporary(self, offset: Optional[int] = None):
+        """A context manager construct for performing actions that would normaly affect the file
+        position, but without actually affecting it.
+        """
+
+        fpos = self._fobj.tell()
+
+        if offset is not None:
+            self._fobj.seek(offset)
+
+        try:
+            yield self._fobj
+        finally:
+            self._fobj.seek(fpos)
+
+    def get_file_size(self) -> int:
+        with self.__seek_temporary() as fobj:
+            return fobj.seek(0, os.SEEK_END)
+
+    def inc_file_size(self, addendum: int) -> None:
+        with self.__seek_temporary() as fobj:
+            fobj.seek(0, os.SEEK_END)
+            fobj.write(b'\x00' * addendum)
+
+    def read(self, offset: int, size: int) -> bytes:
+        with self.__seek_temporary(offset) as fobj:
+            return fobj.read(size)
+
+    def write(self, offset: int, data: bytes) -> None:
+        with self.__seek_temporary(offset) as fobj:
+            fobj.write(data)
+
+
 def _CreateFileMapping(ql: Qiling, address: int, params):
     hFile = params['hFile']
+    dwMaximumSizeHigh = params['dwMaximumSizeHigh']
+    dwMaximumSizeLow = params['dwMaximumSizeLow']
     lpName = params['lpName']
 
-    new_handle = Handle(obj=hFile, name=lpName)
-    ql.os.handle_manager.append(new_handle)
+    req_size = (dwMaximumSizeHigh << 32) | dwMaximumSizeLow
 
-    return new_handle.id
+    if hFile == ql.unpack(ql.packs(INVALID_HANDLE_VALUE)):
+        fmobj = FileMappingMem()
+
+    else:
+        # look for an existing mapping handle with the same name
+        if lpName:
+            existing_handle = ql.os.handle_manager.search(lpName)
+
+            # if found, return it
+            if existing_handle is not None:
+                return existing_handle
+
+        fhandle = ql.os.handle_manager.get(hFile)
+
+        # wrap the opened file with an accessor class
+        fmobj = FileMappingFile(fhandle.obj)
+        fsize = fmobj.get_file_size()
+
+        # if requeted mapping is size is larger than the file size, enlarge it
+        if req_size > fsize:
+            fmobj.inc_file_size(req_size - fsize)
+
+    fm_handle = Handle(obj=fmobj, name=lpName or None)
+    ql.os.handle_manager.append(fm_handle)
+
+    return fm_handle.id
 
 # HANDLE CreateFileMappingA(
 #   HANDLE                hFile,
@@ -667,29 +779,49 @@ def hook_CreateFileMappingW(ql: Qiling, address: int, params):
 })
 def hook_MapViewOfFile(ql: Qiling, address: int, params):
     hFileMappingObject = params['hFileMappingObject']
+    dwFileOffsetHigh = params['dwFileOffsetHigh']
     dwFileOffsetLow = params['dwFileOffsetLow']
     dwNumberOfBytesToMap = params['dwNumberOfBytesToMap']
 
-    map_file_handle = ql.os.handle_manager.search_by_obj(hFileMappingObject)
+    handles: HandleManager = ql.os.handle_manager
+    fm_handle = handles.get(hFileMappingObject)
+
+    if fm_handle is None:
+        return 0
+
+    fmobj = fm_handle.obj
+
+    # the respective file mapping hFile was set to INVALID_HANDLE_VALUE (that is, mapping is backed by page file)
+    if isinstance(fmobj, FileMappingMem):
+        mapview = ql.os.heap.alloc(dwNumberOfBytesToMap)
+
+        if not mapview:
+            return 0
 
-    if map_file_handle is None:
-        ret = ql.os.heap.alloc(dwNumberOfBytesToMap)
-        new_handle = Handle(obj=hFileMappingObject, name=ret)
-        ql.os.handle_manager.append(new_handle)
     else:
-        ret = map_file_handle.name
+        offset = (dwFileOffsetHigh << 32) | dwFileOffsetLow
+        mapview_size = dwNumberOfBytesToMap or (fmobj.get_file_size() - offset)
+
+        if mapview_size < 1:
+            return 0
 
-    hFile = ql.os.handle_manager.get(hFileMappingObject).obj
+        mapview = ql.os.heap.alloc(mapview_size)
 
-    if ql.os.handle_manager.get(hFile):
-        f = ql.os.handle_manager.get(hFile).obj
+        if not mapview:
+            return 0
 
-        if type(f) is file:
-            f.seek(dwFileOffsetLow, 0)
-            data = f.read(dwNumberOfBytesToMap)
-            ql.mem.write(ret, data)
+        # read content from file but retain original position.
+        # not sure this is actually required since all accesses to this memory area are monitored
+        # and relect file content rather than what is currently in memory
+        data = fmobj.read(offset, mapview_size)
+        ql.mem.write(mapview, data)
 
-    return ret
+        fmobj.map_view(ql, offset, mapview, mapview + mapview_size - 1)
+
+    # although file views are not strictly handles, it would be easier to manage them as such
+    handles.append(Handle(id=mapview, obj=fmobj))
+
+    return mapview
 
 # BOOL UnmapViewOfFile(
 #   LPCVOID lpBaseAddress
@@ -700,15 +832,19 @@ def hook_MapViewOfFile(ql: Qiling, address: int, params):
 def hook_UnmapViewOfFile(ql: Qiling, address: int, params):
     lpBaseAddress = params['lpBaseAddress']
 
-    map_file_hande = ql.os.handle_manager.search(lpBaseAddress)
+    handles: HandleManager = ql.os.handle_manager
+    fv_handle = handles.get(lpBaseAddress)
 
-    if not map_file_hande:
-        return 0
+    if fv_handle:
+        if isinstance(fv_handle.obj, FileMappingFile):
+            fv_handle.obj.unmap_view()
 
-    ql.os.heap.free(map_file_hande.name)
-    ql.os.handle_manager.delete(map_file_hande.id)
+        ql.os.heap.free(lpBaseAddress)
+        handles.delete(fv_handle.id)
 
-    return 1
+        return 1
+
+    return 0
 
 # BOOL CopyFileA(
 #   LPCSTR lpExistingFileName,

qiling/os/windows/dlls/kernel32/handleapi.py

@@ -3,10 +3,18 @@
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 #
 
-from qiling import Qiling
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, IO
+
 from qiling.os.windows.api import *
-from qiling.os.windows.const import *
-from qiling.os.windows.fncc import *
+from qiling.os.windows.const import ERROR_INVALID_HANDLE, HANDLE_FLAG_PROTECT_FROM_CLOSE
+from qiling.os.windows.fncc import STDCALL, winsdkapi
+
+
+if TYPE_CHECKING:
+    from qiling import Qiling
+
 
 # BOOL DuplicateHandle(
 #   HANDLE   hSourceProcessHandle,
@@ -53,6 +61,10 @@ def hook_CloseHandle(ql: Qiling, address: int, params):
         # FIXME: add error
         return 0
 
+    # if this a file handle, close it
+    if isinstance(handle.obj, IO):
+        handle.obj.close()
+
     ql.os.handle_manager.delete(value)
 
     return 1

qiling/os/windows/dlls/kernel32/heapapi.py

@@ -3,9 +3,36 @@
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 #
 
-from qiling import Qiling
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
 from qiling.os.windows.api import *
-from qiling.os.windows.fncc import *
+from qiling.os.windows.const import HEAP_ZERO_MEMORY
+from qiling.os.windows.fncc import STDCALL, winsdkapi
+
+
+if TYPE_CHECKING:
+    from qiling import Qiling
+    from qiling.os.memory import QlMemoryManager
+
+
+def __zero_mem(mem: QlMemoryManager, ptr: int, size: int) -> None:
+    """Zero a memory range, but avoid hogging to much on host resources.
+    """
+
+    # go by page granularity
+    npages, remainder = divmod(size, mem.pagesize)
+
+    if npages:
+        zeros = b'\x00' * mem.pagesize
+
+        for _ in range(npages):
+            mem.write(ptr, zeros)
+            ptr += len(zeros)
+
+    if remainder:
+        mem.write(ptr, b'\x00' * remainder)
+
 
 # HANDLE HeapCreate(
 #   DWORD  flOptions,
@@ -33,9 +60,15 @@ def hook_HeapCreate(ql: Qiling, address: int, params):
     'dwBytes' : SIZE_T
 })
 def hook_HeapAlloc(ql: Qiling, address: int, params):
+    dwFlags = params["dwFlags"]
     dwBytes = params["dwBytes"]
 
-    return ql.os.heap.alloc(dwBytes)
+    ptr = ql.os.heap.alloc(dwBytes)
+
+    if ptr and (dwFlags & HEAP_ZERO_MEMORY):
+        __zero_mem(ql.mem, ptr, dwBytes)
+
+    return ptr
 
 # SIZE_T HeapSize(
 #   HANDLE  hHeap,