Windows: Make API formatting fallback to format.com in case of elevation. remove dead code from ExternalFormatFs

We also modify UacFormatNtfs/UacFormatFs to return actual error code in case of failure

src/Common/BaseCom.cpp

@@ -45,12 +45,13 @@ HRESULT CreateElevatedComObject (HWND hwnd, REFGUID guid, REFIID iid, void **ppv
 BOOL ComGetInstanceBase (HWND hWnd, REFCLSID clsid, REFIID iid, void **tcServer)
 {
 	BOOL r;
+	HRESULT hr;
 
 	if (IsUacSupported ())
 	{
 		while (true)
 		{
-			r = CreateElevatedComObject (hWnd, clsid, iid, tcServer) == S_OK;
+			r = (hr = CreateElevatedComObject (hWnd, clsid, iid, tcServer)) == S_OK;
 			if (r)
 				break;
 			else
@@ -64,11 +65,16 @@ BOOL ComGetInstanceBase (HWND hWnd, REFCLSID clsid, REFIID iid, void **tcServer)
 	}
 	else
 	{
-		r = CoCreateInstance (clsid, NULL, CLSCTX_LOCAL_SERVER, iid, tcServer) == S_OK;
+		r = (hr = CoCreateInstance (clsid, NULL, CLSCTX_LOCAL_SERVER, iid, tcServer)) == S_OK;
 		if (!r)
 			Error ("UAC_INIT_ERROR", hWnd);
 	}
 
+	if (!r)
+	{
+		SetLastError((DWORD) hr);
+	}
+
 	return r;
 }
 

src/Common/Format.c

@@ -799,18 +799,16 @@ error:
 		retCode = ExternalFormatFs (driveNo, volParams->clusterSize, fsType);
 		if (retCode != 0)
 		{
-			wchar_t auxLine[2048];
-			StringCbPrintfW (auxLine, sizeof(auxLine), GetString ("FORMAT_EXTERNAL_FAILED"), retCode);
-			WarningDirect(auxLine, volParams->hwndDlg);
 
 			/* fallback to using FormatEx function from fmifs.dll */
 			if (!Silent && !IsAdmin () && IsUacSupported ())
 				retCode = UacFormatFs (volParams->hwndDlg, driveNo, volParams->clusterSize, fsType);
 			else
-				retCode = FormatFs (driveNo, volParams->clusterSize, fsType);
+				retCode = FormatFs (driveNo, volParams->clusterSize, fsType, FALSE); /* no need to fallback to format.com since we have already tried it without elevation */
 			
 			if (retCode != 0)
 			{
+				wchar_t auxLine[2048];
 				StringCbPrintfW (auxLine, sizeof(auxLine), GetString ("FORMATEX_API_FAILED"), FormatExGetMessage(retCode));
 				ErrorDirect(auxLine, volParams->hwndDlg);
 			}
@@ -1143,7 +1141,7 @@ BOOLEAN __stdcall FormatExCallback (int command, DWORD subCommand, PVOID paramet
 	return (FormatExError? FALSE : TRUE);
 }
 
-int FormatFs (int driveNo, int clusterSize, int fsType)
+int FormatFs (int driveNo, int clusterSize, int fsType, BOOL bFallBackExternal)
 {
 	wchar_t dllPath[MAX_PATH] = {0};
 	WCHAR dir[8] = { (WCHAR) driveNo + L'A', 0 };
@@ -1204,28 +1202,29 @@ int FormatFs (int driveNo, int clusterSize, int fsType)
 	Sleep (4000);
 
 	FreeLibrary (hModule);
+
+	if (FormatExError && bFallBackExternal)
+	{
+		return ExternalFormatFs (driveNo, clusterSize, fsType);
+	}
+
 	return FormatExError? FormatExErrorCommand : 0;
 }
 
-int FormatNtfs (int driveNo, int clusterSize)
+int FormatNtfs (int driveNo, int clusterSize, BOOL bFallBackExternal)
 {
-	return FormatFs (driveNo, clusterSize, FILESYS_NTFS);
+	return FormatFs (driveNo, clusterSize, FILESYS_NTFS, bFallBackExternal);
 }
 
 /* call Windows format.com program to perform formatting */
 int ExternalFormatFs (int driveNo, int clusterSize, int fsType)
 {
 	wchar_t exePath[MAX_PATH] = {0};
-	HANDLE hChildStd_IN_Rd = NULL;
-	HANDLE hChildStd_IN_Wr = NULL;
-	HANDLE hChildStd_OUT_Rd = NULL;
-	HANDLE hChildStd_OUT_Wr = NULL;
 	WCHAR szFsFormat[16];
 	TCHAR szCmdline[2 * MAX_PATH];
 	STARTUPINFO siStartInfo;
 	PROCESS_INFORMATION piProcInfo;
 	BOOL bSuccess = FALSE; 
-	SECURITY_ATTRIBUTES saAttr; 
 	int iRet = 0;
 
 	switch (fsType)
@@ -1243,35 +1242,6 @@ int ExternalFormatFs (int driveNo, int clusterSize, int fsType)
 			return FALSE;
 	}
 
-	/* Set the bInheritHandle flag so pipe handles are inherited.  */
-	saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); 
-	saAttr.bInheritHandle = TRUE; 
-	saAttr.lpSecurityDescriptor = NULL; 
-
-	/* Create a pipe for the child process's STDOUT. */
-	if ( !CreatePipe(&hChildStd_OUT_Rd, &hChildStd_OUT_Wr, &saAttr, 0) ) 
-		return FALSE;
-
-	/* Ensure the read handle to the pipe for STDOUT is not inherited. */
-	/* Create a pipe for the child process's STDIN.  */ 
-	if (	!SetHandleInformation(hChildStd_OUT_Rd, HANDLE_FLAG_INHERIT, 0) 
-		||	!CreatePipe(&hChildStd_IN_Rd, &hChildStd_IN_Wr, &saAttr, 0))
-	{
-		CloseHandle (hChildStd_OUT_Rd);
-		CloseHandle (hChildStd_OUT_Wr);
-		return FALSE;
-	}
-
-	/* Ensure the write handle to the pipe for STDIN is not inherited. */ 
-	if ( !SetHandleInformation(hChildStd_IN_Wr, HANDLE_FLAG_INHERIT, 0))
-	{
-		CloseHandle (hChildStd_OUT_Rd);
-		CloseHandle (hChildStd_OUT_Wr);
-		CloseHandle (hChildStd_IN_Rd);
-		CloseHandle (hChildStd_IN_Wr);
-		return FALSE;
-	}
-
 	if (GetSystemDirectory (exePath, MAX_PATH))
 	{
 		StringCchCatW(exePath, ARRAYSIZE(exePath), L"\\format.com");
@@ -1302,15 +1272,11 @@ int ExternalFormatFs (int driveNo, int clusterSize, int fsType)
    ZeroMemory( &piProcInfo, sizeof(PROCESS_INFORMATION) ); 
 
    /* Set up members of the STARTUPINFO structure. 
-	  This structure specifies the STDIN and STDOUT handles for redirection.
 	*/ 
    ZeroMemory( &siStartInfo, sizeof(STARTUPINFO) );
    siStartInfo.cb = sizeof(STARTUPINFO); 
-   siStartInfo.hStdError = hChildStd_OUT_Wr;
-   siStartInfo.hStdOutput = hChildStd_OUT_Wr;
-   siStartInfo.hStdInput = hChildStd_IN_Rd;
    siStartInfo.wShowWindow = SW_HIDE;
-   siStartInfo.dwFlags |= STARTF_USESTDHANDLES | STARTF_USESHOWWINDOW;
+   siStartInfo.dwFlags |= STARTF_USESHOWWINDOW;
  
    /* Create the child process.      */
    bSuccess = CreateProcess(NULL, 
@@ -1347,11 +1313,6 @@ int ExternalFormatFs (int driveNo, int clusterSize, int fsType)
 	   iRet = (int) GetLastError();
    }
 
-	CloseHandle(hChildStd_OUT_Wr);
-	CloseHandle(hChildStd_OUT_Rd);
-	CloseHandle(hChildStd_IN_Rd);
-	CloseHandle(hChildStd_IN_Wr);
-
    return iRet;
 }
 

src/Format/FormatCom.cpp

@@ -92,7 +92,7 @@ public:
 
 	virtual BOOL STDMETHODCALLTYPE FormatNtfs (int driveNo, int clusterSize)
 	{
-		return ::FormatNtfs (driveNo, clusterSize);
+		return ::FormatNtfs (driveNo, clusterSize, TRUE);
 	}
 
 	virtual int STDMETHODCALLTYPE AnalyzeHiddenVolumeHost (
@@ -134,7 +134,7 @@ public:
 
 	virtual BOOL STDMETHODCALLTYPE FormatFs (int driveNo, int clusterSize, int fsType)
 	{
-		return ::FormatFs (driveNo, clusterSize, fsType);
+		return ::FormatFs (driveNo, clusterSize, fsType, TRUE);
 	}
 
 	virtual DWORD STDMETHODCALLTYPE GetFileSize (BSTR filePath, unsigned __int64 *pSize)
@@ -250,7 +250,7 @@ extern "C" int UacFormatNtfs (HWND hWnd, int driveNo, int clusterSize)
 	if (ComGetInstance (hWnd, &tc))
 		r = tc->FormatNtfs (driveNo, clusterSize);
 	else
-		r = 0;
+		r = (int) GetLastError();
 
 	CoUninitialize ();
 
@@ -267,7 +267,7 @@ extern "C" int UacFormatFs (HWND hWnd, int driveNo, int clusterSize, int fsType)
 	if (ComGetInstance (hWnd, &tc))
 		r = tc->FormatFs (driveNo, clusterSize, fsType);
 	else
-		r = 0;
+		r = (int) GetLastError();
 
 	CoUninitialize ();