Merge pull request #10 from otavepto/patch-steam-http

Fixes for `Steam_Http` class

dll/dll/steam_http.h

@@ -30,12 +30,15 @@ struct Steam_Http_Request {
 	bool requires_valid_ssl = false;
 
 	constexpr const static char STEAM_DEFAULT_USER_AGENT[] = "Valve/Steam HTTP Client 1.0";
-	// GET or POST parameter value on the request
+	// check Steam_HTTP::SetHTTPRequestHeaderValue() and make sure to bypass the ones that should be reserved
 	std::map<std::string, std::string> headers{
-		{"User-Agent", STEAM_DEFAULT_USER_AGENT},
+		{ "User-Agent", STEAM_DEFAULT_USER_AGENT },
+		{ "Cache-Control", "max-age=0" },
+		{ "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7" },
+		{ "Upgrade-Insecure-Requests", "1" },
 	};
 
-	// GET or POST parameter value on the request
+	// GET or POST parameter value of the request
 	std::map<std::string, std::string> get_or_post_params{};
 	std::string post_raw{};
 
@@ -64,7 +67,7 @@ public ISteamHTTP
 	std::vector<Steam_Http_Request> requests{};
 
 	Steam_Http_Request *get_request(HTTPRequestHandle hRequest);
-	void online_http_request(Steam_Http_Request *request, SteamAPICall_t *pCallHandle);
+	void online_http_request(Steam_Http_Request *request, SteamAPICall_t call_res_id);
 
 public:
 	Steam_HTTP(class Settings *settings, class Networking *network, class SteamCallResults *callback_results, class SteamCallBacks *callbacks);

dll/network.cpp

@@ -249,7 +249,7 @@ static void run_at_startup()
         return;
     }
 #if defined(STEAM_WIN32)
-    WSADATA wsaData;
+    WSADATA wsaData{};
     if (WSAStartup(MAKEWORD(2, 2), &wsaData) != NO_ERROR) {
         PRINT_DEBUG("Networking WSAStartup error");
         return;
@@ -257,8 +257,13 @@ static void run_at_startup()
 
     for (int i = 0; i < 10; ++i) {
         //hack: the game Full Mojo Rampage calls WSACleanup on startup so we call WSAStartup a few times so it doesn't get deallocated.
-        WSAStartup(MAKEWORD(2, 2), &wsaData);
+        WSADATA wsaData{};
+        if (WSAStartup(MAKEWORD(2, 2), &wsaData) != NO_ERROR) {
+            PRINT_DEBUG("Networking WSAStartup error");
+            return;
+        }
     }
+    PRINT_DEBUG("Networking WSAStartup success!");
 #else
 
 #endif

dll/steam_http.cpp

@@ -38,7 +38,7 @@ Steam_Http_Request *Steam_HTTP::get_request(HTTPRequestHandle hRequest)
 // or such.
 HTTPRequestHandle Steam_HTTP::CreateHTTPRequest( EHTTPMethod eHTTPRequestMethod, const char *pchAbsoluteURL )
 {
-    PRINT_DEBUG("%i %s", eHTTPRequestMethod, pchAbsoluteURL);
+    PRINT_DEBUG("%i '%s'", eHTTPRequestMethod, pchAbsoluteURL);
     std::lock_guard<std::recursive_mutex> lock(global_mutex);
 
     if (!pchAbsoluteURL) return INVALID_HTTPREQUEST_HANDLE;
@@ -83,7 +83,7 @@ HTTPRequestHandle Steam_HTTP::CreateHTTPRequest( EHTTPMethod eHTTPRequestMethod,
 // sending the request.  This is just so the caller can easily keep track of which callbacks go with which request data.
 bool Steam_HTTP::SetHTTPRequestContextValue( HTTPRequestHandle hRequest, uint64 ulContextValue )
 {
-    PRINT_DEBUG_ENTRY();
+    PRINT_DEBUG("%llu", ulContextValue);
     std::lock_guard<std::recursive_mutex> lock(global_mutex);
 
     Steam_Http_Request *request = get_request(hRequest);
@@ -122,15 +122,19 @@ bool Steam_HTTP::SetHTTPRequestHeaderValue( HTTPRequestHandle hRequest, const ch
     std::lock_guard<std::recursive_mutex> lock(global_mutex);
 
     if (!pchHeaderName || !pchHeaderValue) return false;
-    std::string headerName(pchHeaderName);
+    if (common_helpers::str_cmp_insensitive(pchHeaderName, "User-Agent")) return false;
-    std::transform(headerName.begin(), headerName.end(), headerName.begin(), [](char c){ return (char)std::toupper(c); });
-    if (headerName == "USER-AGENT") return false;
 
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
     }
 
+    // FIX: appid 1902490 adds the header "Cache-Control: only-if-cached, max-stale=2678400"
+    // which means a response is returned back only if it was already cached, otherwise the server has to send a 504 "Gateway Timeout"
+    // just bypass the known ones to be on the safe side
+    if (common_helpers::str_cmp_insensitive(pchHeaderName, "Cache-Control")) return true;
+    if (common_helpers::str_cmp_insensitive(pchHeaderName, "Accept")) return true;
+
     request->headers[pchHeaderName] = pchHeaderValue;
     return true;
 }
@@ -160,13 +164,37 @@ bool Steam_HTTP::SetHTTPRequestGetOrPostParameter( HTTPRequestHandle hRequest, c
     return true;
 }
 
+static int curl_debug_trace(
+    CURL *handle, curl_infotype type,
+    char *data, size_t size,
+    void *clientp
+)
+{
+    // https://curl.se/libcurl/c/CURLOPT_DEBUGFUNCTION.html
+    std::string text{};
+    switch (type) {
+        case CURLINFO_TEXT: text = "Info: " + std::string(data, size); break;
+        case CURLINFO_HEADER_IN: text = "<= Recv header"; break;
+        case CURLINFO_HEADER_OUT: text = "=> Send header"; break;
+        case CURLINFO_DATA_IN: text = "<= Recv data"; break;
+        case CURLINFO_DATA_OUT: text = "=> Send data"; break;
+        case CURLINFO_SSL_DATA_OUT: text = "=> Send SSL data"; break;
+        case CURLINFO_SSL_DATA_IN: text = "<= Recv SSL data"; break;
 
-void Steam_HTTP::online_http_request(Steam_Http_Request *request, SteamAPICall_t *pCallHandle)
+        default: text = "[X] ERROR: unknown callback type"; break;
+    }
+
+    PRINT_DEBUG("%s", text.c_str());
+    return 0;
+}
+
+
+void Steam_HTTP::online_http_request(Steam_Http_Request *request, SteamAPICall_t call_res_id)
 {
     PRINT_DEBUG("attempting to download from url: '%s', target filepath: '%s'",
         request->url.c_str(), request->target_filepath.c_str());
 
-    const auto send_callresult = [&]() -> void {
+    const auto send_callresult = [=]() -> void {
         struct HTTPRequestCompleted_t data{};
         data.m_hRequest = request->handle;
         data.m_ulContextValue = request->context_value;
@@ -174,16 +202,13 @@ void Steam_HTTP::online_http_request(Steam_Http_Request *request, SteamAPICall_t
         if (request->response.empty() && !settings->force_steamhttp_success) {
             data.m_bRequestSuccessful = false;
             data.m_eStatusCode = k_EHTTPStatusCode404NotFound;
-            
         } else {
             data.m_bRequestSuccessful = true;
             data.m_eStatusCode = k_EHTTPStatusCode200OK;
         }
 
-        auto callres = callback_results->addCallResult(data.k_iCallback, &data, sizeof(data), 0.1);
+        callback_results->addCallResult(call_res_id, data.k_iCallback, &data, sizeof(data));
-        if (pCallHandle) *pCallHandle = callres;
+        callbacks->addCBResult(data.k_iCallback, &data, sizeof(data));
-
-        callbacks->addCBResult(data.k_iCallback, &data, sizeof(data), 0.1);
     };
 
     std::size_t filename_part = request->target_filepath.find_last_of("\\/");
@@ -213,6 +238,11 @@ void Steam_HTTP::online_http_request(Steam_Http_Request *request, SteamAPICall_t
         send_callresult();
         return;
     }
+
+#ifndef EMU_RELEASE_BUILD
+    curl_easy_setopt(chttp, CURLOPT_DEBUGFUNCTION, curl_debug_trace);
+    curl_easy_setopt(chttp, CURLOPT_VERBOSE, 1L);
+#endif
     
     // headers
     std::vector<std::string> headers{};
@@ -279,6 +309,7 @@ void Steam_HTTP::online_http_request(Steam_Http_Request *request, SteamAPICall_t
     curl_easy_setopt(chttp, CURLOPT_TIMEOUT, request->timeout_sec);
     curl_easy_setopt(chttp, CURLOPT_NOSIGNAL, 1L);
     curl_easy_setopt(chttp, CURLOPT_USE_SSL, request->requires_valid_ssl ? CURLUSESSL_TRY : CURLUSESSL_NONE);
+    curl_easy_setopt(chttp, CURLOPT_SSL_VERIFYPEER, 0L);
 
     // post data, or get params
     std::string post_data{};
@@ -314,11 +345,12 @@ void Steam_HTTP::online_http_request(Steam_Http_Request *request, SteamAPICall_t
     fclose(hfile);
     headers.clear();
 
-    PRINT_DEBUG("CURL error code for '%s' [%i] (OK == 0)", request->url.c_str(), (int)res_curl);
+    PRINT_DEBUG("CURL error code for '%s' [%i = '%s'] (OK == 0)", request->url.c_str(), (int)res_curl, curl_easy_strerror(res_curl));
     
     unsigned int file_size = file_size_(request->target_filepath);
     if (file_size) {
-        long long read = Local_Storage::get_file_data(request->target_filepath, (char *)&request->response[0], file_size, 0);
+        request->response.resize(static_cast<size_t>(file_size));
+        long long read = Local_Storage::get_file_data(request->target_filepath, (char *)&request->response[0], file_size);
         if (read < 0) read = 0;
         request->response.resize(static_cast<size_t>(read));
     }
@@ -343,7 +375,10 @@ bool Steam_HTTP::SendHTTPRequest( HTTPRequestHandle hRequest, SteamAPICall_t *pC
 
     if (request->response.empty() && request->target_filepath.size() &&
         !settings->disable_networking && settings->download_steamhttp_requests) {
-        std::thread(&Steam_HTTP::online_http_request, this, request, pCallHandle).detach();
+        auto call_res_id = callback_results->reserveCallResult();
+        if (pCallHandle) *pCallHandle = call_res_id;
+
+        std::thread(&Steam_HTTP::online_http_request, this, request, call_res_id).detach();
     } else {
         struct HTTPRequestCompleted_t data{};
         data.m_hRequest = request->handle;
@@ -352,7 +387,6 @@ bool Steam_HTTP::SendHTTPRequest( HTTPRequestHandle hRequest, SteamAPICall_t *pC
         if (request->response.empty() && !settings->force_steamhttp_success) {
             data.m_bRequestSuccessful = false;
             data.m_eStatusCode = k_EHTTPStatusCode404NotFound;
-            
         } else {
             data.m_bRequestSuccessful = true;
             data.m_eStatusCode = k_EHTTPStatusCode200OK;

premake5-deps.lua

@@ -498,23 +498,41 @@ end
 -- https://github.com/Kitware/CMake/blob/a6853135f569f0b040a34374a15a8361bb73901b/Modules/FindZLIB.cmake#L98C4-L98C13
 
 local zlib_name = ''
+local mbedtls_name = ''
+local mbedcrypto_name = ''
+local mbedx509_name = ''
 -- name
 if _ACTION and os.target() == 'windows' then
     if string.match(_ACTION, 'vs.+') then
         zlib_name = 'zlibstatic'
+        mbedtls_name = 'mbedtls'
+        mbedcrypto_name = 'mbedcrypto'
+        mbedx509_name = 'mbedx509'
     elseif string.match(_ACTION, 'gmake.*') then
         zlib_name = 'libzlibstatic'
+        mbedtls_name = 'libmbedtls'
+        mbedcrypto_name = 'libmbedcrypto'
+        mbedx509_name = 'libmbedx509'
     else
         error('unsupported os/action: ' .. os.target() .. ' / ' .. _ACTION)
     end
 else -- linux or macos
     zlib_name = 'libz'
+    mbedtls_name = 'libmbedtls'
+    mbedcrypto_name = 'libmbedcrypto'
+    mbedx509_name = 'mbedx509'
 end
 -- extension
 if _ACTION and string.match(_ACTION, 'vs.+') then
     zlib_name = zlib_name .. '.lib'
+    mbedtls_name = mbedtls_name .. '.lib'
+    mbedcrypto_name = mbedcrypto_name .. '.lib'
+    mbedx509_name = mbedx509_name .. '.lib'
 else
     zlib_name = zlib_name .. '.a'
+    mbedtls_name = mbedtls_name .. '.a'
+    mbedcrypto_name = mbedcrypto_name .. '.a'
+    mbedx509_name = mbedx509_name .. '.a'
 end
 
 local wild_zlib_path_32 = path.join(deps_dir, 'zlib', 'install32', 'lib', zlib_name)
@@ -532,18 +550,55 @@ local wild_zlib_64 = {
     'ZLIB_LIBRARY="' .. wild_zlib_path_64 .. '"',
 }
 
+if _OPTIONS["build-mbedtls"] or _OPTIONS["all-build"] then
+    local mbedtls_common_defs = {
+        "USE_STATIC_MBEDTLS_LIBRARY=ON",
+        "USE_SHARED_MBEDTLS_LIBRARY=OFF",
+        "ENABLE_TESTING=OFF",
+        "ENABLE_PROGRAMS=OFF",
+        "MBEDTLS_FATAL_WARNINGS=OFF",
+    }
+    if os.target() == 'windows' and string.match(_ACTION, 'vs.+') then
+        table.insert(mbedtls_common_defs, "MSVC_STATIC_RUNTIME=ON")
+    else -- linux or macos or MinGW on Windows
+        table.insert(mbedtls_common_defs, "LINK_WITH_PTHREAD=ON")
+    end
+
+    local mbedtls_32_bit_fixes = {}
+    if _OPTIONS["32-build"] and string.match(_ACTION, 'gmake.*') then
+        table.insert(mbedtls_32_bit_fixes, '-mpclmul')
+        table.insert(mbedtls_32_bit_fixes, '-msse2')
+        table.insert(mbedtls_32_bit_fixes, '-maes')
+    end
+
+    if _OPTIONS["32-build"] then
+        cmake_build('mbedtls', true, mbedtls_common_defs, mbedtls_32_bit_fixes)
+    end
+    if _OPTIONS["64-build"] then
+        cmake_build('mbedtls', false, mbedtls_common_defs)
+    end
+end
+
 if _OPTIONS["build-curl"] or _OPTIONS["all-build"] then
     local curl_common_defs = {
         "BUILD_CURL_EXE=OFF",
-        "BUILD_SHARED_LIBS=OFF",
         "BUILD_STATIC_CURL=OFF", -- "Build curl executable with static libcurl"
+
+        "BUILD_SHARED_LIBS=OFF",
         "BUILD_STATIC_LIBS=ON",
         "BUILD_MISC_DOCS=OFF",
         "BUILD_TESTING=OFF",
         "BUILD_LIBCURL_DOCS=OFF",
         "ENABLE_CURL_MANUAL=OFF",
+
         "CURL_USE_OPENSSL=OFF",
         "CURL_ZLIB=ON",
+        
+        "CURL_USE_MBEDTLS=ON",
+        -- "CURL_USE_SCHANNEL=ON",
+        "CURL_CA_FALLBACK=ON",
+
+        -- fix building on Arch Linux
         "CURL_USE_LIBSSH2=OFF",
         "CURL_USE_LIBPSL=OFF",
         "USE_LIBIDN2=OFF",
@@ -555,10 +610,20 @@ if _OPTIONS["build-curl"] or _OPTIONS["all-build"] then
     end
 
     if _OPTIONS["32-build"] then
-        cmake_build('curl', true, merge_list(curl_common_defs, wild_zlib_32))
+        cmake_build('curl', true, merge_list(curl_common_defs, merge_list(wild_zlib_32, {
+            'MBEDTLS_INCLUDE_DIRS="' .. path.join(deps_dir, 'mbedtls', 'install32', 'include') .. '"',
+            'MBEDTLS_LIBRARY="' .. path.join(deps_dir, 'mbedtls', 'install32', 'lib', mbedtls_name) .. '"',
+            'MBEDCRYPTO_LIBRARY="' .. path.join(deps_dir, 'mbedtls', 'install32', 'lib', mbedcrypto_name) .. '"',
+            'MBEDX509_LIBRARY="' .. path.join(deps_dir, 'mbedtls', 'install32', 'lib', mbedx509_name) .. '"',
+        })))
     end
     if _OPTIONS["64-build"] then
-        cmake_build('curl', false, merge_list(curl_common_defs, wild_zlib_64))
+        cmake_build('curl', false, merge_list(curl_common_defs, merge_list(wild_zlib_64, {
+            'MBEDTLS_INCLUDE_DIRS="' .. path.join(deps_dir, 'mbedtls', 'install64', 'include') .. '"',
+            'MBEDTLS_LIBRARY="' .. path.join(deps_dir, 'mbedtls', 'install64', 'lib', mbedtls_name) .. '"',
+            'MBEDCRYPTO_LIBRARY="' .. path.join(deps_dir, 'mbedtls', 'install64', 'lib', mbedcrypto_name) .. '"',
+            'MBEDX509_LIBRARY="' .. path.join(deps_dir, 'mbedtls', 'install64', 'lib', mbedx509_name) .. '"',
+        })))
     end
 end
 
@@ -588,32 +653,6 @@ if _OPTIONS["build-protobuf"] or _OPTIONS["all-build"] then
     end
 end
 
-if _OPTIONS["build-mbedtls"] or _OPTIONS["all-build"] then
-    local mbedtls_common_defs = {
-        "USE_STATIC_MBEDTLS_LIBRARY=ON",
-        "USE_SHARED_MBEDTLS_LIBRARY=OFF",
-        "ENABLE_TESTING=OFF",
-        "ENABLE_PROGRAMS=OFF",
-        "MBEDTLS_FATAL_WARNINGS=OFF",
-    }
-    if os.target() == 'windows' and string.match(_ACTION, 'vs.+') then
-        table.insert(mbedtls_common_defs, "MSVC_STATIC_RUNTIME=ON")
-    else -- linux or macos or MinGW on Windows
-        table.insert(mbedtls_common_defs, "LINK_WITH_PTHREAD=ON")
-    end
-
-    if _OPTIONS["32-build"] then
-        cmake_build('mbedtls', true, mbedtls_common_defs, {
-            '-mpclmul',
-            '-msse2',
-            '-maes',
-        })
-    end
-    if _OPTIONS["64-build"] then
-        cmake_build('mbedtls', false, mbedtls_common_defs)
-    end
-end
-
 if _OPTIONS["build-ingame_overlay"] or _OPTIONS["all-build"] then
     -- fixes 32-bit compilation of DX12
     local overaly_imgui_cfg_file = path.join(deps_dir, 'ingame_overlay', 'imconfig.imcfg')

premake5.lua

@@ -273,6 +273,8 @@ local deps_link = {
     zlib_archive_name    .. static_postfix,
     lib_prefix .. "curl" .. static_postfix,
     "mbedcrypto"         .. static_postfix,
+    "mbedtls"            .. static_postfix,
+    "mbedx509"           .. static_postfix,
 }
 -- add protobuf libs
 table_append(deps_link, {