* more accurately handle and download steamhttp requests in multi-threaded manner

* allow forcing the API `Steam_HTTP::SendHTTPRequest()` to succeed via a config file

* change http_online.txt to download_steamhttp_requests.txt

CHANGELOG.md

@@ -1,3 +1,15 @@
+* more accurately handle and download steamhttp requests in multi-threaded manner:
+  - hanlde `GET`, `HEAD`, `POST`
+  - properly set `POST` data (raw and parameterized), and `GET` parameters
+  - properly set request headers
+
+* new config file `force_steamhttp_success.txt` in `steam_settings` folder, which forces the API `Steam_HTTP::SendHTTPRequest()` to always succeed
+
+---
+* **[breaking]** deprecated the config file `http_online.txt` in favor of the new one `download_steamhttp_requests.txt`
+
+---
+
 # 2424/2/20
 
 * generate_emu_config: allow setting the steam id of apps/games owners from an external file `top_owners_ids.txt` beside the script, suggested by **[M4RCK5]**

dll/dll/common_includes.h

@@ -192,6 +192,7 @@ constexpr const char * const whitespaces = " \t\r\n";
             auto __prnt_dbg_micro = std::chrono::duration_cast<std::chrono::duration<unsigned long long, std::micro>>(__prnt_dbg_duration);                  \
             auto __prnt_dbg_ms = std::chrono::duration_cast<std::chrono::duration<unsigned long long, std::milli>>(__prnt_dbg_duration);                     \
             auto __prnt_dbg_f = fopen(dbg_log_file.c_str(), "a");                                                                                            \
+            if (!__prnt_dbg_f) break;                                                                                                                        \
             fprintf(__prnt_dbg_f, "[%llu ms, %llu us] [tid %lu] " a, __prnt_dbg_ms.count(), __prnt_dbg_micro.count(), GetCurrentThreadId(), __VA_ARGS__);    \
             fclose(__prnt_dbg_f);                                                                                                                            \
             WSASetLastError(0);                                                                                                                              \
@@ -204,6 +205,7 @@ constexpr const char * const whitespaces = " \t\r\n";
             auto __prnt_dbg_micro = std::chrono::duration_cast<std::chrono::duration<unsigned long long, std::micro>>(__prnt_dbg_duration);                   \
             auto __prnt_dbg_ms = std::chrono::duration_cast<std::chrono::duration<unsigned long long, std::milli>>(__prnt_dbg_duration);                      \
             auto __prnt_dbg_f = fopen(dbg_log_file.c_str(), "a");                                                                                             \
+            if (!__prnt_dbg_f) break;                                                                                                                         \
             fprintf(__prnt_dbg_f, "[%llu ms, %llu us] [tid %ld] " a, __prnt_dbg_ms.count(), __prnt_dbg_micro.count(), syscall(SYS_gettid), ##__VA_ARGS__);    \
             fclose(__prnt_dbg_f);                                                                                                                             \
         } while (0)

dll/dll/settings.h

@@ -300,7 +300,8 @@ public:
     std::string local_save;
 
     //steamhttp external download support
-    bool http_online = false;
+    bool download_steamhttp_requests = false;
+    bool force_steamhttp_success = false;
 
     //steam deck flag
     bool steam_deck = false;

dll/dll/steam_http.h

@@ -22,9 +22,30 @@
 
 struct Steam_Http_Request {
 	HTTPRequestHandle handle;
+	EHTTPMethod request_method;
+	std::string url{};
+	uint64 timeout_sec = 60;
+	bool requires_valid_ssl = false;
+
+	constexpr const static char STEAM_DEFAULT_USER_AGENT[] = "Valve/Steam HTTP Client 1.0";
+	// GET or POST parameter value on the request
+	std::map<std::string, std::string> headers{
+		{"User-Agent", STEAM_DEFAULT_USER_AGENT},
+	};
+
+	// GET or POST parameter value on the request
+	std::map<std::string, std::string> get_or_post_params{};
+	std::string post_raw{};
+
 	uint64 context_value;
 
-	std::string response;
+	// target local filepath to save
+	std::string target_filepath{};
+
+	// TODO
+	HTTPCookieContainerHandle cookie_container_handle = INVALID_HTTPCOOKIE_HANDLE;
+
+	std::string response{};
 };
 
 class Steam_HTTP :
@@ -40,6 +61,8 @@ public ISteamHTTP
 	std::vector<Steam_Http_Request> requests;
 
 	Steam_Http_Request *get_request(HTTPRequestHandle hRequest);
+	void online_http_request(Steam_Http_Request *request, SteamAPICall_t *pCallHandle);
+
 public:
 	Steam_HTTP(class Settings *settings, class Networking *network, class SteamCallResults *callback_results, class SteamCallBacks *callbacks);
 

dll/settings_parser.cpp

@@ -1194,7 +1194,8 @@ uint32 create_localstorage_settings(Settings **settings_client_out, Settings **s
 
     bool steam_offline_mode = false;
     bool steam_deck_mode = false;
-    bool steamhttp_online_mode = false;
+    bool download_steamhttp_requests = false;
+    bool force_steamhttp_success = false;
     bool disable_networking = false;
     bool disable_overlay = false;
     bool disable_overlay_achievement_notification = false;
@@ -1225,8 +1226,10 @@ uint32 create_localstorage_settings(Settings **settings_client_out, Settings **s
                 steam_offline_mode = true;
             } else if (p == "steam_deck.txt") {
                 steam_deck_mode = true;
-            } else if (p == "http_online.txt") {
+            } else if (p == "download_steamhttp_requests.txt") {
-                steamhttp_online_mode = true;
+                download_steamhttp_requests = true;
+            } else if (p == "force_steamhttp_success.txt") {
+                force_steamhttp_success = true;
             } else if (p == "disable_networking.txt") {
                 disable_networking = true;
             } else if (p == "disable_overlay.txt") {
@@ -1315,8 +1318,10 @@ uint32 create_localstorage_settings(Settings **settings_client_out, Settings **s
     settings_server->supported_languages = supported_languages;
     settings_client->steam_deck = steam_deck_mode;
     settings_server->steam_deck = steam_deck_mode;
-    settings_client->http_online = steamhttp_online_mode;
+    settings_client->download_steamhttp_requests = download_steamhttp_requests;
-    settings_server->http_online = steamhttp_online_mode;
+    settings_server->download_steamhttp_requests = download_steamhttp_requests;
+    settings_client->force_steamhttp_success = force_steamhttp_success;
+    settings_server->force_steamhttp_success = force_steamhttp_success;
     settings_client->achievement_bypass = achievement_bypass;
     settings_server->achievement_bypass = achievement_bypass;
     settings_client->is_beta_branch = is_beta_branch;

dll/steam_http.cpp

@@ -39,7 +39,10 @@ Steam_Http_Request *Steam_HTTP::get_request(HTTPRequestHandle hRequest)
 HTTPRequestHandle Steam_HTTP::CreateHTTPRequest( EHTTPMethod eHTTPRequestMethod, const char *pchAbsoluteURL )
 {
     PRINT_DEBUG("Steam_HTTP::CreateHTTPRequest %i %s\n", eHTTPRequestMethod, pchAbsoluteURL);
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+
     if (!pchAbsoluteURL) return INVALID_HTTPREQUEST_HANDLE;
+
     std::string url = pchAbsoluteURL;
     unsigned url_index = 0;
     if (url.rfind("https://", 0) == 0) {
@@ -48,56 +51,25 @@ HTTPRequestHandle Steam_HTTP::CreateHTTPRequest( EHTTPMethod eHTTPRequestMethod,
         url_index = sizeof("http://") - 1;
     }
 
-    struct Steam_Http_Request request;
+    struct Steam_Http_Request request{};
+    request.request_method = eHTTPRequestMethod;
+    request.url = url;
     if (url_index) {
-        if (url[url.size() - 1] == '/') url += "index.html";
+        if (url.back() == '/') url += "index.html";
         std::string file_path = Local_Storage::get_game_settings_path() + "http/" + Local_Storage::sanitize_string(url.substr(url_index));
+        request.target_filepath = file_path;
         unsigned long long file_size = file_size_(file_path);
         if (file_size) {
             request.response.resize(file_size);
             long long read = Local_Storage::get_file_data(file_path, (char *)request.response.data(), file_size, 0);
             if (read < 0) read = 0;
             if (read != file_size) request.response.resize(read);
-        } else if (!settings->disable_networking && settings->http_online) {
-            std::lock_guard<std::recursive_mutex> lock(global_mutex);
-
-            std::size_t url_directory = file_path.rfind("/");
-            std::string directory_path;
-            std::string file_name;
-            if (url_directory != std::string::npos) {
-                directory_path = file_path.substr(0, url_directory);
-                file_name = file_path.substr(url_directory);
-            }
-            Local_Storage::store_file_data(directory_path, file_name, (char *)"", sizeof(""));
-
-#if defined(STEAM_WIN32)
-            FILE *hfile = fopen(file_path.c_str(), "wb");
-#else
-            FILE *hfile = fopen(file_path.c_str(), "w");
-#endif
-            CURL *chttp = curl_easy_init();
-            curl_easy_setopt(chttp, CURLOPT_URL, url.c_str());
-            curl_easy_setopt(chttp, CURLOPT_WRITEDATA, (void *)hfile);
-            curl_easy_setopt(chttp, CURLOPT_TIMEOUT, 60L);
-            curl_easy_setopt(chttp, CURLOPT_NOSIGNAL, 1L);
-            curl_easy_setopt(chttp, CURLOPT_USE_SSL, CURLUSESSL_TRY);
-            curl_easy_perform(chttp);
-            curl_easy_cleanup(chttp);
-            fclose(hfile);
-
-            file_size = file_size_(file_path);
-            if (file_size) {
-                request.response.resize(file_size);
-                long long read = Local_Storage::get_file_data(file_path, (char *)request.response.data(), file_size, 0);
-                if (read < 0) read = 0;
-                if (read != file_size) request.response.resize(read);
-            }
         }
     }
 
-    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+    static HTTPRequestHandle h = 0;
-    static HTTPRequestHandle h;
     ++h;
+    if (!h) ++h;
 
     request.handle = h;
     request.context_value = 0;
@@ -112,6 +84,8 @@ HTTPRequestHandle Steam_HTTP::CreateHTTPRequest( EHTTPMethod eHTTPRequestMethod,
 bool Steam_HTTP::SetHTTPRequestContextValue( HTTPRequestHandle hRequest, uint64 ulContextValue )
 {
     PRINT_DEBUG("Steam_HTTP::SetHTTPRequestContextValue\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
@@ -128,11 +102,14 @@ bool Steam_HTTP::SetHTTPRequestContextValue( HTTPRequestHandle hRequest, uint64
 bool Steam_HTTP::SetHTTPRequestNetworkActivityTimeout( HTTPRequestHandle hRequest, uint32 unTimeoutSeconds )
 {
     PRINT_DEBUG("Steam_HTTP::SetHTTPRequestNetworkActivityTimeout\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
     }
 
+    request->timeout_sec = unTimeoutSeconds;
     return true;
 }
 
@@ -141,12 +118,20 @@ bool Steam_HTTP::SetHTTPRequestNetworkActivityTimeout( HTTPRequestHandle hReques
 // return false if the handle is invalid or the request is already sent.
 bool Steam_HTTP::SetHTTPRequestHeaderValue( HTTPRequestHandle hRequest, const char *pchHeaderName, const char *pchHeaderValue )
 {
-    PRINT_DEBUG("Steam_HTTP::SetHTTPRequestHeaderValue %s %s\n", pchHeaderName, pchHeaderValue);
+    PRINT_DEBUG("Steam_HTTP::SetHTTPRequestHeaderValue '%s' = '%s'\n", pchHeaderName, pchHeaderValue);
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+
+    if (!pchHeaderName || !pchHeaderValue) return false;
+    std::string headerName(pchHeaderName);
+    std::transform(headerName.begin(), headerName.end(), headerName.begin(), [](char c){ return (char)std::toupper(c); });
+    if (headerName == "USER-AGENT") return false;
+
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
     }
 
+    request->headers[pchHeaderName] = pchHeaderValue;
     return true;
 }
 
@@ -156,16 +141,188 @@ bool Steam_HTTP::SetHTTPRequestHeaderValue( HTTPRequestHandle hRequest, const ch
 // handle is invalid or the request is already sent.
 bool Steam_HTTP::SetHTTPRequestGetOrPostParameter( HTTPRequestHandle hRequest, const char *pchParamName, const char *pchParamValue )
 {
-    PRINT_DEBUG("Steam_HTTP::SetHTTPRequestGetOrPostParameter\n");
+    PRINT_DEBUG("Steam_HTTP::SetHTTPRequestGetOrPostParameter '%s' = '%s'\n", pchParamName, pchParamValue);
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+
+    if (!pchParamName || !pchParamValue) return false;
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
     }
+    if (request->request_method != EHTTPMethod::k_EHTTPMethodGET &&
+        request->request_method != EHTTPMethod::k_EHTTPMethodHEAD &&
+        request->request_method != EHTTPMethod::k_EHTTPMethodPOST) {
+        return false;
+    }
+    if (request->post_raw.size()) return false;
 
+    request->get_or_post_params[pchParamName] = pchParamValue;
     return true;
 }
 
 
+void Steam_HTTP::online_http_request(Steam_Http_Request *request, SteamAPICall_t *pCallHandle)
+{
+    PRINT_DEBUG("Steam_HTTP::online_http_request attempting to download from url: '%s', target filepath: '%s'\n",
+        request->url.c_str(), request->target_filepath.c_str());
+
+    const auto send_callresult = [&]() -> void {
+        struct HTTPRequestCompleted_t data{};
+        data.m_hRequest = request->handle;
+        data.m_ulContextValue = request->context_value;
+        data.m_unBodySize = request->response.size();
+        if (request->response.empty() && !settings->force_steamhttp_success) {
+            data.m_bRequestSuccessful = false;
+            data.m_eStatusCode = k_EHTTPStatusCode404NotFound;
+            
+        } else {
+            data.m_bRequestSuccessful = true;
+            data.m_eStatusCode = k_EHTTPStatusCode200OK;
+        }
+
+        if (pCallHandle) *pCallHandle = callback_results->addCallResult(data.k_iCallback, &data, sizeof(data), 0.1);
+    };
+
+    std::size_t filename_part = request->target_filepath.find_last_of("\\/");
+    std::string directory_path{};
+    std::string file_name{};
+    if (filename_part != std::string::npos) {
+        filename_part += 1; // point at filename, not the '/' or '\'
+        directory_path = request->target_filepath.substr(0, filename_part);
+        file_name = request->target_filepath.substr(filename_part);
+    } else {
+        directory_path = ".";
+        file_name = request->target_filepath;
+    }
+    PRINT_DEBUG("Steam_HTTP::online_http_request directory: '%s', filename '%s'\n", directory_path.c_str(), file_name.c_str());
+    Local_Storage::store_file_data(directory_path, file_name, (char *)"", sizeof(""));
+
+    FILE *hfile = std::fopen(request->target_filepath.c_str(), "wb");
+    if (!hfile) {
+        PRINT_DEBUG("Steam_HTTP::online_http_request failed to open file for writing\n");
+        send_callresult();
+        return;
+    }
+    CURL *chttp = curl_easy_init();
+    if (!chttp) {
+        fclose(hfile);
+        PRINT_DEBUG("Steam_HTTP::online_http_request curl_easy_init() failed\n");
+        send_callresult();
+        return;
+    }
+    
+    // headers
+    std::vector<std::string> headers{};
+    for (const auto &hdr : request->headers) {
+        std::string new_header = hdr.first + ": " + hdr.second;
+        PRINT_DEBUG("Steam_HTTP::online_http_request CURL header: '%s'\n", new_header.c_str());
+        headers.push_back(new_header);
+    }
+
+    struct curl_slist *headers_list = nullptr;
+    for (const auto &hrd : headers) {
+        headers_list = curl_slist_append(headers_list, hrd.c_str());
+    }
+    curl_easy_setopt(chttp, CURLOPT_HTTPHEADER, headers_list);
+    
+    // request method
+    switch (request->request_method)
+    {
+    case EHTTPMethod::k_EHTTPMethodGET:
+        PRINT_DEBUG("Steam_HTTP::online_http_request CURL method type: GET\n");
+        curl_easy_setopt(chttp, CURLOPT_HTTPGET, 1L);
+    break;
+    
+    case EHTTPMethod::k_EHTTPMethodHEAD:
+        PRINT_DEBUG("Steam_HTTP::online_http_request CURL method type: HEAD\n");
+        curl_easy_setopt(chttp, CURLOPT_NOBODY, 1L);
+    break;
+    
+    case EHTTPMethod::k_EHTTPMethodPOST:
+        PRINT_DEBUG("Steam_HTTP::online_http_request CURL method type: POST\n");
+        curl_easy_setopt(chttp, CURLOPT_POST, 1L);
+    break;
+    
+    case EHTTPMethod::k_EHTTPMethodPUT:
+        PRINT_DEBUG("TODO Steam_HTTP::online_http_request CURL method type: PUT\n");
+        curl_easy_setopt(chttp, CURLOPT_UPLOAD, 1L); // CURLOPT_PUT "This option is deprecated since version 7.12.1. Use CURLOPT_UPLOAD."
+    break;
+    
+    case EHTTPMethod::k_EHTTPMethodDELETE:
+        PRINT_DEBUG("TODO Steam_HTTP::online_http_request CURL method type: DELETE\n");
+        headers_list = curl_slist_append(headers_list, "Content-Type: application/x-www-form-urlencoded");
+        headers_list = curl_slist_append(headers_list, "Accept: application/json,application/x-www-form-urlencoded,text/html,application/xhtml+xml,application/xml");
+        curl_easy_setopt(chttp, CURLOPT_CUSTOMREQUEST, "DELETE"); // https://stackoverflow.com/a/34751940
+    break;
+    
+    case EHTTPMethod::k_EHTTPMethodOPTIONS:
+        PRINT_DEBUG("TODO Steam_HTTP::online_http_request CURL method type: OPTIONS\n");
+        curl_easy_setopt(chttp, CURLOPT_CUSTOMREQUEST, "OPTIONS");
+    break;
+    
+    case EHTTPMethod::k_EHTTPMethodPATCH:
+        PRINT_DEBUG("TODO Steam_HTTP::online_http_request CURL method type: PATCH\n");
+        headers_list = curl_slist_append(headers_list, "Content-Type: application/x-www-form-urlencoded");
+        headers_list = curl_slist_append(headers_list, "Accept: application/json,application/x-www-form-urlencoded,text/html,application/xhtml+xml,application/xml");
+        curl_easy_setopt(chttp, CURLOPT_CUSTOMREQUEST, "PATCH");
+    break;
+    
+    default:
+        break;
+    }
+    
+    curl_easy_setopt(chttp, CURLOPT_FOLLOWLOCATION, 1L);
+    curl_easy_setopt(chttp, CURLOPT_WRITEDATA, (void *)hfile);
+    curl_easy_setopt(chttp, CURLOPT_TIMEOUT, request->timeout_sec);
+    curl_easy_setopt(chttp, CURLOPT_NOSIGNAL, 1L);
+    curl_easy_setopt(chttp, CURLOPT_USE_SSL, request->requires_valid_ssl ? CURLUSESSL_TRY : CURLUSESSL_NONE);
+
+    // post data, or get params
+    std::string post_data{};
+    if (request->get_or_post_params.size()) {
+        for (const auto &pdata : request->get_or_post_params) {
+            char *form_encoded_key = curl_easy_escape(chttp, pdata.first.c_str(), (int)pdata.first.size());
+            char *form_encoded_val = curl_easy_escape(chttp, pdata.second.c_str(), (int)pdata.second.size());
+            if (form_encoded_key && form_encoded_val) {
+                post_data += form_encoded_key + std::string("=") + form_encoded_val + "&";
+            }
+            if (form_encoded_key) curl_free(form_encoded_key);
+            if (form_encoded_val) curl_free(form_encoded_val);
+        }
+        if (post_data.size()) post_data = post_data.substr(0, post_data.size() - 1); // remove the last "&"
+        if (request->request_method == EHTTPMethod::k_EHTTPMethodGET) {
+            request->url += "?" + post_data;
+            PRINT_DEBUG("Steam_HTTP::online_http_request GET URL with params (url-encoded): '%s'\n", request->url.c_str());
+        } else {
+            PRINT_DEBUG("Steam_HTTP::online_http_request POST form data (url-encoded): '%s'\n", post_data.c_str());
+            curl_easy_setopt(chttp, CURLOPT_POSTFIELDS, post_data.c_str());
+        }
+    } else if (request->post_raw.size()) {
+        PRINT_DEBUG("Steam_HTTP::online_http_request POST form data (raw): '%s'\n", request->post_raw.c_str());
+        curl_easy_setopt(chttp, CURLOPT_POSTFIELDS, request->post_raw.c_str());
+    }
+
+    curl_easy_setopt(chttp, CURLOPT_URL, request->url.c_str());
+    
+    CURLcode res_curl = curl_easy_perform(chttp);
+    curl_slist_free_all(headers_list);
+    curl_easy_cleanup(chttp);
+
+    fclose(hfile);
+    headers.clear();
+
+    PRINT_DEBUG("Steam_HTTP::online_http_request CURL for '%s' error code (0 == OK 0): [%i]\n", request->url.c_str(), (int)res_curl);
+    
+    unsigned int file_size = file_size_(request->target_filepath);
+    if (file_size) {
+        long long read = Local_Storage::get_file_data(request->target_filepath, (char *)request->response.data(), file_size, 0);
+        if (read < 0) read = 0;
+        request->response.resize(read);
+    }
+    
+    send_callresult();
+}
+
 // Sends the HTTP request, will return false on a bad handle, otherwise use SteamCallHandle to wait on
 // asynchronous response via callback.
 //
@@ -174,26 +331,31 @@ bool Steam_HTTP::SetHTTPRequestGetOrPostParameter( HTTPRequestHandle hRequest, c
 bool Steam_HTTP::SendHTTPRequest( HTTPRequestHandle hRequest, SteamAPICall_t *pCallHandle )
 {
     PRINT_DEBUG("Steam_HTTP::SendHTTPRequest %u %p\n", hRequest, pCallHandle);
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
     }
 
-    struct HTTPRequestCompleted_t data = {};
+    if (request->response.empty() && request->target_filepath.size() &&
+        !settings->disable_networking && settings->download_steamhttp_requests) {
+        std::thread(&Steam_HTTP::online_http_request, this, request, pCallHandle).detach();
+    } else {
+        struct HTTPRequestCompleted_t data{};
         data.m_hRequest = request->handle;
         data.m_ulContextValue = request->context_value;
-    if (request->response.size() == 0) {
+        data.m_unBodySize = request->response.size();
+        if (request->response.empty() && !settings->force_steamhttp_success) {
             data.m_bRequestSuccessful = false;
             data.m_eStatusCode = k_EHTTPStatusCode404NotFound;
-        data.m_unBodySize = request->response.size();
+            
         } else {
             data.m_bRequestSuccessful = true;
             data.m_eStatusCode = k_EHTTPStatusCode200OK;
-        data.m_unBodySize = request->response.size();
         }
 
-    if (pCallHandle) {
+        if (pCallHandle) *pCallHandle = callback_results->addCallResult(data.k_iCallback, &data, sizeof(data), 0.1);
-        *pCallHandle = callback_results->addCallResult(data.k_iCallback, &data, sizeof(data), 0.1);
     }
 
     return true;
@@ -206,7 +368,9 @@ bool Steam_HTTP::SendHTTPRequest( HTTPRequestHandle hRequest, SteamAPICall_t *pC
 bool Steam_HTTP::SendHTTPRequestAndStreamResponse( HTTPRequestHandle hRequest, SteamAPICall_t *pCallHandle )
 {
     PRINT_DEBUG("Steam_HTTP::SendHTTPRequestAndStreamResponse\n");
-    return false;
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+
+    return SendHTTPRequest(hRequest, pCallHandle);
 }
 
 
@@ -215,6 +379,8 @@ bool Steam_HTTP::SendHTTPRequestAndStreamResponse( HTTPRequestHandle hRequest, S
 bool Steam_HTTP::DeferHTTPRequest( HTTPRequestHandle hRequest )
 {
     PRINT_DEBUG("Steam_HTTP::DeferHTTPRequest\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
@@ -229,6 +395,8 @@ bool Steam_HTTP::DeferHTTPRequest( HTTPRequestHandle hRequest )
 bool Steam_HTTP::PrioritizeHTTPRequest( HTTPRequestHandle hRequest )
 {
     PRINT_DEBUG("Steam_HTTP::PrioritizeHTTPRequest\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
@@ -243,9 +411,21 @@ bool Steam_HTTP::PrioritizeHTTPRequest( HTTPRequestHandle hRequest )
 // GetHTTPResponseHeaderValue.
 bool Steam_HTTP::GetHTTPResponseHeaderSize( HTTPRequestHandle hRequest, const char *pchHeaderName, uint32 *unResponseHeaderSize )
 {
-    PRINT_DEBUG("Steam_HTTP::GetHTTPResponseHeaderSize\n");
+    PRINT_DEBUG("Steam_HTTP::GetHTTPResponseHeaderSize '%s'\n", pchHeaderName);
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+
+    if (!pchHeaderName) return false;
+
+    Steam_Http_Request *request = get_request(hRequest);
+    if (!request) {
         return false;
     }
+    const auto hdr = request->headers.find(pchHeaderName);
+    if (request->headers.end() == hdr) return false;
+
+    if (unResponseHeaderSize) *unResponseHeaderSize = (uint32)hdr->second.size();
+    return true;
+}
 
 
 // Gets header values from a HTTP response given a handle from HTTPRequestCompleted_t, will return false if the
@@ -254,8 +434,23 @@ bool Steam_HTTP::GetHTTPResponseHeaderSize( HTTPRequestHandle hRequest, const ch
 bool Steam_HTTP::GetHTTPResponseHeaderValue( HTTPRequestHandle hRequest, const char *pchHeaderName, uint8 *pHeaderValueBuffer, uint32 unBufferSize )
 {
     PRINT_DEBUG("Steam_HTTP::GetHTTPResponseHeaderValue\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+
+    if (!pchHeaderName) return false;
+
+    Steam_Http_Request *request = get_request(hRequest);
+    if (!request) {
         return false;
     }
+    const auto hdr = request->headers.find(pchHeaderName);
+    if (request->headers.end() == hdr) return false;
+    if (unBufferSize < hdr->second.size()) return false;
+    if (pHeaderValueBuffer) {
+        memset(pHeaderValueBuffer, 0, unBufferSize);
+        hdr->second.copy((char *)pHeaderValueBuffer, unBufferSize);
+    }
+    return true;
+}
 
 
 // Gets the size of the body data from a HTTP response given a handle from HTTPRequestCompleted_t, will return false if the 
@@ -263,12 +458,14 @@ bool Steam_HTTP::GetHTTPResponseHeaderValue( HTTPRequestHandle hRequest, const c
 bool Steam_HTTP::GetHTTPResponseBodySize( HTTPRequestHandle hRequest, uint32 *unBodySize )
 {
     PRINT_DEBUG("Steam_HTTP::GetHTTPResponseBodySize\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
     }
 
-    if (unBodySize) *unBodySize = request->response.size();
+    if (unBodySize) *unBodySize = (uint32)request->response.size();
     return true;
 }
 
@@ -278,17 +475,19 @@ bool Steam_HTTP::GetHTTPResponseBodySize( HTTPRequestHandle hRequest, uint32 *un
 // the correct buffer size to use.
 bool Steam_HTTP::GetHTTPResponseBodyData( HTTPRequestHandle hRequest, uint8 *pBodyDataBuffer, uint32 unBufferSize )
 {
-    PRINT_DEBUG("Steam_HTTP::GetHTTPResponseBodyData\n");
+    PRINT_DEBUG("Steam_HTTP::GetHTTPResponseBodyData %p %u\n", pBodyDataBuffer, unBufferSize);
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+    
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
     }
-
+    PRINT_DEBUG("  Steam_HTTP::GetHTTPResponseBodyData required buffer size = %zu\n", request->response.size());
-    if (unBufferSize < request->response.size()) {
+    if (unBufferSize < request->response.size()) return false;
-        return false;
+    if (pBodyDataBuffer) {
+        memset(pBodyDataBuffer, 0, unBufferSize);
+        request->response.copy((char *)pBodyDataBuffer, unBufferSize);
     }
-
-    if (pBodyDataBuffer) memcpy(pBodyDataBuffer, request->response.data(), request->response.size());
     return true;
 }
 
@@ -299,6 +498,18 @@ bool Steam_HTTP::GetHTTPResponseBodyData( HTTPRequestHandle hRequest, uint8 *pBo
 bool Steam_HTTP::GetHTTPStreamingResponseBodyData( HTTPRequestHandle hRequest, uint32 cOffset, uint8 *pBodyDataBuffer, uint32 unBufferSize )
 {
     PRINT_DEBUG("Steam_HTTP::GetHTTPStreamingResponseBodyData\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+    
+    Steam_Http_Request *request = get_request(hRequest);
+    if (!request) {
+        return false;
+    }
+
+    if (pBodyDataBuffer && cOffset <= request->response.size()) {
+        memset(pBodyDataBuffer, 0, unBufferSize);
+        request->response.copy((char *)pBodyDataBuffer, unBufferSize, cOffset);
+        return true;
+    }
     return false;
 }
 
@@ -330,8 +541,15 @@ bool Steam_HTTP::ReleaseHTTPRequest( HTTPRequestHandle hRequest )
 bool Steam_HTTP::GetHTTPDownloadProgressPct( HTTPRequestHandle hRequest, float *pflPercentOut )
 {
     PRINT_DEBUG("Steam_HTTP::GetHTTPDownloadProgressPct\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+    
+    Steam_Http_Request *request = get_request(hRequest);
+    if (!request) {
         return false;
     }
+    if (pflPercentOut) *pflPercentOut = 100.0f;
+    return true;
+}
 
 
 // Sets the body for an HTTP Post request.  Will fail and return false on a GET request, and will fail if POST params
@@ -340,11 +558,21 @@ bool Steam_HTTP::GetHTTPDownloadProgressPct( HTTPRequestHandle hRequest, float *
 bool Steam_HTTP::SetHTTPRequestRawPostBody( HTTPRequestHandle hRequest, const char *pchContentType, uint8 *pubBody, uint32 unBodyLen )
 {
     PRINT_DEBUG("Steam_HTTP::SetHTTPRequestRawPostBody %s\n", pchContentType);
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+    
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
     }
 
+    if (request->request_method != EHTTPMethod::k_EHTTPMethodPOST &&
+        request->request_method != EHTTPMethod::k_EHTTPMethodPUT &&
+        request->request_method != EHTTPMethod::k_EHTTPMethodPATCH) {
+        return false;
+    }
+    if (request->get_or_post_params.size()) return false;
+
+    request->post_raw = std::string((char *)pubBody, unBodyLen);
     return true;
 }
 
@@ -356,15 +584,23 @@ bool Steam_HTTP::SetHTTPRequestRawPostBody( HTTPRequestHandle hRequest, const ch
 // repeat executions of your process.
 HTTPCookieContainerHandle Steam_HTTP::CreateCookieContainer( bool bAllowResponsesToModify )
 {
-    PRINT_DEBUG("Steam_HTTP::CreateCookieContainer\n");
+    PRINT_DEBUG("TODO Steam_HTTP::CreateCookieContainer\n");
-    return false;
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+    
+    static HTTPCookieContainerHandle handle = 0;
+    ++handle;
+    if (!handle) ++handle;
+    
+    return INVALID_HTTPCOOKIE_HANDLE;
 }
 
 
 // Release a cookie container you are finished using, freeing it's memory
 bool Steam_HTTP::ReleaseCookieContainer( HTTPCookieContainerHandle hCookieContainer )
 {
-    PRINT_DEBUG("Steam_HTTP::ReleaseCookieContainer\n");
+    PRINT_DEBUG("TODO Steam_HTTP::ReleaseCookieContainer\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+
     return false;
 }
 
@@ -372,7 +608,9 @@ bool Steam_HTTP::ReleaseCookieContainer( HTTPCookieContainerHandle hCookieContai
 // Adds a cookie to the specified cookie container that will be used with future requests.
 bool Steam_HTTP::SetCookie( HTTPCookieContainerHandle hCookieContainer, const char *pchHost, const char *pchUrl, const char *pchCookie )
 {
-    PRINT_DEBUG("Steam_HTTP::SetCookie\n");
+    PRINT_DEBUG("TODO Steam_HTTP::SetCookie\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+    
     return false;
 }
 
@@ -380,7 +618,9 @@ bool Steam_HTTP::SetCookie( HTTPCookieContainerHandle hCookieContainer, const ch
 // Set the cookie container to use for a HTTP request
 bool Steam_HTTP::SetHTTPRequestCookieContainer( HTTPRequestHandle hRequest, HTTPCookieContainerHandle hCookieContainer )
 {
-    PRINT_DEBUG("Steam_HTTP::SetHTTPRequestCookieContainer\n");
+    PRINT_DEBUG("TODO Steam_HTTP::SetHTTPRequestCookieContainer\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+    
     return false;
 }
 
@@ -389,11 +629,18 @@ bool Steam_HTTP::SetHTTPRequestCookieContainer( HTTPRequestHandle hRequest, HTTP
 bool Steam_HTTP::SetHTTPRequestUserAgentInfo( HTTPRequestHandle hRequest, const char *pchUserAgentInfo )
 {
     PRINT_DEBUG("Steam_HTTP::SetHTTPRequestUserAgentInfo\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+    
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
     }
 
+    if (!pchUserAgentInfo || !pchUserAgentInfo[0]) {
+        request->headers["User-Agent"] = Steam_Http_Request::STEAM_DEFAULT_USER_AGENT;
+    } else {
+        request->headers["User-Agent"] += std::string(" ") + pchUserAgentInfo;
+    }
     return true;
 }
 
@@ -402,11 +649,14 @@ bool Steam_HTTP::SetHTTPRequestUserAgentInfo( HTTPRequestHandle hRequest, const
 bool Steam_HTTP::SetHTTPRequestRequiresVerifiedCertificate( HTTPRequestHandle hRequest, bool bRequireVerifiedCertificate )
 {
     PRINT_DEBUG("Steam_HTTP::SetHTTPRequestRequiresVerifiedCertificate\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+    
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
     }
 
+    request->requires_valid_ssl = bRequireVerifiedCertificate;
     return true;
 }
 
@@ -416,11 +666,14 @@ bool Steam_HTTP::SetHTTPRequestRequiresVerifiedCertificate( HTTPRequestHandle hR
 bool Steam_HTTP::SetHTTPRequestAbsoluteTimeoutMS( HTTPRequestHandle hRequest, uint32 unMilliseconds )
 {
     PRINT_DEBUG("Steam_HTTP::SetHTTPRequestAbsoluteTimeoutMS\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+    
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;
     }
 
+    request->timeout_sec = (uint64)(unMilliseconds / 1000.0);
     return true;
 }
 
@@ -429,6 +682,8 @@ bool Steam_HTTP::SetHTTPRequestAbsoluteTimeoutMS( HTTPRequestHandle hRequest, ui
 bool Steam_HTTP::GetHTTPRequestWasTimedOut( HTTPRequestHandle hRequest, bool *pbWasTimedOut )
 {
     PRINT_DEBUG("Steam_HTTP::GetHTTPRequestWasTimedOut\n");
+    std::lock_guard<std::recursive_mutex> lock(global_mutex);
+    
     Steam_Http_Request *request = get_request(hRequest);
     if (!request) {
         return false;

dll/steam_matchmaking_servers.cpp

@@ -702,7 +702,7 @@ void Steam_Matchmaking_Servers::RunCallbacks()
         }
     }
 
-    PRINT_DEBUG("Steam_Matchmaking_Servers::REQUESTS %zu gs: %zu\n", requests.size(), gameservers.size());
+    PRINT_DEBUG("Steam_Matchmaking_Servers::RunCallbacks requests count = %zu, servers count = %zu\n", requests.size(), gameservers.size());
 
     for (auto &r : requests) {
         if (r.cancelled || r.completed) continue;

post_build/README.release.md

@@ -474,6 +474,32 @@ Check the exaxmple file in the `steam_settings` folder
 
 ---
 
+## Donwload Steam HTTP(S) requests:
+
+You can make the emu attempt to download external http(s) requests madia via `Steam_HTTP::SendHTTPRequest()`, by creating a file called `download_steamhttp_requests.txt` inside the `steam_settings` folder.  
+All the responses will be downloaded and saved locally inside: `steam_settings\http\`.  
+
+Make sure to:  
+* Add the file `disable_lan_only.txt`  
+* Remove the file `disable_networking.txt` if it's present  
+
+Note that this will **not** work if the app is using native/OS web APIs, also support for this feature is very basic and will fail in many cases.  
+
+You can use this feature, for eaxmple, to know which requests are made by the app.  
+It's up to you afterwards to specify the correct responses for these requests by changing the content of the files inside `steam_settings\http\`.  
+
+Check the exaxmple file in the `steam_settings` folder
+
+---
+
+## Force the API `Steam_HTTP::SendHTTPRequest()` to always succeed:
+
+You can force the API `Steam_HTTP::SendHTTPRequest()` to always report success, by creating a file called `force_steamhttp_success.txt` inside the `steam_settings` folder.  
+
+Check the exaxmple file in the `steam_settings` folder
+
+---
+
 ## More configurations:
 Due to the various changes and additions, it became tedious to document everything,  
 so it is recommended to check each example file in the `steam_settings` folder.

post_build/steam_settings.EXAMPLE/download_steamhttp_requests.EXAMPLE.txt

@@ -0,0 +1,4 @@
+Rename this to: download_steamhttp_requests.txt to attempt to download external HTTP(S) requests made via Steam_HTTP::SendHTTPRequest().
+Make sure to:
+* Add the file `disable_lan_only.txt`
+* Remove the file `disable_networking.txt` if it's present

post_build/steam_settings.EXAMPLE/force_steamhttp_success.EXAMPLE.txt

@@ -0,0 +1 @@
+Rename this file to: force_steamhttp_success.txt to force the API Steam_HTTP::SendHTTPRequest() to always succeed.

post_build/steam_settings.EXAMPLE/http_online.EXAMPLE.txt

@@ -1 +0,0 @@
-Rename this to: http_online.txt to enable external HTTP download functionality.