Improved error handling of gta_data_service. (#412)

* Bug fix for #403

* Fixed Issue #403

* Minor fixes.

* Improved error handling of gta_data_service.

Fixed an issue where the game crashes when throwing invalid json format exception.

* Minor fixes.

* Minor fixes.

* Improved error handling of gta_data_service.

Also extended the timeout to 30s.

* Improved error handling of gta_data_service.

- Fixed an issue where http_request doesn't handle HEAD requests and socket.recv error correctly.

- Limited gta_data_service to have only 1 active http request.

* Minor fixes.

* Minor fixes.

BigBaseV2/src/api/http_request.hpp

@@ -598,8 +598,16 @@ namespace http
             {
                 const auto size = socket.recv(tempBuffer.data(), tempBuffer.size(),
                     (timeout.count() >= 0) ? getRemainingMilliseconds(stopTime) : -1);
-                if (size == 0) // disconnected
+
+                if (size == 0)
+                {
+                    // download complete
                     return response;
+                }
+                else if (size < 0)
+                {
+                    throw ResponseError("Socket error");
+                }
 
                 responseData.insert(responseData.end(), tempBuffer.begin(), tempBuffer.begin() + size);
 
@@ -683,8 +691,11 @@ namespace http
 
                 if (state == State::body)
                 {
-                    // Content-Length must be ignored if Transfer-Encoding is received
+                    if (method == "HEAD") // Body must be ignored for HEAD requests
-                    if (chunkedResponse)
+                    {
+                        return response;
+                    }
+                    else if (chunkedResponse) // Content-Length must be ignored if Transfer-Encoding is received
                     {
                         for (;;)
                         {

BigBaseV2/src/api/remote.hpp

@@ -22,42 +22,47 @@ namespace big::remote
 
 	inline bool update_binary(const std::string_view file_url, const std::filesystem::path& file_location, const std::filesystem::path& etag_location)
 	{
-		try
+		std::string local_etag = "";
-		{
+		std::string remote_etag = "";
-			std::string local_etag = "";
-			std::string remote_etag = "";
 
-			const std::vector<std::string> headers = { "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/6.0" };
+		try {
 
-			try {
+			{
+				std::ifstream file_etag_ifstream(etag_location, std::ios::binary);
+				std::stringstream file_etag_stringstream;
+				file_etag_stringstream << file_etag_ifstream.rdbuf();
+				local_etag = file_etag_stringstream.str();
+			}
 
+			if (!local_etag.empty())
+			{
+				http::Request req(file_url.data());
+				http::Response res = req.send("HEAD", "", {}, 20s);
+
+				if (res.status == http::Response::Status::Ok)
 				{
-					std::ifstream file_etag_ifstream(etag_location, std::ios::binary);
-					std::stringstream file_etag_stringstream;
-					file_etag_stringstream << file_etag_ifstream.rdbuf();
-					local_etag = file_etag_stringstream.str();
-				}
-
-				if (!local_etag.empty())
-				{
-					http::Request req(file_url.data());
-					http::Response res = req.send("HEAD", "", headers, 15s);
-
 					remote_etag = get_etag_from_headers(res.headers);
 
 					if (remote_etag == local_etag)
 					{
-						return false;
+						return true;
 					}
 				}
+				else
+				{
+					throw std::exception(std::to_string(res.status).c_str());
+				}
 			}
-			catch (const std::exception& e)
+		}
-			{
+		catch (const std::exception& e)
-				LOG(INFO) << "Update Error: " << e.what();
+		{
-			}
+			LOG(WARNING) << "Update Error (HEAD): " << e.what();
+		}
 
+		try
+		{
 			http::Request req(file_url.data());
-			http::Response res = req.send("GET", "", headers, 30s);
+			http::Response res = req.send("GET", "", {}, 20s);
 
 			if (res.status == http::Response::Status::Ok)
 			{
@@ -72,12 +77,14 @@ namespace big::remote
 
 				return true;
 			}
-
+			else
-			return false;
+			{
+				throw std::exception(std::to_string(res.status).c_str());
+			}
 		}
 		catch (const std::exception& e)
 		{
-			LOG(INFO) << "Failed to download binary, is the host down?: " << e.what();
+			LOG(WARNING) << "Update Error (GET): " << e.what();
 		}
 
 		return false;

BigBaseV2/src/services/gta_data/gta_data_service.cpp

@@ -9,31 +9,40 @@ namespace big
 {
 	gta_data_service::gta_data_service()
 	{
-		const std::string url_prefix = "http://github-proxy.damon.sh/DurtyFree/gta-v-data-dumps/master/";
+		g_thread_pool->push([this] {
+			while (!g_running)
+				std::this_thread::sleep_for(1s);
 
-		load_from_file(
+			const std::string url_prefix = "http://github-proxy.damon.sh/DurtyFree/gta-v-data-dumps/master/";
-			"./lib/vehicles.json",
-			"./lib/vehicles_etag.txt",
-			url_prefix + "vehicles.json",
-			&gta_data_service::load_vehicles,
-			"Vehicle"
-		);
 
-		load_from_file(
+			this->load_from_file(
-			"./lib/peds.json",
+				"./lib/vehicles.json",
-			"./lib/peds_etag.txt",
+				"./lib/vehicles_etag.txt",
-			url_prefix + "peds.json",
+				url_prefix + "vehicles.json",
-			&gta_data_service::load_peds,
+				&gta_data_service::load_vehicles,
-			"Ped"
+				"Vehicle"
-		);
+			);
 
-		load_from_file(
+			std::this_thread::sleep_for(1s);
-			"./lib/weapons.json",
+
-			"./lib/weapons_etag.txt",
+			this->load_from_file(
-			url_prefix + "weapons.json",
+				"./lib/peds.json",
-			&gta_data_service::load_weapons,
+				"./lib/peds_etag.txt",
-			"Weapon"
+				url_prefix + "peds.json",
-		);
+				&gta_data_service::load_peds,
+				"Ped"
+			);
+
+			std::this_thread::sleep_for(1s);
+
+			this->load_from_file(
+				"./lib/weapons.json",
+				"./lib/weapons_etag.txt",
+				url_prefix + "weapons.json",
+				&gta_data_service::load_weapons,
+				"Weapon"
+			);
+		});
 
 		g_gta_data_service = this;
 	}
@@ -127,64 +136,85 @@ namespace big
 	}
 
 
-	void gta_data_service::load_from_file(std::string file_path, std::string etag_path, std::string url, bool(gta_data_service::* load_func)(file), std::string data_name)
+	void gta_data_service::load_from_file(
-	{
+		std::string file_path, std::string etag_path, std::string url,
+		bool(gta_data_service::* load_func)(std::filesystem::path), std::string data_name
+	) {
 		file file_to_load(g_file_manager->get_project_file(file_path));
+		file file_etag(g_file_manager->get_project_file(etag_path));
+		auto file_to_load_path = file_to_load.get_path();
+		auto file_etag_path = file_etag.get_path();
 
-		if (file_to_load.exists())
+		bool up_to_date = false;
+
+		for (int retry = 0; retry < 3 && g_running; retry++)
 		{
-			if ((this->*load_func)(file_to_load))
+			LOG(INFO) << "Checking update (attempt: " << (retry + 1) << "/3): " << data_name;
-			{
-				LOG(INFO) << "Data loaded: " + data_name;
-			}
-			else
-			{
-				LOG(INFO) << "Data invalid: " + data_name;
-			}
-		}
 
-		g_thread_pool->push([this, file_path, etag_path, url, load_func, data_name]() {
+			try
-			file file_to_load(g_file_manager->get_project_file(file_path));
-			file file_etag(g_file_manager->get_project_file(etag_path));
-
-			for (int retry = 0; retry < 2; retry++)
 			{
 				bool ret = remote::update_binary(
 					url,
-					file_to_load.get_path(),
+					file_to_load_path,
-					file_etag.get_path()
+					file_etag_path
 				);
 
 				if (ret)
 				{
-					if ((this->*load_func)(file_to_load))
+					up_to_date = true;
+
+					LOG(INFO) << "Data updated: " << data_name;
+
+					if ((this->*load_func)(file_to_load_path))
 					{
 						LOG(INFO) << "Data loaded: " + data_name;
+						break;
 					}
 					else
 					{
-						LOG(INFO) << "Data invalid: " + data_name;
+						std::filesystem::remove(file_to_load_path);
-
+						std::filesystem::remove(file_etag_path);
-						try
-						{
-							std::filesystem::remove(file_to_load.get_path());
-							std::filesystem::remove(file_etag.get_path());
-						}
-						catch (...) { }
 					}
-					break;
-				}
-				else if (!file_to_load.exists())
-				{
-					LOG(WARNING) << "Failed to download data: " + data_name;
 				}
 			}
-		});
+			catch (...)
+			{
+				LOG(WARNING) << "Data invalid: " + data_name;
+			}
+
+			std::this_thread::sleep_for(2s);
+		}
+
+		if (!up_to_date)
+		{
+			LOG(WARNING) << "Data not updated: " + data_name;
+
+			try
+			{
+				if (file_to_load.exists())
+				{
+					if ((this->*load_func)(file_to_load_path))
+					{
+						LOG(INFO) << "Cache loaded: " + data_name;
+					}
+					else
+					{
+						std::filesystem::remove(file_to_load_path);
+						std::filesystem::remove(file_etag_path);
+						throw std::exception("");
+					}
+				}
+			}
+			catch (...)
+			{
+				LOG(WARNING) << "Cache invalid: " + data_name;
+			}
+		}
 	}
 
-	bool gta_data_service::load_vehicles(file file_to_load)
+	bool gta_data_service::load_vehicles(std::filesystem::path path)
 	{
-		std::ifstream file(file_to_load.get_path());
+		std::ifstream file(path);
 		nlohmann::json all_vehicles;
 
 		try
@@ -235,9 +265,9 @@ namespace big
 	}
 
 
-	bool gta_data_service::load_peds(file file_to_load)
+	bool gta_data_service::load_peds(std::filesystem::path path)
 	{
-		std::ifstream file(file_to_load.get_path());
+		std::ifstream file(path);
 		nlohmann::json all_peds;
 
 		try
@@ -286,9 +316,9 @@ namespace big
 	}
 
 
-	bool gta_data_service::load_weapons(file file_to_load)
+	bool gta_data_service::load_weapons(std::filesystem::path path)
 	{
-		std::ifstream file(file_to_load.get_path());
+		std::ifstream file(path);
 		nlohmann::json all_weapons;
 
 		try

BigBaseV2/src/services/gta_data/gta_data_service.hpp

@@ -41,11 +41,14 @@ namespace big
 		const std::vector<weapon_item>& get_weapon_arr();
 
 	private:
-		void load_from_file(std::string file_path, std::string etag_path, std::string url, bool(gta_data_service::* load_func)(file), std::string data_name);
+		void load_from_file(
+			std::string file_path, std::string etag_path, std::string url, 
+			bool(gta_data_service::* load_func)(std::filesystem::path), std::string data_name
+		);
 
-		bool load_vehicles(file file_to_load);
+		bool load_vehicles(std::filesystem::path path);
-		bool load_peds(file file_to_load);
+		bool load_peds(std::filesystem::path path);
-		bool load_weapons(file file_to_load);
+		bool load_weapons(std::filesystem::path path);
 	};
 
 	inline gta_data_service* g_gta_data_service{};