Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Compress GAME_START message #3870

Closed
wants to merge 3 commits into from
Closed

Compress GAME_START message #3870

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8715a48
Compress GAME_START message
o01eg Jun 2, 2022
25d71f4
Add compression level parameter
o01eg Jun 3, 2022
1f0e188
Force to reset after finish message
o01eg Jun 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
49 changes: 37 additions & 12 deletions network/Message.cpp
View file
Open in desktop
Expand Up @@ -17,6 +17,8 @@
#include "../util/Version.h"
#include <boost/algorithm/string/erase.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/zlib.hpp>
#include <boost/serialization/deque.hpp>
#include <boost/serialization/list.hpp>
#include <boost/serialization/map.hpp>
Expand Down Expand Up @@ -179,12 +181,15 @@ Message GameStartMessage(bool single_player_game, int empire_id,
CombatLogManager& combat_logs, const SupplyManager& supply,
const std::map<int, PlayerInfo>& players,
GalaxySetupData galaxy_setup_data,
bool use_binary_serialization)
bool use_binary_serialization, int zlib_level)
{
std::ostringstream os;
{
boost::iostreams::filtering_ostream zos;
o01eg marked this conversation as resolved.
Show resolved Hide resolved
zos.push(boost::iostreams::zlib_compressor(boost::iostreams::zlib_params{zlib_level}));
zos.push(os);
if (use_binary_serialization) {
freeorion_bin_oarchive oa(os);
freeorion_bin_oarchive oa(zos);
oa << BOOST_SERIALIZATION_NVP(single_player_game)
<< BOOST_SERIALIZATION_NVP(empire_id)
<< BOOST_SERIALIZATION_NVP(current_turn);
Expand All @@ -200,7 +205,7 @@ Message GameStartMessage(bool single_player_game, int empire_id,
galaxy_setup_data.encoding_empire = empire_id;
oa << BOOST_SERIALIZATION_NVP(galaxy_setup_data);
} else {
freeorion_xml_oarchive oa(os);
freeorion_xml_oarchive oa(zos);
oa << BOOST_SERIALIZATION_NVP(single_player_game)
<< BOOST_SERIALIZATION_NVP(empire_id)
<< BOOST_SERIALIZATION_NVP(current_turn);
Expand All @@ -216,6 +221,9 @@ Message GameStartMessage(bool single_player_game, int empire_id,
galaxy_setup_data.encoding_empire = empire_id;
oa << BOOST_SERIALIZATION_NVP(galaxy_setup_data);
}
if (!zos.strict_sync()) {
zos.reset();
}
}
return Message{Message::MessageType::GAME_START, std::move(os).str()};
}
Expand All @@ -227,12 +235,15 @@ Message GameStartMessage(bool single_player_game, int empire_id,
const std::map<int, PlayerInfo>& players,
const OrderSet& orders, const SaveGameUIData* ui_data,
GalaxySetupData galaxy_setup_data,
bool use_binary_serialization)
bool use_binary_serialization, int zlib_level)
{
std::ostringstream os;
{
boost::iostreams::filtering_ostream zos;
zos.push(boost::iostreams::zlib_compressor(boost::iostreams::zlib_params{zlib_level}));
zos.push(os);
if (use_binary_serialization) {
freeorion_bin_oarchive oa(os);
freeorion_bin_oarchive oa(zos);
oa << BOOST_SERIALIZATION_NVP(single_player_game)
<< BOOST_SERIALIZATION_NVP(empire_id)
<< BOOST_SERIALIZATION_NVP(current_turn);
Expand All @@ -255,7 +266,7 @@ Message GameStartMessage(bool single_player_game, int empire_id,
galaxy_setup_data.encoding_empire = empire_id;
oa << BOOST_SERIALIZATION_NVP(galaxy_setup_data);
} else {
freeorion_xml_oarchive oa(os);
freeorion_xml_oarchive oa(zos);
oa << BOOST_SERIALIZATION_NVP(single_player_game)
<< BOOST_SERIALIZATION_NVP(empire_id)
<< BOOST_SERIALIZATION_NVP(current_turn);
Expand Down Expand Up @@ -285,6 +296,9 @@ Message GameStartMessage(bool single_player_game, int empire_id,
galaxy_setup_data.encoding_empire = empire_id;
oa << BOOST_SERIALIZATION_NVP(galaxy_setup_data);
}
if (!zos.strict_sync()) {
zos.reset();
}
}
return Message{Message::MessageType::GAME_START, std::move(os).str()};
}
Expand All @@ -296,12 +310,15 @@ Message GameStartMessage(bool single_player_game, int empire_id,
const std::map<int, PlayerInfo>& players,
const OrderSet& orders, const std::string* save_state_string,
GalaxySetupData galaxy_setup_data,
bool use_binary_serialization)
bool use_binary_serialization, int zlib_level)
{
std::ostringstream os;
{
boost::iostreams::filtering_ostream zos;
zos.push(boost::iostreams::zlib_compressor(boost::iostreams::zlib_params{zlib_level}));
zos.push(os);
if (use_binary_serialization) {
freeorion_bin_oarchive oa(os);
freeorion_bin_oarchive oa(zos);
oa << BOOST_SERIALIZATION_NVP(single_player_game)
<< BOOST_SERIALIZATION_NVP(empire_id)
<< BOOST_SERIALIZATION_NVP(current_turn);
Expand All @@ -324,7 +341,7 @@ Message GameStartMessage(bool single_player_game, int empire_id,
galaxy_setup_data.encoding_empire = empire_id;
oa << BOOST_SERIALIZATION_NVP(galaxy_setup_data);
} else {
freeorion_xml_oarchive oa(os);
freeorion_xml_oarchive oa(zos);
oa << BOOST_SERIALIZATION_NVP(single_player_game)
<< BOOST_SERIALIZATION_NVP(empire_id)
<< BOOST_SERIALIZATION_NVP(current_turn);
Expand Down Expand Up @@ -354,6 +371,9 @@ Message GameStartMessage(bool single_player_game, int empire_id,
galaxy_setup_data.encoding_empire = empire_id;
oa << BOOST_SERIALIZATION_NVP(galaxy_setup_data);
}
if (!zos.strict_sync()) {
zos.reset();
}
}
return Message{Message::MessageType::GAME_START, std::move(os).str()};
}
Expand Down Expand Up @@ -853,8 +873,11 @@ void ExtractGameStartMessageData(std::string text, bool& single_player_game, int
try {
// first attempt binary deserialziation
std::istringstream is(text);
boost::iostreams::filtering_istream zis;
zis.push(boost::iostreams::zlib_decompressor());
zis.push(is);

freeorion_bin_iarchive ia(is);
freeorion_bin_iarchive ia(zis);
ia >> BOOST_SERIALIZATION_NVP(single_player_game)
>> BOOST_SERIALIZATION_NVP(empire_id)
>> BOOST_SERIALIZATION_NVP(current_turn);
Expand Down Expand Up @@ -898,8 +921,11 @@ void ExtractGameStartMessageData(std::string text, bool& single_player_game, int
if (try_xml) {
// if binary deserialization failed, try more-portable XML deserialization
std::istringstream is(text);
boost::iostreams::filtering_istream zis;
zis.push(boost::iostreams::zlib_decompressor());
zis.push(is);

freeorion_xml_iarchive ia(is);
freeorion_xml_iarchive ia(zis);
ia >> BOOST_SERIALIZATION_NVP(single_player_game)
>> BOOST_SERIALIZATION_NVP(empire_id)
>> BOOST_SERIALIZATION_NVP(current_turn);
Expand Down Expand Up @@ -945,7 +971,6 @@ void ExtractGameStartMessageData(std::string text, bool& single_player_game, int
} catch (const std::exception& err) {
ErrorLogger() << "ExtractGameStartMessageData(...) failed! Message probably long, so not outputting to log.\n"
<< "Error: " << err.what();
TraceLogger() << "Message: " << text;
throw err;
}
}
Expand Down
9 changes: 6 additions & 3 deletions network/Message.h
View file
Open in desktop
Expand Up @@ -207,7 +207,8 @@ FO_COMMON_API Message GameStartMessage(
const EmpireManager& empires, const Universe& universe,
const SpeciesManager& species, CombatLogManager& combat_logs,
const SupplyManager& supply, const std::map<int, PlayerInfo>& players,
GalaxySetupData galaxy_setup_data, bool use_binary_serialization);
GalaxySetupData galaxy_setup_data, bool use_binary_serialization,
int zlib_level);

/** creates a GAME_START message. Contains the initial game state visible to
* the player. Also includes data loaded from a saved game. */
Expand All @@ -217,7 +218,8 @@ FO_COMMON_API Message GameStartMessage(
const SpeciesManager& species, CombatLogManager& combat_logs,
const SupplyManager& supply, const std::map<int, PlayerInfo>& players,
const OrderSet& orders, const SaveGameUIData* ui_data,
GalaxySetupData galaxy_setup_data, bool use_binary_serialization);
GalaxySetupData galaxy_setup_data, bool use_binary_serialization,
int zlib_level);

/** creates a GAME_START message. Contains the initial game state visible to
* the player. Also includes state string loaded from a saved game. */
Expand All @@ -227,7 +229,8 @@ FO_COMMON_API Message GameStartMessage(
const SpeciesManager& species, CombatLogManager& combat_logs,
const SupplyManager& supply, const std::map<int, PlayerInfo>& players,
const OrderSet& orders, const std::string* save_state_string,
GalaxySetupData galaxy_setup_data, bool use_binary_serialization);
GalaxySetupData galaxy_setup_data, bool use_binary_serialization,
int zlib_level);

/** creates a HOST_SP_GAME acknowledgement message. The \a player_id is the ID
* of the receiving player. This message should only be sent by the server.*/
Expand Down
18 changes: 12 additions & 6 deletions server/ServerApp.cpp
View file
Open in desktop
Expand Up @@ -8,6 +8,7 @@
#include <boost/filesystem/fstream.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/functional/hash.hpp>
#include <boost/iostreams/filter/zlib.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/uuid/random_generator.hpp>
#include <boost/uuid/uuid_io.hpp>
Expand Down Expand Up @@ -920,7 +921,7 @@ void ServerApp::SendNewGameStartMessages() {
m_universe, GetSpeciesManager(),
GetCombatLogManager(), GetSupplyManager(),
player_info_map, m_galaxy_setup_data,
use_binary_serialization));
use_binary_serialization, boost::iostreams::zlib::default_compression));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

are there plans to do something other than default_compression ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suppose it possible to introduce some option.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm working on porting over the user option and client-server compression usage requests from #3854. Since those work as bool flags, my message creation functions look like:

Message MakeMessage(const std::string& data, bool use_compression) {
    std::ostringstream os;
    {
        boost::iostreams::zlib_params params;
        params.level = boost::iostreams::zlib::no_compression;
        if (use_compression)
            params.level = boost::iostreams::zlib::default_compression;

        boost::iostreams::filtering_ostream zos;
        zos.push(boost::iostreams::zlib_compressor(params));
        zos.push(os);

        freeorion_xml_oarchive oa(zos);
        oa << BOOST_SERIALIZATION_NVP(data);
    }
    return Message{Message::MessageType::SOME_MSG, std::move(os).str()};
}

Would it be preferable for the user option to be the compression level, rather than just a true/false toggle?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hard to say, in most cases big messages are sent from server to client.

}
}

Expand Down Expand Up @@ -1498,15 +1499,17 @@ void ServerApp::LoadGameInit(const std::vector<PlayerSaveGameData>& player_save_
m_current_turn, m_empires, m_universe,
GetSpeciesManager(), GetCombatLogManager(),
GetSupplyManager(), player_info_map, *orders, sss,
m_galaxy_setup_data, use_binary_serialization));
m_galaxy_setup_data, use_binary_serialization,
boost::iostreams::zlib::default_compression));

} else if (client_type == Networking::ClientType::CLIENT_TYPE_HUMAN_PLAYER) {
player_connection->SendMessage(GameStartMessage(m_single_player_game, empire_id,
m_current_turn, m_empires, m_universe,
GetSpeciesManager(), GetCombatLogManager(),
GetSupplyManager(), player_info_map, *orders,
psgd.ui_data.get(), m_galaxy_setup_data,
use_binary_serialization));
use_binary_serialization,
boost::iostreams::zlib::default_compression));

} else if (client_type == Networking::ClientType::CLIENT_TYPE_HUMAN_OBSERVER ||
client_type == Networking::ClientType::CLIENT_TYPE_HUMAN_MODERATOR)
Expand All @@ -1516,7 +1519,8 @@ void ServerApp::LoadGameInit(const std::vector<PlayerSaveGameData>& player_save_
m_current_turn, m_empires, m_universe,
GetSpeciesManager(), GetCombatLogManager(),
GetSupplyManager(), player_info_map,
m_galaxy_setup_data, use_binary_serialization));
m_galaxy_setup_data, use_binary_serialization,
boost::iostreams::zlib::default_compression));
} else {
ErrorLogger() << "ServerApp::CommonGameInit unsupported client type: skipping game start message.";
}
Expand Down Expand Up @@ -1841,7 +1845,8 @@ void ServerApp::AddObserverPlayerIntoGame(const PlayerConnectionPtr& player_conn
m_current_turn, m_empires, m_universe,
GetSpeciesManager(), GetCombatLogManager(),
GetSupplyManager(), player_info_map,
m_galaxy_setup_data, use_binary_serialization));
m_galaxy_setup_data, use_binary_serialization,
boost::iostreams::zlib::default_compression));
} else {
ErrorLogger() << "ServerApp::CommonGameInit unsupported client type: skipping game start message.";
}
Expand Down Expand Up @@ -2056,7 +2061,8 @@ int ServerApp::AddPlayerIntoGame(const PlayerConnectionPtr& player_connection, i
GetSupplyManager(), player_info_map, orders,
ui_data,
m_galaxy_setup_data,
use_binary_serialization));
use_binary_serialization,
boost::iostreams::zlib::default_compression));

return empire_id;
}
Expand Down