Initial commit

engine/network/CMakeLists.txt

@@ -0,0 +1,156 @@
+add_library(NetworkPlayer STATIC $<TARGET_OBJECTS:RakNet>)
+add_library(NetworkStudio STATIC $<TARGET_OBJECTS:RakNet>)
+add_library(NetworkServer STATIC $<TARGET_OBJECTS:RakNet>)
+
+set(SOURCES
+  src/API.cpp
+  src/BoostAppend.cpp
+  src/BoostAppend.hpp
+  src/ChatFilter.cpp
+  src/ChatFilter.hpp
+  src/Client.cpp
+  src/Client.hpp
+  src/ClientReplicator.cpp
+  src/ClientReplicator.hpp
+  src/ClusterUpdateBuffer.cpp
+  src/ClusterUpdateBuffer.hpp
+  src/Compressor.cpp
+  src/Compressor.hpp
+  src/CrispProxy.hpp
+  src/ConcurrentRakPeer.cpp
+  src/ConcurrentRakPeer.hpp
+  src/DataBlockEncryptor.cpp
+  src/DataBlockEncryptor.hpp
+  src/Dictionary.cpp
+  src/Dictionary.hpp
+  src/DirectPhysicsReceiver.cpp
+  src/DirectPhysicsReceiver.hpp
+  src/ErrorCompPhysicsSender.cpp
+  src/ErrorCompPhysicsSender.hpp
+  src/ErrorCompPhysicsSender2.cpp
+  src/ErrorCompPhysicsSender2.hpp
+  src/GameConfigurer.cpp
+  src/GameConfigurer.hpp
+  src/GamePerfMonitor.cpp
+  src/GamePerfMonitor.hpp
+  src/GuidRegistryService.cpp
+  src/GuidRegistryService.hpp
+  src/InterpolatingPhysicsReceiver.cpp
+  src/InterpolatingPhysicsReceiver.hpp
+  src/Item.cpp
+  src/Item.hpp
+  src/Marker.cpp
+  src/Marker.hpp
+  src/MechanismItem.cpp
+  src/MechanismItem.hpp
+  src/MovementHistoryJob.cpp
+  src/MovementHistoryJob.hpp
+  src/NetworkClusterPacketCache.cpp
+  src/NetworkFilter.cpp
+  src/NetworkFilter.hpp
+  src/NetworkOwnerJob.cpp
+  src/NetworkOwnerJob.hpp
+  src/NetworkPacketCache.cpp
+  src/NetworkProfiler.cpp
+  src/NetworkProfiler.hpp
+  src/NetworkSettings.cpp
+  src/NetworkSettings.hpp
+  src/PacketIds.hpp
+  src/Peer.cpp
+  src/Peer.hpp
+  src/PersistentDataStore.cpp
+  src/PersistentDataStore.hpp
+  src/PhysicsReceiver.cpp
+  src/PhysicsReceiver.hpp
+  src/PhysicsSender.cpp
+  src/PhysicsSender.hpp
+  src/Player.cpp
+  src/Players.cpp
+  src/PropertySynchronization.hpp
+  src/Replicator.ChangePropertyItem.cpp
+  src/Replicator.ChangePropertyItem.hpp
+  src/Replicator.cpp
+  src/Replicator.DeleteInstanceItem.cpp
+  src/Replicator.DeleteInstanceItem.hpp
+  src/Replicator.EventInvocationItem.cpp
+  src/Replicator.EventInvocationItem.hpp
+  src/Replicator.GCJob.cpp
+  src/Replicator.GCJob.hpp
+  src/Replicator.hpp
+  src/Replicator.ItemSender.cpp
+  src/Replicator.ItemSender.hpp
+  src/Replicator.JoinDataItem.cpp
+  src/Replicator.JoinDataItem.hpp
+  src/Replicator.MarkerItem.cpp
+  src/Replicator.MarkerItem.hpp
+  src/Replicator.NewInstanceItem.cpp
+  src/Replicator.NewInstanceItem.hpp
+  src/Replicator.PingBackItem.cpp
+  src/Replicator.PingBackItem.hpp
+  src/Replicator.PingItem.cpp
+  src/Replicator.PingItem.hpp
+  src/Replicator.PingJob.hpp
+  src/Replicator.ProcessPacketsJob.hpp
+  src/Replicator.ReferencePropertyChangedItem.cpp
+  src/Replicator.ReferencePropertyChangedItem.hpp
+  src/Replicator.SendDataJob.hpp
+  src/Replicator.StatsItem.cpp
+  src/Replicator.StatsItem.hpp
+  src/Replicator.StreamJob.cpp
+  src/Replicator.StreamJob.hpp
+  src/Replicator.TagItem.cpp
+  src/Replicator.TagItem.hpp
+  src/ReplicatorStats.cpp
+  src/ReplicatorStats.hpp
+  src/Rijndael-Boxes.hpp
+  src/rijndael.cpp
+  src/Rijndael.hpp
+  src/RoundRobinPhysicsSender.cpp
+  src/RoundRobinPhysicsSender.hpp
+  src/Server.cpp
+  src/Server.hpp
+  src/ServerReplicator.cpp
+  src/ServerReplicator.hpp
+  src/Streaming.cpp
+  src/Streaming.hpp
+  src/StreamingUtil.hpp
+  src/TopNErrorsPhysicsSender.cpp
+  src/TopNErrorsPhysicsSender.hpp
+  src/Util.cpp
+  src/Util.hpp
+  src/WebChatFilter.cpp
+  src/WebChatFilter.hpp
+)
+
+set(PRECOMPILED_HEADERS
+  src/API.hpp
+  src/NetworkClusterPacketCache.hpp
+  src/NetworkOwner.hpp
+  src/NetworkPacketCache.hpp
+  src/Player.hpp
+  src/Players.hpp
+)
+
+if(AYA_OS_WINDOWS)
+  list(APPEND SOURCES
+    src/CrashReporter.cpp
+    src/CrashReporter.hpp
+    src/RakNetFast.hpp
+  )
+endif()
+
+target_include_directories(NetworkPlayer PUBLIC src PRIVATE ${THIRD_PARTY_DIR}/BulletPhysics/src ${THIRD_PARTY_DIR}/RakNet/src ${ENGINE_DIR}/app/src ${ENGINE_DIR}/core/src ${ENGINE_DIR}/3d/src ${ENGINE_DIR}/gfx/src)
+target_include_directories(NetworkStudio PUBLIC src PRIVATE ${THIRD_PARTY_DIR}/BulletPhysics/src ${THIRD_PARTY_DIR}/RakNet/src ${ENGINE_DIR}/app/src ${ENGINE_DIR}/core/src ${ENGINE_DIR}/3d/src ${ENGINE_DIR}/gfx/src)
+target_include_directories(NetworkServer PUBLIC src PRIVATE ${THIRD_PARTY_DIR}/BulletPhysics/src ${THIRD_PARTY_DIR}/RakNet/src ${ENGINE_DIR}/app/src ${ENGINE_DIR}/core/src ${ENGINE_DIR}/3d/src ${ENGINE_DIR}/gfx/src)
+
+target_precompile_headers(NetworkPlayer PUBLIC ${PRECOMPILED_HEADERS})
+target_precompile_headers(NetworkStudio PUBLIC ${PRECOMPILED_HEADERS})
+target_precompile_headers(NetworkServer PUBLIC ${PRECOMPILED_HEADERS})
+
+target_sources(NetworkPlayer PRIVATE ${SOURCES})
+target_sources(NetworkStudio PRIVATE ${SOURCES})
+target_sources(NetworkServer PRIVATE ${SOURCES})
+
+target_link_libraries(NetworkPlayer PRIVATE $<TARGET_OBJECTS:AppPlayer>)
+target_link_libraries(NetworkStudio PRIVATE $<TARGET_OBJECTS:AppStudio>)
+target_link_libraries(NetworkServer PRIVATE $<TARGET_OBJECTS:AppServer>)

engine/network/src/API.cpp

@@ -0,0 +1,263 @@
+
+
+
+// third time's the charm
+#include "API.hpp"
+
+#include "Client.hpp"
+#include "Server.hpp"
+#include "ServerReplicator.hpp"
+#include "ClientReplicator.hpp"
+#include "Players.hpp"
+#include "Player.hpp"
+#include "Marker.hpp"
+
+#include "NetworkSettings.hpp"
+#include "DataModel/DataModel.hpp"
+
+#include "DataModel/GlobalSettings.hpp"
+
+#include "DataModel/HackDefines.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+#include "GuidRegistryService.hpp"
+#include "RakNet/RakNetVersion.hpp"
+#include "Utility/Statistics.hpp"
+#include "Utility/URL.hpp"
+#include "Utility/Utilities.hpp"
+
+#include "FastLog.hpp"
+
+#ifdef __APPLE__
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <sys/user.h>
+#elif defined(__unix__) || defined(__posix__)
+#include <sys/ptrace.h>
+#include <unistd.h>
+#include <fstream>
+#endif
+
+// RakNet
+#include "RakNet/StringCompressor.hpp"
+#include "RakNet/StringTable.hpp"
+
+#include <boost/algorithm/string.hpp>
+
+FASTSTRINGVARIABLE(ClientExternalBrowserUserAgent, "Roblox/WinInet")
+
+std::string Aya::Network::password;
+
+#if RAKNET_PROTOCOL_VERSION != 5
+#error
+#endif
+
+
+AYA_REGISTER_CLASS(Aya::Network::Client);
+AYA_REGISTER_CLASS(Aya::Network::Server);
+AYA_REGISTER_CLASS(Aya::Network::Player);
+AYA_REGISTER_CLASS(Aya::Network::Players);
+AYA_REGISTER_CLASS(Aya::NetworkSettings);
+AYA_REGISTER_CLASS(Aya::Network::Peer);
+AYA_REGISTER_CLASS(Aya::Network::Marker);
+AYA_REGISTER_CLASS(Aya::Network::Replicator);
+AYA_REGISTER_CLASS(Aya::Network::ServerReplicator);
+AYA_REGISTER_CLASS(Aya::Network::ClientReplicator);
+AYA_REGISTER_CLASS(Aya::Network::GuidRegistryService);
+
+AYA_REGISTER_ENUM(PacketPriority);
+AYA_REGISTER_ENUM(PacketReliability);
+AYA_REGISTER_ENUM(Aya::Network::FilterResult);
+AYA_REGISTER_ENUM(Aya::Network::Player::MembershipType);
+AYA_REGISTER_ENUM(Aya::Network::Player::ChatMode);
+AYA_REGISTER_ENUM(Aya::Network::Players::PlayerChatType);
+AYA_REGISTER_ENUM(Aya::Network::Players::ChatOption);
+AYA_REGISTER_ENUM(Aya::NetworkSettings::PhysicsSendMethod);
+AYA_REGISTER_ENUM(Aya::NetworkSettings::PhysicsReceiveMethod);
+
+namespace Aya
+{
+namespace Network
+{
+// Prevent string compressors from being created at the same time
+class SafeInitFree
+{
+public:
+    SafeInitFree()
+    {
+        RakNet::StringCompressor::AddReference();
+        RakNet::StringTable::AddReference();
+    }
+    ~SafeInitFree()
+    {
+        RakNet::StringCompressor::RemoveReference();
+        RakNet::StringTable::RemoveReference();
+    }
+};
+} // namespace Network
+} // namespace Aya
+
+static bool _isPlayerAuthenticationEnabled;
+
+bool Aya::Network::isPlayerAuthenticationEnabled()
+{
+    return _isPlayerAuthenticationEnabled;
+}
+
+bool Aya::Network::isNetworkClient(const Instance* context)
+{
+    return ServiceProvider::find<Client>(context) != NULL;
+}
+
+#if defined(AYA_SERVER)
+static shared_ptr<Aya::Network::ServerReplicator> createSecureReplicator(
+    RakNet::SystemAddress a, Aya::Network::Server* s, Aya::NetworkSettings* networkSettings)
+{
+    return Aya::Creatable<Aya::Instance>::create<Aya::Network::CheatHandlingServerReplicator>(a, s, networkSettings);
+}
+#endif
+
+void Aya::Network::init()
+{
+    static SafeInitFree safeInitFree;
+
+    Client::classDescriptor();
+    Server::classDescriptor();
+    Player::classDescriptor();
+    Players::classDescriptor();
+    GlobalAdvancedSettings::classDescriptor();
+    NetworkSettings::classDescriptor();
+    NetworkSettings::singleton();
+
+#if defined(AYA_SERVER)
+    _isPlayerAuthenticationEnabled = true;
+    Server::createReplicator = createSecureReplicator;
+#endif
+}
+
+void Aya::Network::setPassword(const char* _password)
+{
+    if (_password)
+        password = _password;
+}
+bool Aya::Network::isTrustedContent(const char* url)
+{
+    if (!Aya::ContentProvider::isUrl(url))
+        return false;
+
+    bool kSkipNetworkTrustedContentCheck = true;
+    if (kSkipNetworkTrustedContentCheck)
+    {
+        return true;
+    }
+
+    std::string urlString(url);
+    boost::algorithm::to_lower(urlString);
+
+    bool isRobloxLabsUrl = false;
+
+    const Aya::Url baseUrlParsed = Aya::Url::fromString(GetBaseURL());
+
+    size_t foundPos = urlString.find(baseUrlParsed.host().c_str() + '/');
+    if (foundPos == std::string::npos)
+    {
+        foundPos = urlString.find('.' + baseUrlParsed.host().c_str() + '/');
+        if (foundPos == std::string::npos)
+            return false;
+
+        isRobloxLabsUrl = true;
+    }
+
+    urlString = urlString.substr(foundPos, std::string::npos); // remove all of string before URL_IDENTIFIER
+
+    // put our iterator at end of URL_IDENTIFIER
+    if (isRobloxLabsUrl)
+        foundPos = sizeof('.' + baseUrlParsed.host().c_str() + '/') - 1;
+    else
+        foundPos = sizeof(baseUrlParsed.host().c_str()) - 1;
+
+    while (foundPos < urlString.size() && (urlString[foundPos] == '\\' || urlString[foundPos] == '/'))
+        ++foundPos;
+
+    if (foundPos >= urlString.size())
+        return false;
+
+    return urlString.substr(foundPos, 5) == "asset" || urlString.substr(foundPos, 4) == "game" || urlString.substr(foundPos, 9) == "analytics" ||
+           urlString.substr(foundPos, 3) == "ide" || urlString.substr(foundPos, 6) == "images" || urlString.substr(foundPos, 6) == "thumbs" ||
+           urlString.substr(foundPos, 2) == "ui" || urlString.substr(foundPos, 11) == "persistence" || urlString.substr(foundPos, 8) == "rolesets" ||
+           urlString.substr(foundPos, 4) == "auth" || urlString.substr(foundPos, 8) == "currency" ||
+           urlString.substr(foundPos, 11) == "marketplace" || urlString.substr(foundPos, 9) == "ownership" ||
+           urlString.substr(foundPos, 13) == "placerolesets";
+}
+
+namespace
+{
+
+bool isDebuggerPresent()
+{
+#if defined(_WIN32)
+    return ::IsDebuggerPresent() != 0;
+#elif defined(__APPLE__)
+    // macOS/iOS
+    int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid()};
+    struct kinfo_proc info = {};
+    size_t size = sizeof(info);
+
+    if (sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &size, NULL, 0) == 0)
+        return (info.kp_proc.p_flag & P_TRACED) != 0;
+
+    return false;
+#elif defined(__linux__)
+    // Linux - check /proc/self/status for TracerPid
+    std::ifstream statusFile("/proc/self/status");
+    std::string line;
+
+    while (std::getline(statusFile, line))
+    {
+        if (line.compare(0, 10, "TracerPid:") == 0)
+        {
+            int tracerPid = std::stoi(line.substr(10));
+            return tracerPid != 0;
+        }
+    }
+    return false;
+#elif defined(__unix__) || defined(__posix__)
+    // Generic Unix fallback using ptrace
+    if (ptrace(PTRACE_TRACEME, 0, 1, 0) < 0)
+        return true; // Already being traced
+
+    ptrace(PTRACE_DETACH, 0, 1, 0);
+    return false;
+#else
+    return false; // Unknown platform
+#endif
+}
+
+void isDebuggedDirectThreadFunc(weak_ptr<Aya::DataModel> weakDataModel)
+{
+    if (IsInsecureMode())
+        return;
+
+    static const int kSleepBetweenChecksMillis = 1500;
+
+    while (true)
+    {
+        shared_ptr<Aya::DataModel> dataModel = weakDataModel.lock();
+        if (!dataModel)
+        {
+            break;
+        }
+
+        unsigned int mask = static_cast<unsigned int>(isDebuggerPresent()) * HATE_DEBUGGER;
+        dataModel->addHackFlag(mask);
+
+        boost::this_thread::sleep(boost::posix_time::milliseconds(kSleepBetweenChecksMillis));
+    }
+}
+} // namespace
+
+void Aya::spawnDebugCheckThreads(weak_ptr<Aya::DataModel> dataModel)
+{
+    boost::thread t(boost::bind(&isDebuggedDirectThreadFunc, dataModel));
+}

engine/network/src/API.hpp

@@ -0,0 +1,36 @@
+
+
+#pragma once
+
+#include <string>
+#include <boost/scoped_ptr.hpp>
+#include <boost/weak_ptr.hpp>
+#include <vector>
+
+namespace Aya
+{
+class Instance;
+class DataModel;
+namespace Network
+{
+typedef enum
+{
+    Accept,
+    Reject
+} FilterResult;
+
+extern std::string password;
+extern std::string securityKey;
+bool isPlayerAuthenticationEnabled();
+void init();
+bool isNetworkClient(const Instance* context);
+bool getSystemUrlLocal(DataModel* dataModel);
+
+// Used for debugging and development:
+void setPassword(const char* password);
+
+bool isTrustedContent(const char* url);
+} // namespace Network
+
+void spawnDebugCheckThreads(boost::weak_ptr<Aya::DataModel> weakDataModel);
+} // namespace Aya

engine/network/src/BoostAppend.cpp

@@ -0,0 +1,15 @@
+
+
+#include "BoostAppend.hpp"
+
+#include "intrusive_ptr_target.hpp"
+
+#include "DataModel/PartInstance.hpp"
+
+#include <boost/multi_index/hashed_index.hpp>
+
+std::size_t boost::hash_value(const shared_ptr<Aya::PartInstance>& b)
+{
+    boost::hash<void*> hasher;
+    return hasher(b.get());
+}

engine/network/src/BoostAppend.hpp

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "boost.hpp"
+
+
+namespace Aya
+{
+class PartInstance;
+}
+
+
+namespace boost
+{
+std::size_t hash_value(const shared_ptr<Aya::PartInstance>& b);
+}

engine/network/src/ChatFilter.cpp

@@ -0,0 +1,26 @@
+#include "ChatFilter.hpp"
+
+#include "Utility/Http.hpp"
+
+
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/shared_ptr.hpp>
+
+#include <string>
+
+DYNAMIC_LOGGROUP(WebChatFiltering)
+
+using namespace Aya;
+using namespace Aya::Network;
+
+const char* const Aya::Network::sChatFilter = "ChatFilter";
+
+bool ChatFilter::filterMessageBase(shared_ptr<Player> sourcePlayer, shared_ptr<Instance> receiver, const std::string& message, const ChatFilter::FilteredChatMessageCallback callback)
+{
+    // no filter
+    Result result;
+    result.whitelistFilteredMessage = message;
+    result.blacklistFilteredMessage = message;
+    boost::thread t(boost::bind(callback, result));
+    return true;
+}

engine/network/src/ChatFilter.hpp

@@ -0,0 +1,38 @@
+#pragma once
+
+#include "Tree/Service.hpp"
+
+#include "Tree/Instance.hpp"
+
+#include "Player.hpp"
+
+namespace Aya
+{
+namespace Network
+{
+
+class CustomChatFilter;
+
+extern const char* const sChatFilter;
+class ChatFilter
+    : public DescribedNonCreatable<ChatFilter, Instance, sChatFilter, Reflection::ClassDescriptor::INTERNAL_LOCAL>
+    , public Service
+{
+public:
+    struct Result
+    {
+        std::string whitelistFilteredMessage;
+        std::string blacklistFilteredMessage;
+    };
+    typedef boost::function<void(const Result&)> FilteredChatMessageCallback;
+
+    ChatFilter() {}
+
+    bool filterMessageBase(
+        shared_ptr<Player> sourcePlayer, shared_ptr<Instance> receiver, const std::string& message, const FilteredChatMessageCallback callback);
+
+    virtual void filterMessage(
+        shared_ptr<Player> sourcePlayer, shared_ptr<Instance> receiver, const std::string& message, const FilteredChatMessageCallback callback) = 0;
+};
+} // namespace Network
+} // namespace Aya

engine/network/src/Client.cpp

@@ -0,0 +1,419 @@
+
+
+#include "Client.hpp"
+#include "ClientReplicator.hpp"
+
+#include "RakNet/PacketLogger.hpp"
+#include "Util.hpp"
+#include "ConcurrentRakPeer.hpp"
+#include "Players.hpp"
+
+#include "NetworkOwner.hpp"
+
+#include "Utility/StandardOut.hpp"
+
+#include "Utility/ProtectedString.hpp"
+
+#include "Utility/RbxStringTable.hpp"
+
+#include "CPUCount.hpp"
+#include "FastLog.hpp"
+#include "AyaDbgInfo.hpp"
+
+#include "DataModel/HackDefines.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+#include "DataModel/DataModel.hpp"
+
+#include "DataModel/DebugSettings.hpp"
+
+#include "DataModel/TeleportService.hpp"
+
+
+#include "Script/ScriptContext.hpp"
+
+#include "RakNet/RakNetStatistics.hpp"
+
+#include "Utility/AyaService.hpp"
+
+LOGGROUP(US14116)
+DYNAMIC_FASTFLAG(DebugDisableTimeoutDisconnect)
+FASTFLAG(DebugLocalRccServerConnection)
+
+DYNAMIC_LOGGROUP(NetworkJoin)
+FASTFLAG(DebugProtocolSynchronization)
+
+#ifndef _WIN32
+// For inet_addr() call used below
+#include <arpa/inet.h>
+#endif
+
+const char* const Aya::Network::sClient = "NetworkClient";
+
+namespace Aya
+{
+extern const char* const sHopper;
+class Instance;
+} // namespace Aya
+
+using namespace Aya;
+using namespace Aya::Network;
+using namespace RakNet;
+
+
+Reflection::BoundProp<std::string> Client::prop_Ticket("Ticket", "Authentication", &Client::ticket);
+static Reflection::BoundFuncDesc<Client, shared_ptr<Instance>(int, std::string, int, int, int)> f_connect(
+    &Client::playerConnect, "PlayerConnect", "userId", "server", "serverPort", "clientPort", 0, "threadSleepTime", 30, Security::Plugin);
+static Reflection::BoundFuncDesc<Client, void(int)> f_disconnect(&Client::disconnect, "Disconnect", "blockDuration", 3000, Security::LocalUser);
+static Reflection::BoundFuncDesc<Client, void(std::string)> func_setGameSessionID(
+    &Client::setGameSessionID, "SetGameSessionID", "gameSessionID", Security::Roblox);
+static Reflection::EventDesc<Client, void(std::string, shared_ptr<Aya::Instance>)> event_ConnectionAccepted(
+    &Client::connectionAcceptedSignal, "ConnectionAccepted", "peer", "replicator");
+static Reflection::EventDesc<Client, void(std::string)> event_ConnectionRejected(&Client::connectionRejectedSignal, "ConnectionRejected", "peer");
+static Reflection::EventDesc<Client, void(std::string, int, std::string)> event_ConnectionFailed(
+    &Client::connectionFailedSignal, "ConnectionFailed", "peer", "code", "reason");
+REFLECTION_END();
+
+Client::Client()
+    : userId(-1)
+    , networkSettings(&NetworkSettings::singleton())
+{
+    Aya::Security::Context::current().requirePermission(Aya::Security::Plugin, "create a NetworkClient");
+    setName(sClient);
+
+    FASTLOG(FLog::Network, "NetworkClient:Create");
+}
+
+Client::~Client(void)
+{
+    FASTLOG(FLog::Network, "NetworkClient:Remove");
+}
+
+Client* Client::findClient(const Aya::Instance* context, bool testInDatamodel)
+{
+    const ServiceProvider* serviceProvider = ServiceProvider::findServiceProvider(context);
+    AYAASSERT(!testInDatamodel || serviceProvider != NULL);
+    return ServiceProvider::find<Client>(serviceProvider);
+}
+
+bool Client::clientIsPresent(const Aya::Instance* context, bool testInDatamodel)
+{
+    return findClient(context, testInDatamodel) != NULL;
+}
+
+bool Client::physicsOutBandwidthExceeded(const Aya::Instance* context)
+{
+    if (Client* client = Client::findClient(context))
+    {
+        if (ClientReplicator* clientRep = client->findFirstChildOfType<ClientReplicator>())
+        {
+            return clientRep->isLimitedByOutgoingBandwidthLimit();
+        }
+    }
+    return true;
+}
+
+double Client::getNetworkBufferHealth(const Aya::Instance* context)
+{
+    if (Client* client = Client::findClient(context))
+    {
+        return client->rakPeer->GetBufferHealth();
+    }
+    return 0.0f;
+}
+
+const Aya::SystemAddress Client::findLocalSimulatorAddress(const Aya::Instance* context)
+{
+    if (Client* client = Client::findClient(context, false))
+    {
+        if (ClientReplicator* clientRep = client->findFirstChildOfType<ClientReplicator>())
+        {
+            return RakNetToRbxAddress(clientRep->getClientAddress());
+        }
+    }
+    return Network::NetworkOwner::Unassigned();
+}
+
+shared_ptr<Instance> Client::playerConnect(int userId, std::string server, int serverPort, int clientPort, int threadSleepTime)
+{
+    FASTLOG3(FLog::Network, "Client:Connect serverPort(%d) clientPort(%d) threadSleepTime(%d)", serverPort, clientPort, threadSleepTime);
+
+    this->userId = userId;
+    Players* players = ServiceProvider::create<Players>(this);
+    if (!players)
+        throw Aya::runtime_error("Cannot get players");
+
+    shared_ptr<Instance> player = players->createLocalPlayer(userId, TeleportService::getPreviousPlaceId() > 0);
+
+    if (clientPort == 0)
+    {
+        clientPort = networkSettings->preferredClientPort;
+    }
+
+    RakNet::SocketDescriptor d(clientPort, "");
+    StartupResult startRes = rakPeer->rawPeer()->Startup(1, &d, 1);
+    if (startRes != RakNet::RAKNET_STARTED)
+    {
+        if (clientPort == 0)
+        {
+            throw Aya::runtime_error("Failed to start the network client");
+        }
+        else
+        {
+            throw Aya::runtime_error("Failed to start the network client on port %d", clientPort);
+        }
+    }
+
+    // allow local and LAN games only.
+    if (server != "localhost")
+    {
+        bool lansubnet = false;
+        for (int i = 0; !lansubnet && i < MAXIMUM_NUMBER_OF_INTERNAL_IDS; ++i)
+        {
+            RakNet::SystemAddress localAddress = rakPeer->rawPeer()->GetInternalID(RakNet::UNASSIGNED_SYSTEM_ADDRESS, i);
+            RakNet::SystemAddress remoteAddress(server.c_str(), serverPort);
+
+            // match the a and b records
+            lansubnet = (localAddress.GetBinaryAddress() & 0x00FF) == (remoteAddress.GetBinaryAddress() & 0x00FF);
+        }
+        if (!FFlag::DebugLocalRccServerConnection)
+        {
+            if (!lansubnet)
+            {
+                Aya::Security::Context::current().requirePermission(Aya::Security::Roblox, " connect to an extranet game");
+            }
+        }
+    }
+
+    RakNet::ConnectionAttemptResult connectRes =
+        rakPeer->rawPeer()->Connect(server.c_str(), serverPort, Network::password.c_str(), Network::password.size());
+    if (connectRes != RakNet::CONNECTION_ATTEMPT_STARTED)
+    {
+        throw Aya::runtime_error("Failed to connect to server, id %d", connectRes);
+    }
+    FASTLOG1F(DFLog::NetworkJoin, "playerConnect connecting to server @ %f s", Time::nowFastSec());
+
+    if (DFFlag::DebugDisableTimeoutDisconnect || AyaService::localServer)
+        rakPeer->rawPeer()->SetTimeoutTime(10 * 60 * 1000, UNASSIGNED_SYSTEM_ADDRESS);
+
+
+    // StandardOut::singleton()->printf(MESSAGE_SENSITIVE, "Connecting to %s:%d", server.c_str(), serverPort);
+
+    FASTLOG2(FLog::Network, "Connecting to server, IP(inet_addr): %u Port: %u", inet_addr(server.c_str()), serverPort);
+
+    Aya::AyaDbgInfo::SetServerIP(server.c_str());
+
+    return player;
+}
+
+
+
+void Client::disconnect(int blockDuration)
+{
+    FASTLOG(FLog::Network, "Client:Disconnect");
+
+    // The following line will remove the Replicator
+    this->visitChildren(boost::bind(&Instance::unlockParent, _1));
+    this->removeAllChildren();
+
+    if (rakPeer)
+    {
+        rakPeer->rawPeer()->CloseConnection(this->serverId, true);
+        rakPeer->rawPeer()->Shutdown(blockDuration);
+    }
+}
+
+void Client::setGameSessionID(std::string value)
+{
+    if (value != Http::gameSessionID)
+    {
+        Http::gameSessionID = value;
+    }
+}
+
+void Client::onServiceProvider(ServiceProvider* oldProvider, ServiceProvider* newProvider)
+{
+    if (oldProvider)
+    {
+        closingConnection.disconnect();
+
+        disconnect(); // We should have disconnected by now (in response to the Closing event)
+
+        Players* players = ServiceProvider::find<Players>(oldProvider);
+        players->setConnection(NULL);
+    }
+
+    Super::onServiceProvider(oldProvider, newProvider);
+
+    if (newProvider)
+    {
+        // We're in multiplayer mode, so burn out the studio tools
+        if (Aya::DataModel* dataModel = Aya::DataModel::get(this))
+        {
+            if (dataModel->lockVerb.get())
+                dataModel->lockVerb->doIt(NULL);
+        }
+
+        Players* players = ServiceProvider::create<Players>(newProvider);
+        players->setConnection(rakPeer.get());
+
+        // Disconnect now before we start getting DescendantRemoving events
+        // If we don't disconnect first, then we'll send a shower of delete messages
+        // to the Server
+        closingConnection = newProvider->closingSignal.connect(boost::bind(&Client::disconnect, this));
+    }
+}
+
+void Client::sendVersionInfo()
+{
+    RakNet::BitStream bitStream;
+    bitStream << (unsigned char)ID_PROTOCOL_SYNC;
+    bitStream << protocolVersion;
+
+    rakPeer->rawPeer()->Send(&bitStream, networkSettings->getDataSendPriority(), DATAMODEL_RELIABILITY, DATA_CHANNEL, serverId, false);
+}
+
+void Client::sendTicket()
+{
+    RakNet::BitStream bitStream;
+    bitStream << (unsigned char)ID_SUBMIT_TICKET;
+
+    bitStream << userId;
+    serializeStringCompressed(ticket, bitStream);
+
+    serializeStringCompressed(Aya::DataModel::hash, bitStream);
+
+    bitStream << protocolVersion;
+
+    serializeStringCompressed(password, bitStream);
+
+    // TODO: better way to track protocol changes between versions
+    // Network Protocol version 2
+    serializeStringCompressed(DebugSettings::singleton().osPlatform(), bitStream);
+    serializeStringCompressed(DebugSettings::singleton().getRobloxProductName(), bitStream);
+
+    serializeStringCompressed(Http::gameSessionID, bitStream);
+
+    encryptDataPart(bitStream);
+
+    // Send ID_SUBMIT_TICKET
+    rakPeer->rawPeer()->Send(&bitStream, networkSettings->getDataSendPriority(), DATAMODEL_RELIABILITY, DATA_CHANNEL, serverId, false);
+}
+
+std::string rakIdToString(int id)
+{
+    switch (id)
+    {
+    case ID_INVALID_PASSWORD:
+    case ID_HASH_MISMATCH:
+        return "Aya version is out of date. Please uninstall and try again.";
+    case ID_CONNECTION_ATTEMPT_FAILED:
+        return "Connection attempt failed.";
+    case ID_SECURITYKEY_MISMATCH:
+        return "Version not compatible with server. Please uninstall and try again.";
+    default:
+        return Aya::format("Network error %d", id);
+    }
+}
+
+void Client::OnFailedConnectionAttempt(RakNet::Packet* packet, RakNet::PI2_FailedConnectionAttemptReason failedConnectionAttemptReason)
+{
+    std::string message = rakIdToString(packet->data[0]);
+    StandardOut::singleton()->printf(
+        MESSAGE_SENSITIVE, "Failed to connect to %s. %s\n", RakNetAddressToString(packet->systemAddress).c_str(), message.c_str());
+    connectionFailedSignal(RakNetAddressToString(packet->systemAddress), (int)packet->data[0], message);
+}
+
+void Client::sendPreferedSpawnName() const
+{
+    RakNet::BitStream bitStream;
+
+    bitStream << (unsigned char)ID_SPAWN_NAME;
+
+    serializeStringCompressed(TeleportService::GetSpawnName(), bitStream);
+
+    FASTLOGS(FLog::Network, "serverId: %s", RakNetAddressToString(serverId).c_str());
+
+    rakPeer->rawPeer()->Send(&bitStream, networkSettings->getDataSendPriority(), DATAMODEL_RELIABILITY, DATA_CHANNEL, serverId, false);
+}
+
+void Client::HandleConnection(RakNet::Packet* packet)
+{
+    shared_ptr<Replicator> proxy;
+    try
+    {
+        // send previous placeId
+        RakNet::BitStream bitStream;
+        bitStream << (unsigned char)ID_PLACEID_VERIFICATION;
+
+        bitStream << TeleportService::getPreviousPlaceId();
+
+        rakPeer->rawPeer()->Send(&bitStream, networkSettings->getDataSendPriority(), DATAMODEL_RELIABILITY, DATA_CHANNEL, serverId, false);
+
+        sendTicket();
+
+        sendPreferedSpawnName();
+
+        Workspace* workspace = Workspace::findWorkspace(this);
+        workspace->clearTerrain();
+
+        proxy =
+            Creatable<Instance>::create<ClientReplicator>(packet->systemAddress, this, rakPeer->rawPeer()->GetExternalID(serverId), networkSettings);
+
+        proxy->setAndLockParent(this);
+
+        connectionAcceptedSignal(RakNetAddressToString(packet->systemAddress), proxy);
+    }
+    catch (Aya::base_exception& e)
+    {
+        Aya::StandardOut::singleton()->printf(Aya::MESSAGE_ERROR, "Error in ID_CONNECTION_REQUEST_ACCEPTED: %s", e.what());
+        if (proxy)
+        {
+            // Disconnect
+            proxy->unlockParent();
+            proxy->setParent(NULL);
+        }
+    }
+}
+
+RakNet::PluginReceiveResult Client::OnReceive(RakNet::Packet* packet)
+{
+    RakNet::PluginReceiveResult result = Super::OnReceive(packet);
+    if (result != RR_CONTINUE_PROCESSING)
+        return result;
+
+    switch ((unsigned char)packet->data[0])
+    {
+    case ID_CONNECTION_REQUEST_ACCEPTED:
+    {
+        StandardOut::singleton()->printf(MESSAGE_SENSITIVE, "Connection accepted from %s\n", RakNetAddressToString(packet->systemAddress).c_str());
+
+        serverId = packet->systemAddress;
+
+        HandleConnection(packet);
+        sendVersionInfo();
+    }
+        return RR_CONTINUE_PROCESSING;
+
+    case ID_INVALID_PASSWORD:
+        StandardOut::singleton()->printf(MESSAGE_SENSITIVE, "Invalid password from %s", RakNetAddressToString(packet->systemAddress).c_str());
+        connectionFailedSignal(RakNetAddressToString(packet->systemAddress), (int)packet->data[0], rakIdToString(packet->data[0]));
+        connectionRejectedSignal(RakNetAddressToString(packet->systemAddress));
+        return RR_STOP_PROCESSING_AND_DEALLOCATE;
+
+    case ID_HASH_MISMATCH:
+    case ID_SECURITYKEY_MISMATCH:
+        connectionFailedSignal(RakNetAddressToString(packet->systemAddress), (int)packet->data[0], rakIdToString(packet->data[0]));
+        return RR_STOP_PROCESSING_AND_DEALLOCATE;
+
+    case ID_DISCONNECTION_NOTIFICATION:
+    case ID_CONNECTION_LOST:
+        AYAASSERT(packet->systemAddress == serverId);
+        serverId = UNASSIGNED_SYSTEM_ADDRESS;
+        return RR_CONTINUE_PROCESSING;
+
+    default:
+        return RR_CONTINUE_PROCESSING;
+    }
+}

engine/network/src/Client.hpp

@@ -0,0 +1,72 @@
+
+
+#pragma once
+
+#include "Peer.hpp"
+#include "Utility/SystemAddress.hpp"
+
+
+
+namespace Aya
+{
+
+namespace Network
+{
+
+extern const char* const sClient;
+class Client
+    : public DescribedCreatable<Client, Peer, sClient, Reflection::ClassDescriptor::INTERNAL_LOCAL>
+    , public Service
+{
+private:
+    typedef DescribedCreatable<Client, Peer, sClient, Reflection::ClassDescriptor::INTERNAL_LOCAL> Super;
+
+    Aya::signals::scoped_connection closingConnection;
+    RakNet::SystemAddress serverId;
+    int userId;
+    std::string ticket;
+    NetworkSettings* networkSettings;
+
+    static Reflection::BoundProp<std::string> prop_Ticket;
+
+public:
+    Client();
+    ~Client();
+
+    Aya::signal<void(std::string, shared_ptr<Instance>)> connectionAcceptedSignal;
+    Aya::signal<void(std::string, int, std::string)> connectionFailedSignal;
+    Aya::signal<void(std::string)> connectionRejectedSignal;
+
+    void setTicket(const std::string& t)
+    {
+        ticket = t;
+    }
+
+    shared_ptr<Instance> playerConnect(int userId, std::string server, int serverPort, int clientPort, int threadSleepTime);
+    void disconnect(int blockDuration);
+    void disconnect()
+    {
+        disconnect(3000);
+    }
+    void setGameSessionID(std::string gameSessionID);
+
+    /*override*/ RakNet::PluginReceiveResult OnReceive(RakNet::Packet* packet);
+    /*override*/ void OnFailedConnectionAttempt(RakNet::Packet* packet, RakNet::PI2_FailedConnectionAttemptReason failedConnectionAttemptReason);
+
+    static Client* findClient(const Aya::Instance* context, bool testInDatamodel = true);
+    static bool clientIsPresent(const Aya::Instance* context, bool testInDatamodel = true);
+    static const Aya::SystemAddress findLocalSimulatorAddress(const Aya::Instance* context); // if Client == clientAddress, else == NULL;
+    static bool physicsOutBandwidthExceeded(const Aya::Instance* context);
+    static double getNetworkBufferHealth(const Aya::Instance* context);
+
+protected:
+    virtual void onServiceProvider(ServiceProvider* oldProvider, ServiceProvider* newProvider);
+
+private:
+    void sendVersionInfo();
+    void sendTicket();
+    void sendPreferedSpawnName() const;
+    void HandleConnection(RakNet::Packet* packet);
+};
+} // namespace Network
+} // namespace Aya

engine/network/src/ClientReplicator.hpp

@@ -0,0 +1,317 @@
+
+
+#pragma once
+
+#include "Replicator.hpp"
+#include "PropertySynchronization.hpp"
+#include "threadsafe.hpp"
+
+#include "Utility/MemoryStats.hpp"
+
+#include "Replicator.StreamJob.hpp"
+
+namespace Aya
+{
+
+namespace Network
+{
+
+const float kMaxIncomePacketWaitTime = 0.25f;
+
+class Client;
+class StrictNetworkFilter;
+class DeserializedStatsItem;
+class DeserializedTagItem;
+class DeserializedStreamDataItem;
+
+extern const char* const sClientReplicator;
+class ClientReplicator
+    : public Aya::DescribedNonCreatable<ClientReplicator, Replicator, sClientReplicator, Reflection::ClassDescriptor::INTERNAL_LOCAL>
+{
+private:
+    typedef Aya::DescribedNonCreatable<ClientReplicator, Replicator, sClientReplicator, Reflection::ClassDescriptor::INTERNAL_LOCAL> Super;
+
+    class RequestCharacterItem;
+    class ClientCapacityUpdateItem;
+    class ClientStatsItem;
+#ifdef ENABLE_VOICE_CHAT
+    class SendOpusDataJob;
+#endif
+    class GCJob;
+    friend class ClientStatsItem;
+    friend class StatsItem;
+    friend class DeserializedStatsItem;
+    friend class DeserializedTagItem;
+    friend class DeserializedStreamDataItem;
+
+    PropSync::Slave propSync;
+    RakNet::SystemAddress clientAddress;
+    bool receivedGlobals;
+    boost::scoped_ptr<Aya::AutoMemPool> cframePool;
+    boost::shared_ptr<GCJob> gcJob;
+#ifdef ENABLE_VOICE_CHAT
+    boost::shared_ptr<SendOpusDataJob> opusJob;
+#endif
+
+    RunningAverage<double> avgInstancesPerStreamData;
+    RunningAverage<double> avgStreamDataReadTime;
+    RunningAverage<int> avgRequestCount;
+    Aya::MemoryStats::MemoryLevel memoryLevel;
+
+    Aya::signals::scoped_connection playerCharacterAddedConnection;
+    void onPlayerCharacterAdded();
+
+    shared_ptr<Reflection::ValueTable> readStats(RakNet::BitStream& bitStream);
+
+    // streaming
+    double sampleTimer;
+    int clientInstanceQuota;
+    StreamRegion::Id lastReadStreamId;
+
+    int pendingInstanceRequests;
+    int numInstancesRead;
+    bool loggedLowMemWarning;
+    void readStreamData(RakNet::BitStream& bitStream);
+    void readStreamDataItem(DeserializedStreamDataItem* item);
+    void processStreamDataRegionId(Replicator::StreamJob::RegionIteratorSuccessor successorBitMask, StreamRegion::Id id);
+    bool canUpdateClientCapacity();
+    void streamOutTerrain(const Vector3int16& cellPos);
+    void streamOutInstance(Instance* part,
+        bool deleteImmediately = true); // NOTE: passing false to deleteImmediately requires handling auto joints and deleting the instance externally
+    void streamOutPartHelper(const Guid::Data& data, PartInstance* part, shared_ptr<Instance> descendant);
+    void streamOutAutoJointHelper(std::vector<shared_ptr<PartInstance>> pendingRemovalParts, shared_ptr<Instance> instance);
+
+    void markerReceived();
+
+protected:
+    /*override*/ bool isProtectedStringEnabled();
+    /*override*/ std::string encodeProtectedString(
+        const ProtectedString& value, const Instance* instance, const Reflection::PropertyDescriptor& desc);
+    /*override*/ boost::optional<ProtectedString> decodeProtectedString(
+        const std::string& value, const Instance* instance, const Reflection::PropertyDescriptor& desc);
+
+    /*override*/ bool checkDistributedReceive(PartInstance* part);
+    /*override*/ bool checkDistributedSend(const PartInstance* part);
+    /*override*/ bool checkDistributedSendFast(const PartInstance* part);
+    /*override*/ void processPacket(RakNet::Packet* packet);
+    /*override*/ bool canSendItems();
+
+    void readTag(RakNet::BitStream& inBitstream);
+    void readTagItem(DeserializedTagItem* item);
+    void processTag(int tag);
+
+    /*override*/ void readItem(RakNet::BitStream& inBitstream, Aya::Network::Item::ItemType itemType);
+    /*override*/ shared_ptr<DeserializedItem> deserializeItem(RakNet::BitStream& inBitstream, Aya::Network::Item::ItemType itemType);
+    /*override*/ bool isLegalSendInstance(const Instance* instance);
+    /*override*/ bool isLegalSendProperty(Instance* instance, const Reflection::PropertyDescriptor& desc);
+    /*override*/ bool isLegalSendEvent(Instance* instance, const Reflection::EventDescriptor& desc);
+
+    /*override*/ void readChangedProperty(RakNet::BitStream& bitStream, Reflection::Property prop);
+    /*override*/ void readChangedPropertyItem(DeserializedChangePropertyItem* item, Reflection::Property prop);
+    bool processChangedParentPropertyForStreaming(const Guid::Data& parentId, Reflection::Property prop);
+
+    /*override*/ void writeChangedProperty(const Instance* instance, const Reflection::PropertyDescriptor& desc, RakNet::BitStream& outBitStream);
+    /*override*/ void writeChangedRefProperty(
+        const Instance* instance, const Reflection::RefPropertyDescriptor& desc, const Guid::Data& newRefGuid, RakNet::BitStream& outBitStream);
+    /*override*/ void writeProperties(const Instance* instance, RakNet::BitStream& outBitstream, PropertyCacheType cacheType, bool useDictionary);
+
+    /*override*/ Player* findTargetPlayer() const;
+    /*override*/ Player* getRemotePlayer() const
+    {
+        return NULL;
+    }
+    /*override*/ void dataOutStep();
+
+    /*override*/ void setPropSyncExpiration(double value)
+    {
+        propSync.setExpiration(Aya::Time::Interval(value));
+    }
+
+    /*override*/ shared_ptr<Stats> createStatsItem();
+
+    /*override*/ void onServiceProvider(ServiceProvider* oldProvider, ServiceProvider* newProvider);
+
+    /*override*/ FilterResult filterChangedProperty(Instance* instance, const Reflection::PropertyDescriptor& desc);
+    /*override*/ FilterResult filterReceivedParent(Instance* instance, Instance* parent);
+
+    /*override*/ bool wantReplicate(const Instance* source) const;
+    /*override*/ void terrainCellChanged(const Voxel::CellChangeInfo& info);
+    /*override*/ void onTerrainRegionChanged(const Voxel2::Region& region);
+
+    // protocol schema
+    /*override*/ bool ProcessOutdatedChangedProperty(RakNet::BitStream& inBitstream, const Aya::Guid::Data& id, const Instance* instance,
+        const Reflection::PropertyDescriptor* propertyDescriptor, unsigned int propId);
+    /*override*/ bool ProcessOutdatedProperties(RakNet::BitStream& inBitstream, Instance* instance, PropertyCacheType cacheType, bool useDictionary,
+        bool preventBounceBack, std::vector<PropValuePair>* valueArray);
+    /*override*/ bool ProcessOutdatedInstance(RakNet::BitStream& inBitstream, bool isJoinData, const Aya::Guid::Data& id,
+        const Reflection::ClassDescriptor* classDescriptor, unsigned int classId);
+    /*override*/ bool ProcessOutdatedEventInvocation(RakNet::BitStream& inBitstream, const Aya::Guid::Data& id, const Instance* instance,
+        const Reflection::EventDescriptor* eventDescriptor, unsigned int eventId);
+    /*override*/ bool ProcessOutdatedEnumSerialization(
+        const Reflection::Type& type, const Reflection::Variant& value, RakNet::BitStream& outBitStream);
+    /*override*/ bool ProcessOutdatedEnumDeserialization(RakNet::BitStream& inBitStream, const Reflection::Type& type, Reflection::Variant& value);
+    /*override*/ bool ProcessOutdatedPropertyEnumSerialization(const Reflection::ConstProperty& property, RakNet::BitStream& outBitStream);
+    /*override*/ bool ProcessOutdatedPropertyEnumDeserialization(Reflection::Property& property, RakNet::BitStream& inBitStream);
+    /*override*/ bool isClassRemoved(const Instance* instance);
+    /*override*/ bool isPropertyRemoved(const Instance* instance, const Name& propertyName);
+    /*override*/ bool isEventRemoved(const Instance* instance, const Name& eventName);
+
+    bool needGC();
+    bool hasEnoughMemoryToReceiveInstances();
+
+public:
+    Aya::signal<void()> receivedGlobalsSignal;
+    Aya::signal<void()> gameLoadedSignal;
+    Aya::signal<void(shared_ptr<const Reflection::ValueTable>)> statsReceivedSignal;
+
+    ClientReplicator(RakNet::SystemAddress systemAddress, Client* client, RakNet::SystemAddress clientAddress, NetworkSettings* networkSettings);
+    ~ClientReplicator();
+
+    void requestServerStats(bool request);
+    void requestCharacterImpl();
+    /*override*/ void requestCharacter();
+    /*override*/ void updateClientCapacity();
+    /*override*/ RakNet::PluginReceiveResult OnReceive(RakNet::Packet* packet);
+    /*override*/ bool canUseProtocolVersion(int protocolVersion) const;
+    /*override*/ void receiveCluster(RakNet::BitStream& inBitstream, Instance* instance, bool usingOneQuarterIterator);
+    /*override*/ void postProcessPacket();
+    /*override*/ shared_ptr<Instance> sendMarker();
+
+    const RakNet::SystemAddress getClientAddress() const
+    {
+        return clientAddress;
+    }
+
+    void writePropAcknowledgementIfNeeded(const Instance* instance, const Reflection::PropertyDescriptor& desc, RakNet::BitStream& outBitStream);
+
+    bool isLimitedByOutgoingBandwidthLimit() const;
+    virtual void deserializeSFFlags(RakNet::BitStream& inBitStream);
+
+    void renderStreamedRegions(Adorn* adorn);
+
+    void renderPartMovementPath(Adorn* adorn);
+
+    Aya::MemoryStats::MemoryLevel getMemoryLevel()
+    {
+        return memoryLevel;
+    }
+    void updateMemoryStats();
+
+    // streaming debug
+    int getNumRegionsToGC() const;
+    short getGCDistance() const;
+    int getNumStreamedRegions() const;
+
+private:
+    // ------------- protocol schema --------------
+    class ReflectionPropertyContainer
+    {
+    public:
+        bool needSync;
+        unsigned int id;
+        const Name& name;
+        unsigned int typeId;
+        std::string typeName;
+        const Reflection::Type* type;
+        bool canReplicate;
+        size_t enumMSB;
+        ReflectionPropertyContainer(unsigned int _id, const Name& _name, unsigned int _typeId, std::string _typeName, const Reflection::Type* _type,
+            bool _canReplicate, size_t _enumMSB)
+            : id(_id)
+            , name(_name)
+            , typeId(_typeId)
+            , typeName(_typeName)
+            , type(_type)
+            , canReplicate(_canReplicate)
+            , enumMSB(_enumMSB)
+            , needSync(false)
+        {
+        }
+    };
+    typedef std::vector<shared_ptr<ReflectionPropertyContainer>> ReflectionPropertyList;
+
+    class ReflectionEventContainer
+    {
+    public:
+        bool needSync;
+        unsigned int id;
+        const Name& name;
+        std::vector<const Reflection::Type*> argTypes;
+        ReflectionEventContainer(unsigned int _id, const Name& _name)
+            : id(_id)
+            , name(_name)
+            , needSync(false)
+        {
+        }
+    };
+    typedef std::vector<shared_ptr<ReflectionEventContainer>> ReflectionEventList;
+
+    class ReflectionClassContainer
+    {
+    public:
+        bool needSync;
+        unsigned int id;
+        const Name& name;
+        Reflection::ReplicationLevel replicationLevel;
+        ReflectionPropertyList properties;
+        ReflectionEventList events;
+        ReflectionClassContainer(unsigned int _id, const Name& _name, Reflection::ReplicationLevel _repLevel)
+            : id(_id)
+            , name(_name)
+            , replicationLevel(_repLevel)
+            , needSync(false)
+        {
+        }
+    };
+
+    typedef boost::unordered_map<const Aya::Name*, const shared_ptr<ReflectionClassContainer>> ReflectionClassMap;
+    ReflectionClassMap serverClasses;
+
+    class ReflectionEnumContainer
+    {
+    public:
+        size_t enumMSB;
+        ReflectionEnumContainer(size_t _enumMSB)
+            : enumMSB(_enumMSB)
+        {
+        }
+    };
+
+    typedef boost::unordered_map<const Aya::Name*, ReflectionEnumContainer> ReflectionEnumMap;
+    ReflectionEnumMap serverEnums;
+
+    typedef std::map<Aya::Guid::Data, const shared_ptr<ReflectionClassContainer>> InstanceClassMap;
+    InstanceClassMap serverInstanceClassMap;
+
+    // Maintain a string dictionary for each Property/event type I don't recognize
+    typedef std::map<unsigned int, shared_ptr<SharedStringDictionary>> DummyPropertyStrings;
+    DummyPropertyStrings dummyStrings;
+
+    typedef std::map<unsigned int, shared_ptr<SharedStringProtectedDictionary>> DummyPropertyProtectedStrings;
+    DummyPropertyProtectedStrings dummyProtectedStrings;
+
+    typedef std::map<unsigned int, shared_ptr<SharedStringDictionary>> DummyEventStrings;
+    DummyEventStrings dummyEventStrings;
+
+    typedef std::map<unsigned int, shared_ptr<SharedBinaryStringDictionary>> DummyPropertyBinaryStrings;
+    DummyPropertyBinaryStrings dummyBinaryStrings;
+
+    void learnSchema(RakNet::BitStream& bitStream);
+
+    void skipPropertyValue(RakNet::BitStream& inBitStream, const shared_ptr<ReflectionPropertyContainer>&, bool useDictionary);
+    void skipPropertiesInternal(const shared_ptr<ReflectionPropertyContainer>&, RakNet::BitStream& inBitstream, bool useDictionary);
+    void skipChangedProperty(RakNet::BitStream& bitStream, const shared_ptr<ReflectionPropertyContainer>& prop);
+    void skipEventInvocation(RakNet::BitStream& bitStream, const shared_ptr<ReflectionEventContainer>& event);
+
+    bool getServerBasedProperty(const Name& className, int propertyId, shared_ptr<ReflectionPropertyContainer>& serverBasedProperty);
+    bool getServerBasedEvent(const Name& className, int eventId, shared_ptr<ReflectionEventContainer>& serverBasedEvent);
+
+    SharedStringProtectedDictionary& getSharedPropertyProtectedDictionaryById(unsigned int propertyId);
+    SharedStringDictionary& getSharedPropertyDictionaryById(unsigned int propertyId);
+    SharedStringDictionary& getSharedEventDictionaryById(unsigned int eventId);
+    SharedBinaryStringDictionary& getSharedPropertyBinaryDictionaryById(unsigned int propertyId);
+
+    bool hasEnumChanged(const Reflection::EnumDescriptor& enumDesc, size_t& newMSB);
+};
+} // namespace Network
+} // namespace Aya

engine/network/src/ClusterUpdateBuffer.cpp

@@ -0,0 +1,92 @@
+#include "ClusterUpdateBuffer.hpp"
+
+#include <vector>
+
+#include "Debug.hpp"
+
+#include "Utility/G3DCore.hpp"
+
+#include "DataModel/MegaCluster.hpp"
+
+#include "boost/cstdint.hpp"
+
+namespace Aya
+{
+
+namespace Network
+{
+
+void ClusterUpdateBuffer::computeUintRepresentingLocationInChunk(const ClusterCellUpdate& update, unsigned int* out)
+{
+
+    (*out) = (update.x & kXZ_CHUNK_SIZE_AS_BITMASK) + ((update.z & kXZ_CHUNK_SIZE_AS_BITMASK) << kXZ_CHUNK_SIZE_AS_BITSHIFT) +
+             ((update.y & kY_CHUNK_SIZE_AS_BITMASK) << (2 * kXZ_CHUNK_SIZE_AS_BITSHIFT));
+}
+
+void ClusterUpdateBuffer::computeGlobalLocationFromUintRepresentation(
+    const unsigned int& index, const Vector3int16& baseCellOffset, ClusterCellUpdate* out)
+{
+
+    (*out) = baseCellOffset;
+    out->x += index & kXZ_CHUNK_SIZE_AS_BITMASK;
+    out->z += (index >> kXZ_CHUNK_SIZE_AS_BITSHIFT) & kXZ_CHUNK_SIZE_AS_BITMASK;
+    out->y += (index >> (2 * kXZ_CHUNK_SIZE_AS_BITSHIFT)) & kY_CHUNK_SIZE_AS_BITMASK;
+}
+
+ClusterUpdateBuffer::ClusterUpdateBuffer()
+    : internalSize(0)
+{
+    lastFound = bitSetUpdates.begin();
+}
+
+size_t ClusterUpdateBuffer::size() const
+{
+    return internalSize;
+}
+
+void ClusterUpdateBuffer::push(const ClusterCellUpdate& inputData)
+{
+    unsigned int info;
+    computeUintRepresentingLocationInChunk(inputData, &info);
+
+    internalSize += bitSetUpdates[SpatialRegion::regionContainingVoxel(inputData)].insert(info);
+
+    // insert potentially invalidates iterators; also this preserves the invariant that no updates can be before lastFound in the map
+    lastFound = bitSetUpdates.begin();
+}
+
+bool ClusterUpdateBuffer::chk(const ClusterCellUpdate& test)
+{
+    unsigned int info;
+    computeUintRepresentingLocationInChunk(test, &info);
+
+    BitSetUpdateMap::iterator it = bitSetUpdates.find(SpatialRegion::regionContainingVoxel(test));
+
+    return it != bitSetUpdates.end() && it->second.contains(info);
+}
+
+void ClusterUpdateBuffer::pop(ClusterCellUpdate* out)
+{
+    AYAASSERT(internalSize > 0);
+
+    for (BitSetUpdateMap::iterator it = lastFound; it != bitSetUpdates.end(); ++it)
+    {
+        UintSet& updateVector = it->second;
+
+        if (updateVector.size() > 0)
+        {
+            lastFound = it;
+
+            unsigned int intVal;
+            updateVector.pop_smallest(&intVal);
+            computeGlobalLocationFromUintRepresentation(intVal, SpatialRegion::inclusiveVoxelExtentsOfRegion(it->first).getMinPos(), out);
+            internalSize--;
+            return;
+        }
+    }
+
+    AYAASSERT(false);
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/ClusterUpdateBuffer.hpp

@@ -0,0 +1,62 @@
+#pragma once
+
+#include <vector>
+
+#include "Utility/G3DCore.hpp"
+
+#include "Utility/UintSet.hpp"
+
+#include "boost/cstdint.hpp"
+#include "Utility/ClusterCellIterator.hpp"
+
+#include "boost/unordered_map.hpp"
+
+namespace Aya
+{
+
+// these constants visible for testing
+const int kXZ_CHUNK_SIZE_AS_BITSHIFT = 5;
+const int kXZ_CHUNK_SIZE_AS_BITMASK = 0x1f;
+const int kY_CHUNK_SIZE_AS_BITSHIFT = 4;
+const int kY_CHUNK_SIZE_AS_BITMASK = 0x0f;
+
+namespace Network
+{
+
+typedef Vector3int16 ClusterCellUpdate;
+typedef unsigned int ChunkIndex;
+
+
+struct ClusterUpdateBuffer
+{
+
+private:
+    typedef boost::unordered_map<SpatialRegion::Id, UintSet, SpatialRegion::Id::boost_compatible_hash_value> BitSetUpdateMap;
+    BitSetUpdateMap::iterator lastFound;
+    BitSetUpdateMap bitSetUpdates;
+
+    size_t internalSize;
+
+public:
+    static void computeUintRepresentingLocationInChunk(const ClusterCellUpdate& update, unsigned int* out);
+    static void computeGlobalLocationFromUintRepresentation(const unsigned int& info, const Vector3int16& baseCellOffset, ClusterCellUpdate* out);
+
+    ClusterUpdateBuffer();
+
+    size_t size() const;
+
+    void push(const ClusterCellUpdate& inputData);
+
+    bool chk(const ClusterCellUpdate& test);
+
+    void pop(ClusterCellUpdate* out);
+
+    static inline void nextCellInIterationOrder(const Vector3int16& cellpos, Vector3int16* out)
+    {
+        ClusterChunksIterator::nextCellInIterationOrder(cellpos, out);
+    }
+};
+
+} // namespace Network
+
+} // namespace Aya

engine/network/src/Compressor.cpp

@@ -0,0 +1,214 @@
+
+
+#include "Compressor.hpp"
+#include "Replicator.hpp"
+#include "Utility/Quaternion.hpp"
+
+
+#include <boost/iostreams/filtering_streambuf.hpp>
+#include <boost/iostreams/stream.hpp>
+#include <boost/iostreams/filter/gzip.hpp>
+#include <boost/iostreams/device/array.hpp>
+#include <boost/iostreams/copy.hpp>
+
+using namespace Aya;
+using namespace Aya::Network;
+
+enum
+{
+    translationBits0 = 15,
+    translationBits1 = 14,
+    translationBits2 = 15
+};
+
+static const float translationMin[] = {-1024, -512, -1024};
+static const float translationMax[] = {1024, 512, 1024};
+
+void Compressor::writeRotation(RakNet::BitStream& bitStream, const Matrix3& rotation, CompressionType compressionType)
+{
+    writeCompressionType(bitStream, compressionType);
+
+    Quaternion q(rotation);
+    q.normalize();
+
+    switch (compressionType)
+    {
+    case Compressor::UNCOMPRESSED:
+    {
+        bitStream << q.w;
+        bitStream << q.x;
+        bitStream << q.y;
+        bitStream << q.z;
+        break;
+    }
+    case Compressor::RAKNET_COMPRESSED:
+    {
+        bitStream.WriteNormQuat(q.w, q.x, q.y, q.z);
+        break;
+    }
+    case Compressor::HEAVILY_COMPRESSED:
+    {
+        bitStream.WriteNormQuat(q.w, q.x, q.y, q.z);
+        break;
+    }
+    default:
+    {
+        AYAASSERT(0);
+    }
+    }
+}
+
+
+void Compressor::writeTranslation(RakNet::BitStream& bitStream, const Vector3& translation, CompressionType compressionType)
+{
+    if ((compressionType == Compressor::HEAVILY_COMPRESSED) && !canHeavilyCompressTranslation(translation))
+    {
+        compressionType = Compressor::RAKNET_COMPRESSED;
+    }
+
+    writeCompressionType(bitStream, compressionType);
+
+    switch (compressionType)
+    {
+    case Compressor::UNCOMPRESSED:
+    {
+        bitStream << translation.x;
+        bitStream << translation.y;
+        bitStream << translation.z;
+        break;
+    }
+    case Compressor::RAKNET_COMPRESSED:
+    {
+        bitStream.WriteVector(translation.x, translation.y, translation.z);
+        break;
+    }
+    case Compressor::HEAVILY_COMPRESSED:
+    {
+        // p = (p_i - min)*shift/(max-min)
+        unsigned short x = (unsigned short)((translation.x - translationMin[0]) * (1 << translationBits0) / (translationMax[0] - translationMin[0]));
+        unsigned short y = (unsigned short)((translation.y - translationMin[1]) * (1 << translationBits1) / (translationMax[1] - translationMin[1]));
+        unsigned short z = (unsigned short)((translation.z - translationMin[2]) * (1 << translationBits2) / (translationMax[2] - translationMin[2]));
+
+        // Check for overflow (underflow shouldn't be possible)
+        if (x >= 1 << translationBits0)
+            x = 0xFFFF;
+        if (y >= 1 << translationBits1)
+            y = 0xFFFF;
+        if (z >= 1 << translationBits2)
+            z = 0xFFFF;
+
+        bitStream.WriteBits((const unsigned char*)&x, translationBits0);
+        bitStream.WriteBits((const unsigned char*)&y, translationBits1);
+        bitStream.WriteBits((const unsigned char*)&z, translationBits2);
+        break;
+    }
+    default:
+    {
+        AYAASSERT(0);
+    }
+    }
+}
+
+bool Compressor::canHeavilyCompressTranslation(const Vector3& translation)
+{
+    return !(translation.x > translationMax[0] || translation.x < translationMin[0] || translation.y > translationMax[1] ||
+             translation.y < translationMin[1] || translation.z > translationMax[2] || translation.z < translationMin[2]);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+
+
+void Compressor::readRotation(RakNet::BitStream& bitStream, Matrix3& rotation)
+{
+    CompressionType compressionType = readCompressionType(bitStream);
+
+    Quaternion q;
+
+    switch (compressionType)
+    {
+    case UNCOMPRESSED:
+    {
+        bitStream >> q.w;
+        bitStream >> q.x;
+        bitStream >> q.y;
+        bitStream >> q.z;
+        break;
+    }
+    case RAKNET_COMPRESSED:
+    {
+        if (!bitStream.ReadNormQuat(q.w, q.x, q.y, q.z))
+            throw std::runtime_error("Failed to read Quaternion");
+        break;
+    }
+    case HEAVILY_COMPRESSED:
+    {
+        if (!bitStream.ReadNormQuat(q.w, q.x, q.y, q.z))
+            throw std::runtime_error("Failed to read Quaternion");
+        break;
+    }
+    default:
+    {
+        AYAASSERT(0);
+    }
+    }
+
+    q.toRotationMatrix(rotation);
+}
+
+void Compressor::readTranslation(RakNet::BitStream& bitStream, Vector3& translation)
+{
+
+    CompressionType compressionType = readCompressionType(bitStream);
+
+    switch (compressionType)
+    {
+    case UNCOMPRESSED:
+    {
+        bitStream >> translation.x;
+        bitStream >> translation.y;
+        bitStream >> translation.z;
+        break;
+    }
+    case RAKNET_COMPRESSED:
+    {
+        readVectorFast(bitStream, translation.x, translation.y, translation.z);
+        break;
+    }
+    case HEAVILY_COMPRESSED:
+    {
+        unsigned short x, y, z = 0;
+
+        readFastN<translationBits0>(bitStream, x);
+        readFastN<translationBits1>(bitStream, y);
+        readFastN<translationBits2>(bitStream, z);
+
+        // p_i = p/shift * (max-min) + min
+        translation.x = (float)x * (1.0f / (1 << translationBits0) * (translationMax[0] - translationMin[0])) + translationMin[0];
+        translation.y = (float)y * (1.0f / (1 << translationBits1) * (translationMax[1] - translationMin[1])) + translationMin[1];
+        translation.z = (float)z * (1.0f / (1 << translationBits2) * (translationMax[2] - translationMin[2])) + translationMin[2];
+        break;
+    }
+    default:
+    {
+        AYAASSERT(0);
+    }
+    }
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+//////////////////////////////////////////////////////////////////////////////////////////////////
+
+void Compressor::writeCompressionType(RakNet::BitStream& bitStream, CompressionType compressionType)
+{
+    unsigned short temp = compressionType;
+    bitStream.WriteBits((const unsigned char*)&temp, 2);
+}
+
+Compressor::CompressionType Compressor::readCompressionType(RakNet::BitStream& bitStream)
+{
+    unsigned short temp = 0;
+    readFastN<2>(bitStream, temp);
+
+    CompressionType answer = static_cast<CompressionType>(temp);
+    return answer;
+}

engine/network/src/Compressor.hpp

@@ -0,0 +1,42 @@
+#pragma once
+
+#include "Utility/G3DCore.hpp"
+
+
+namespace RakNet
+{
+class BitStream;
+}
+
+namespace Aya
+{
+
+namespace Network
+{
+
+class Compressor
+{
+public:
+    typedef enum
+    {
+        UNCOMPRESSED = 0,
+        RAKNET_COMPRESSED,
+        HEAVILY_COMPRESSED
+    } CompressionType;
+
+private:
+    static bool canHeavilyCompressTranslation(const Vector3& translation);
+
+    static void writeCompressionType(RakNet::BitStream& bitStream, CompressionType compressionType);
+    static CompressionType readCompressionType(RakNet::BitStream& bitStream);
+
+public:
+    static void writeTranslation(RakNet::BitStream& bitStream, const Vector3& translation, CompressionType compressionType);
+    static void writeRotation(RakNet::BitStream& bitStream, const Matrix3& rotation, CompressionType compressionType);
+
+    static void readTranslation(RakNet::BitStream& bitStream, Vector3& translation);
+    static void readRotation(RakNet::BitStream& bitStream, Matrix3& rotation);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/ConcurrentRakPeer.cpp

@@ -0,0 +1,295 @@
+
+#include "ConcurrentRakPeer.hpp"
+#include "DataModel/DataModel.hpp"
+
+#include "RakNet/RakNetStatistics.hpp"
+#include "NetworkSettings.hpp"
+#include "Utility/SystemAddress.hpp"
+
+
+LOGGROUP(NetworkStatsReport)
+
+using namespace Aya;
+using namespace Aya::Network;
+
+// Used by ConcurrentRakPeer to synchronize the sending of packets in a thread-safe way
+class ConcurrentRakPeer::PacketJob : public Aya::DataModelJob
+{
+public:
+    struct SendData
+    {
+        boost::shared_ptr<const RakNet::BitStream> bitStream;
+        PacketPriority priority;
+        PacketReliability reliability;
+        char orderingChannel;
+        RakNet::SystemAddress systemAddress;
+        bool broadcast;
+    };
+    Aya::timestamped_safe_queue<SendData> sendQueue;
+    bool isQueueErrorComputed;
+
+    weak_ptr<RakNet::RakPeerInterface> peer;
+    PacketJob(shared_ptr<RakNet::RakPeerInterface> peer, DataModel* dataModel)
+        : DataModelJob("Net Peer Send", DataModelJob::RaknetPeer, false, shared_from(dataModel), Time::Interval(0))
+        , peer(peer)
+        , isQueueErrorComputed(NetworkSettings::singleton().isQueueErrorComputed)
+    {
+        cyclicExecutive = true;
+        cyclicPriority = CyclicExecutiveJobPriority_Network_ReceiveIncoming;
+    }
+
+private:
+    virtual Time::Interval sleepTime(const Stats& stats)
+    {
+        return sendQueue.empty() ? Time::Interval::max() : Time::Interval::zero();
+    }
+
+    virtual Error error(const Stats& stats)
+    {
+        Error result;
+        if (TaskScheduler::singleton().isCyclicExecutive() && cyclicExecutive)
+        {
+            if (sendQueue.size())
+            {
+                result.error = 1.0f;
+            }
+            else
+            {
+                result.error = 0.0f;
+            }
+        }
+        else if (isQueueErrorComputed)
+        {
+            double waitTime = sendQueue.head_waittime_sec(stats.timeNow);
+            result.error = waitTime * 100.0;
+            result.error *= std::min<int>(10, sendQueue.size() + 1) / 2.0;
+        }
+        else
+        {
+            double size = sendQueue.size();
+
+            if (size > 0)
+                result.error = stats.timespanSinceLastStep.seconds() * 100.0 * (double)size;
+            else
+                result.error = 0;
+        }
+
+        return result;
+    }
+
+    TaskScheduler::StepResult stepDataModelJob(const Stats& stats)
+    {
+        if (shared_ptr<RakNet::RakPeerInterface> safePeer = peer.lock())
+        {
+            SendData data;
+            while (sendQueue.pop_if_present(data))
+            {
+                bool result = safePeer->Send(data.bitStream.get(), data.priority, data.reliability, data.orderingChannel, data.systemAddress,
+                                  data.broadcast) != 0;
+                AYAASSERT(result);
+            }
+            return TaskScheduler::Stepped;
+        }
+        return TaskScheduler::Done;
+    }
+};
+
+
+class ConcurrentRakPeer::StatsUpdateJob : public Aya::DataModelJob
+{
+private:
+    struct SystemAddressHasher
+    {
+        size_t operator()(const RakNet::SystemAddress& key) const
+        {
+            return RakNet::SystemAddress::ToInteger(key);
+        }
+    };
+
+public:
+    boost::mutex mapMutex;
+    typedef boost::unordered_map<RakNet::SystemAddress, ConnectionStats, SystemAddressHasher> StatsMap;
+    StatsMap statsMap;
+
+    typedef boost::unordered_map<RakNet::SystemAddress, boost::function<void(const ConnectionStats&)>> UpdateCallbackMap;
+    UpdateCallbackMap updateCallbackMap;
+
+    // 0 to 1
+    // 1: good, nothing in buffer, room to grow. 0: bad, buffer is increasing, should probably send less data
+    RunningAverage<> bufferHealth;
+    int prevBufferSize;
+
+    weak_ptr<RakNet::RakPeerInterface> peer;
+    StatsUpdateJob(shared_ptr<RakNet::RakPeerInterface> peer, DataModel* dataModel)
+        : DataModelJob("Net Peer Stats", DataModelJob::RaknetPeer, false, shared_from(dataModel), Time::Interval(0))
+        , peer(peer)
+        , prevBufferSize(0)
+    {
+        cyclicExecutive = true;
+        cyclicPriority = CyclicExecutiveJobPriority_Network_ProcessIncoming;
+    }
+
+    void updateStats(StatsMap::value_type& x)
+    {
+        if (shared_ptr<RakNet::RakPeerInterface> safePeer = peer.lock())
+        {
+            updateStats(x, safePeer.get());
+        }
+    }
+
+private:
+    void updateStats(StatsMap::value_type& x, RakNet::RakPeerInterface* peer)
+    {
+        FASTLOG(FLog::NetworkStatsReport, "updateStatsJob::updateStats running");
+        x.second.mtuSize = peer->GetMTUSize(x.first);
+        x.second.averagePing = peer->GetAveragePing(x.first);
+        x.second.lastPing = peer->GetLastPing(x.first);
+        x.second.lowestPing = peer->GetLowestPing(x.first);
+        RakNet::RakNetStatistics* stats = peer->GetStatistics(x.first);
+        if (stats)
+        {
+            int ourstats = stats->runningTotal[RakNet::ACTUAL_BYTES_RECEIVED];
+            FASTLOG1(FLog::NetworkStatsReport, "updateStatsJob rakStats ACTUAL_BYTES_RECEIVE: %d", ourstats);
+            x.second.rakStats = *stats;
+            x.second.maxPacketloss = std::max(x.second.maxPacketloss, stats->packetlossLastSecond);
+            x.second.averageBandwidthExceeded.sample(stats->isLimitedByOutgoingBandwidthLimit ? 1.0 : 0.0);
+            x.second.averageCongestionControlExceeded.sample(stats->isLimitedByCongestionControl ? 1.0 : 0.0);
+            x.second.kiloBytesSentPerSecond.sample(stats->valueOverLastSecond[RakNet::USER_MESSAGE_BYTES_PUSHED] / 1000.0f);
+            x.second.kiloBytesReceivedPerSecond.sample(stats->valueOverLastSecond[RakNet::USER_MESSAGE_BYTES_RECEIVED_PROCESSED] / 1000.0f);
+
+            x.second.updateBufferHealth(stats->messageInSendBuffer[IMMEDIATE_PRIORITY] + stats->messageInSendBuffer[HIGH_PRIORITY] +
+                                        stats->messageInSendBuffer[MEDIUM_PRIORITY] + stats->messageInSendBuffer[LOW_PRIORITY]);
+        }
+    }
+    virtual Time::Interval sleepTime(const Stats& stats)
+    {
+        return computeStandardSleepTime(stats, 30);
+    }
+
+    virtual Error error(const Stats& stats)
+    {
+        return computeStandardErrorCyclicExecutiveSleeping(stats, 30);
+    }
+
+    TaskScheduler::StepResult stepDataModelJob(const Stats& stats)
+    {
+        if (shared_ptr<RakNet::RakPeerInterface> safePeer = peer.lock())
+        {
+            boost::mutex::scoped_lock lock(mapMutex);
+
+            int bufferSize = 0;
+            for (StatsMap::iterator iter = statsMap.begin(); iter != statsMap.end(); iter++)
+            {
+                updateStats(*iter, safePeer.get());
+                RakNet::RakNetStatistics& stats = iter->second.rakStats;
+                bufferSize += stats.messageInSendBuffer[IMMEDIATE_PRIORITY] + stats.messageInSendBuffer[HIGH_PRIORITY] +
+                              stats.messageInSendBuffer[MEDIUM_PRIORITY] + stats.messageInSendBuffer[LOW_PRIORITY];
+
+                UpdateCallbackMap::iterator callback = updateCallbackMap.find(iter->first);
+                if (callback != updateCallbackMap.end())
+                {
+                    callback->second(iter->second);
+                }
+            }
+
+            calcBufferHealth(bufferSize);
+
+            return TaskScheduler::Stepped;
+        }
+        return TaskScheduler::Done;
+    }
+
+    void calcBufferHealth(int bufferSize)
+    {
+        if (bufferSize > prevBufferSize)
+            bufferHealth.sample(0.0);
+        else if (bufferSize == 0)
+            bufferHealth.sample(1);
+        else
+            bufferHealth.sample(0.5);
+
+        prevBufferSize = bufferSize;
+    }
+};
+
+void ConcurrentRakPeer::addStats(RakNet::SystemAddress address, boost::function<void(const ConnectionStats&)> callback)
+{
+    AYAASSERT(dataModel->write_requested);
+    StatsUpdateJob::StatsMap::value_type p(address, ConnectionStats());
+    statsUpdateJob->updateStats(p);
+
+    boost::mutex::scoped_lock lock(statsUpdateJob->mapMutex);
+    statsUpdateJob->statsMap.insert(p);
+    statsUpdateJob->updateCallbackMap[address] = callback;
+
+    // We need to call this callback
+    // Otherwise we send uninitialized data to Diag sometimes.
+    callback(p.second);
+}
+
+void ConcurrentRakPeer::removeStats(RakNet::SystemAddress address)
+{
+    AYAASSERT(dataModel->write_requested);
+    boost::mutex::scoped_lock lock(statsUpdateJob->mapMutex);
+    statsUpdateJob->statsMap.erase(address);
+    statsUpdateJob->updateCallbackMap.erase(address);
+}
+
+ConcurrentRakPeer::ConcurrentRakPeer(RakNet::RakPeerInterface* peer, Aya::DataModel* dataModel)
+    : peer(shared_ptr<RakNet::RakPeerInterface>(peer))
+    , dataModel(dataModel)
+{
+    packetJob.reset(new PacketJob(this->peer, dataModel));
+    statsUpdateJob.reset(new StatsUpdateJob(this->peer, dataModel));
+    Aya::TaskScheduler::singleton().add(packetJob);
+    Aya::TaskScheduler::singleton().add(statsUpdateJob);
+}
+
+
+ConcurrentRakPeer::~ConcurrentRakPeer()
+{
+    Aya::TaskScheduler::singleton().remove(packetJob);
+    Aya::TaskScheduler::singleton().remove(statsUpdateJob);
+}
+
+void ConcurrentRakPeer::Send(boost::shared_ptr<const RakNet::BitStream> bitStream, PacketPriority priority, PacketReliability reliability,
+    char orderingChannel, RakNet::SystemAddress systemAddress, bool broadcast)
+{
+    PacketJob::SendData data = {bitStream, priority, reliability, orderingChannel, systemAddress, broadcast};
+    packetJob->sendQueue.push(data);
+    TaskScheduler::singleton().reschedule(packetJob);
+}
+
+double ConcurrentRakPeer::GetBufferHealth()
+{
+    boost::mutex::scoped_lock lock(statsUpdateJob->mapMutex);
+    return statsUpdateJob->bufferHealth.value();
+}
+
+const RakNet::RakNetStatistics* ConcurrentRakPeer::GetStatistics(const RakNet::SystemAddress systemAddress) const
+{
+    boost::mutex::scoped_lock lock(statsUpdateJob->mapMutex);
+    return &statsUpdateJob->statsMap[systemAddress].rakStats;
+}
+double ConcurrentRakPeer::GetBandwidthExceeded(const RakNet::SystemAddress systemAddress)
+{
+    boost::mutex::scoped_lock lock(statsUpdateJob->mapMutex);
+    return statsUpdateJob->statsMap[systemAddress].averageBandwidthExceeded.value();
+}
+double ConcurrentRakPeer::GetCongestionControlExceeded(const RakNet::SystemAddress systemAddress)
+{
+    boost::mutex::scoped_lock lock(statsUpdateJob->mapMutex);
+    return statsUpdateJob->statsMap[systemAddress].averageCongestionControlExceeded.value();
+}
+
+
+RakNet::RakPeerInterface* ConcurrentRakPeer::rawPeer()
+{
+    AYAASSERT(dataModel->write_requested);
+    return peer.get();
+}
+const RakNet::RakPeerInterface* ConcurrentRakPeer::rawPeer() const
+{
+    AYAASSERT(dataModel->write_requested);
+    return peer.get();
+}

engine/network/src/ConcurrentRakPeer.hpp

@@ -0,0 +1,116 @@
+#pragma once
+
+#include "RakNet/RakPeerInterface.hpp"
+#include "RakNet/RakNetStatistics.hpp"
+#include "RakNet/MTUSize.hpp"
+#include "boost/noncopyable.hpp"
+#include "boost/scoped_ptr.hpp"
+#include "boost/shared_ptr.hpp"
+#include "DataModel/DataModelJob.hpp"
+
+#include "threadsafe.hpp"
+
+#include <map>
+
+
+namespace Aya
+{
+
+class DataModel;
+
+namespace Network
+{
+struct ConnectionStats
+{
+    int mtuSize;
+    int averagePing;
+    int lastPing;
+    int lowestPing;
+    float maxPacketloss;
+
+    RunningAverage<> bufferHealth;
+    int prevBufferSize;
+
+    Aya::RunningAverage<double> averageBandwidthExceeded;
+    Aya::RunningAverage<double> averageCongestionControlExceeded;
+    Aya::RunningAverage<double> kiloBytesSentPerSecond;
+    Aya::RunningAverage<double> kiloBytesReceivedPerSecond;
+
+    RakNet::RakNetStatistics rakStats;
+
+    ConnectionStats()
+        : mtuSize(MAXIMUM_MTU_SIZE)
+        , averagePing(0)
+        , lastPing(0)
+        , lowestPing(0)
+        , maxPacketloss(0.0f)
+        , prevBufferSize(0)
+        , averageBandwidthExceeded(0.1)
+        , averageCongestionControlExceeded(0.1)
+        , bufferHealth(0.1, 1.0)
+    {
+    }
+
+    void updateBufferHealth(int bufferSize)
+    {
+        if (bufferSize > prevBufferSize)
+            bufferHealth.sample(0.0);
+        else if (bufferSize == 0)
+            bufferHealth.sample(1);
+        else
+            bufferHealth.sample(0.5);
+
+        prevBufferSize = bufferSize;
+    }
+};
+
+/// Guards RakPeerInterface agains concurrent access
+/// Any Job running in DataModelJob::Write can access
+/// the peer interface directly using rawPeer().
+/// All other threads should use the functions provided.
+class ConcurrentRakPeer : boost::noncopyable
+{
+    boost::shared_ptr<RakNet::RakPeerInterface> peer;
+
+    class PacketJob;
+    boost::shared_ptr<PacketJob> packetJob;
+    class StatsUpdateJob;
+    boost::shared_ptr<StatsUpdateJob> statsUpdateJob;
+    Aya::DataModel* const dataModel;
+
+
+
+public:
+    ConcurrentRakPeer(RakNet::RakPeerInterface* peer, Aya::DataModel* dataModel);
+    ~ConcurrentRakPeer();
+
+    void addStats(RakNet::SystemAddress address, boost::function<void(const ConnectionStats&)> updateCallback);
+    void removeStats(RakNet::SystemAddress address);
+
+    // Call raw when you KNOW there are no concurrency issues
+    RakNet::RakPeerInterface* rawPeer();
+    const RakNet::RakPeerInterface* rawPeer() const;
+
+    bool equals(RakNet::RakPeerInterface* peer) const
+    {
+        return this->peer.get() == peer;
+    }
+
+    void Send(boost::shared_ptr<const RakNet::BitStream> bitStream, PacketPriority priority, PacketReliability reliability, char orderingChannel,
+        RakNet::SystemAddress systemAddress, bool broadcast);
+
+    void DeallocatePacket(RakNet::Packet* packet)
+    {
+        // Thread-safe
+        peer->DeallocatePacket(packet);
+    }
+
+    double GetBufferHealth();
+
+    // 0..1
+    double GetBandwidthExceeded(const RakNet::SystemAddress systemAddress);
+    double GetCongestionControlExceeded(const RakNet::SystemAddress systemAddress);
+    const RakNet::RakNetStatistics* GetStatistics(const RakNet::SystemAddress systemAddress) const;
+};
+} // namespace Network
+} // namespace Aya

engine/network/src/CrashReporter.cpp

@@ -0,0 +1,425 @@
+#ifdef _WIN32
+// To compile link with Dbghelp.lib
+// The callstack in release is the same as usual, which means it isn't all that accurate.
+#define NOMINMAX
+#include <Windows.h>
+#endif
+
+#include <intrusive_ptr_target.hpp>
+#include <intrusive_weak_ptr.hpp>
+
+
+#include "CrashReporter.hpp"
+
+#include <stdio.h>
+#ifdef _WIN32
+#include <DbgHelp.h>
+#endif
+#include <stdlib.h>
+#include <time.h>
+
+
+#include "RakNet/EmailSender.hpp"
+#include "RakNet/FileList.hpp"
+#include "RakNet/FileOperations.hpp"
+#include "TaskScheduler.hpp"
+
+#include "Log.hpp"
+
+#include "CEvent.hpp"
+
+#include "Debug.hpp"
+
+#include "boost.hpp"
+#include "FastLog.hpp"
+#include "Utility/StandardOut.hpp"
+
+
+#include "boost/bind.hpp"
+
+#ifdef _WIN32
+#include "StrSafe.h"
+#endif
+
+#pragma optimize("", off)
+
+CrashReporter* CrashReporter::singleton = NULL;
+
+// More info at:
+// http://www.codeproject.com/debug/postmortemdebug_standalone1.asp
+// http://www.codeproject.com/debug/XCrashReportPt3.asp
+// http://www.codeproject.com/debug/XCrashReportPt1.asp
+// http://www.microsoft.com/msj/0898/bugslayer0898.aspx
+
+#pragma warning(disable : 4996) // TODO - revisit this compiler warning
+
+LOGGROUP(HangDetection)
+LOGGROUP(Crash)
+
+DYNAMIC_FASTINTVARIABLE(WriteFullDmpPercent, 0)
+
+CrashReporter::CrashReporter()
+    : threadResult(0)
+    , exceptionInfo(NULL)
+    , reportCrashEvent(FALSE)
+    , hangReportingEnabled(false)
+    , isAlive(true)
+    , deadlockCounter(0)
+    , destructing(false)
+    , immediateUploadEnabled(true)
+{
+    assert(singleton == NULL);
+    singleton = this;
+}
+
+CrashReporter::~CrashReporter()
+{
+    destructing = true;
+
+    if (watcherThread)
+    {
+        FASTLOG(FLog::Crash, "Closing crashreporter thread on destroy");
+        reportCrashEvent.Set();
+        watcherThread->join();
+    }
+}
+
+
+HRESULT CrashReporter::GenerateDmpFileName(__out_ecount(cchdumpFilepath) char* dumpFilepath, int cchdumpFilepath, bool fastLog, bool fullDmp)
+{
+    char appDescriptor[_MAX_PATH];
+    char dumpFilename[_MAX_PATH];
+    HRESULT hr = S_OK;
+
+    StringCchPrintf(appDescriptor, _MAX_PATH, "%s %s", controls.appName, controls.appVersion);
+
+    if (FAILED(hr = StringCchCopy(dumpFilepath, cchdumpFilepath, controls.pathToMinidump)))
+        return hr;
+    WriteFileWithDirectories(dumpFilepath, 0, 0);
+    AddSlash(dumpFilepath);
+    unsigned i, dumpFilenameLen;
+    StringCchCopy(dumpFilename, _MAX_PATH, appDescriptor);
+    dumpFilenameLen = (unsigned)strlen(appDescriptor);
+    for (i = 0; i < dumpFilenameLen; i++)
+        if (dumpFilename[i] == ':')
+            dumpFilename[i] = '.'; // Can't have : in a filename
+
+    if (FAILED(hr = StringCchCat(dumpFilepath, cchdumpFilepath, dumpFilename)))
+        return hr;
+
+    if (fullDmp)
+    {
+        if (FAILED(hr = StringCchCat(dumpFilepath, cchdumpFilepath, ".Full")))
+            return hr;
+    }
+
+    if (FAILED(hr = StringCchCat(dumpFilepath, cchdumpFilepath, fastLog ? ".txt" : controls.crashExtention)))
+        return hr;
+
+    return S_OK;
+}
+
+
+LONG CrashReporter::ProcessExceptionHelper(struct _EXCEPTION_POINTERS* ExceptionInfo, bool writeFullDmp, bool noMsg, char* dumpFilepath)
+{
+    int dumpType = controls.minidumpType;
+
+    if (writeFullDmp)
+        dumpType |= MiniDumpWithFullMemory;
+
+    if (FAILED(GenerateDmpFileName(dumpFilepath, _MAX_PATH, false, writeFullDmp)))
+        return EXCEPTION_CONTINUE_SEARCH;
+
+    HANDLE hFile = CreateFile(dumpFilepath, GENERIC_WRITE, FILE_SHARE_READ, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
+    if (hFile == INVALID_HANDLE_VALUE)
+        return EXCEPTION_CONTINUE_SEARCH;
+
+    MINIDUMP_EXCEPTION_INFORMATION eInfo;
+    eInfo.ThreadId = GetCurrentThreadId();
+    eInfo.ExceptionPointers = ExceptionInfo;
+    eInfo.ClientPointers = FALSE;
+
+    LONG result = EXCEPTION_EXECUTE_HANDLER;
+    if (!MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(), hFile, (MINIDUMP_TYPE)dumpType, ExceptionInfo ? &eInfo : NULL, NULL, NULL))
+    {
+        HRESULT hr = GetLastError();
+        DWORD bytesWritten = 0;
+        TCHAR msg[1024];
+        StringCchPrintf(msg, ARRAYSIZE(msg), "MiniDumpWriteDump() failed with hr = 0x%08x", hr);
+        WriteFile(hFile, &msg, (strlen(msg) + 1) * sizeof(TCHAR), &bytesWritten, NULL);
+
+        result = EXCEPTION_CONTINUE_SEARCH;
+    }
+
+    CloseHandle(hFile);
+    return result;
+}
+
+namespace Aya
+{
+std::string specialCrashType = "first"; // generic name in case there are multiple choices
+bool gCrashIsSpecial = false;
+} // namespace Aya
+
+LONG CrashReporter::ProcessException(struct _EXCEPTION_POINTERS* ExceptionInfo, bool noMsg)
+{
+    static bool allowSpecial = true;
+    LONG result = EXCEPTION_EXECUTE_HANDLER;
+
+    char dumpFilepath[_MAX_PATH] = {};
+
+    return result;
+}
+
+LONG CrashReporter::ProcessExceptionInThead(struct _EXCEPTION_POINTERS* excpInfo)
+{
+    FASTLOG(FLog::Crash, "Processing exception in thread");
+    // stack collection happens in it's own thread.
+    // that way, we can report stacks for stack overflow exceptions.
+    if (watcherThread)
+    {
+        exceptionInfo = excpInfo;
+
+        reportCrashEvent.Set();
+
+        watcherThread->join();
+
+        watcherThread.reset(); // cleanup.
+
+        return this->threadResult;
+    }
+    else
+    {
+        // no thread? no problem!
+        return ProcessException(excpInfo, false);
+    }
+}
+
+void CrashReporter::TheadFunc(struct _EXCEPTION_POINTERS* excpInfo)
+{
+    this->threadResult = ProcessException(excpInfo, false);
+}
+
+
+
+static LONG ProcessExceptionStatic(PEXCEPTION_POINTERS excpInfo)
+{
+    if (excpInfo == NULL)
+    {
+        LONG result;
+        // Generate exception to get proper context in dump
+        __try
+        {
+            RaiseException(EXCEPTION_BREAKPOINT, 0, 0, NULL);
+        }
+        __except (result = ProcessExceptionStatic(GetExceptionInformation()), EXCEPTION_EXECUTE_HANDLER)
+        {
+        }
+        return result;
+    }
+    else
+    {
+        return CrashReporter::singleton->ProcessExceptionInThead(excpInfo);
+    }
+}
+
+LONG WINAPI CrashExceptionFilter(struct _EXCEPTION_POINTERS* excpInfo)
+{
+#ifdef _DEBUG
+    AYAASSERT(false);
+#endif
+    LONG result = ProcessExceptionStatic(excpInfo);
+
+    // Force a hard termination of the process.
+    // exit() would call termination routines, flush buffers, etc.
+    // The justification is that the program is in a bad state and
+    // might deadlock rather than exiting properly.
+    _exit(EXIT_FAILURE);
+}
+
+
+
+// http://blog.kalmbachnet.de/?postid=75
+/*
+
+Many programs are setting an own Unhandled-Exception-Filter , for catching unhandled exceptions and do some reporting or logging (for example creating
+a mini-dump ).
+
+Now, starting with VC8 (VS2005), MS changed the behaviour of the CRT is some security related and special situations.
+The CRT forces the call of the default-debugger (normally Dr.Watson) without informing the registered unhandled exception filter. The situations in
+which this happens are the following:
+
+Calling abort if the abort-behaviour was set to _CALL_REPORTFAULT. This is the default setting for release builds!
+Failures detected by security checks (see also Compiler Security Checks In Depth ) This is also enabled by default!
+If no invalid parameter handler was defined (which is the default setting) and an invalid parameter was detected (this is done in many CRT functions
+by calling _invalid_parameter). So the conclusion is: The are many situations in which your user-defined Unhandled-Exception-Filter will never be
+called. This is a major change to the previous versions of the CRT and IMHO not very well documented.
+
+The solution
+If you don<6F>t want this behavior and you will be sure that your handler will be called, you need to intercept the call to SetUnhandledExceptionFilter
+which is used by the CRT to disable all previously installed filters. You can achieve this for x86 with the following code:
+
+*/
+#ifndef _M_IX86
+
+LPTOP_LEVEL_EXCEPTION_FILTER WINAPI MyDummySetUnhandledExceptionFilter(LPTOP_LEVEL_EXCEPTION_FILTER lpTopLevelExceptionFilter)
+{
+    return NULL;
+}
+BOOL PreventSetUnhandledExceptionFilter()
+{
+    HMODULE hKernel32 = LoadLibrary("kernel32.dll");
+    if (hKernel32 == NULL)
+        return FALSE;
+    void* pOrgEntry = GetProcAddress(hKernel32, "SetUnhandledExceptionFilter");
+    if (pOrgEntry == NULL)
+        return FALSE;
+    unsigned char newJump[100];
+    DWORD dwOrgEntryAddr = (DWORD)pOrgEntry;
+    dwOrgEntryAddr += 5; // add 5 for 5 op-codes for jmp far
+    void* pNewFunc = &MyDummySetUnhandledExceptionFilter;
+    DWORD dwNewEntryAddr = (DWORD)pNewFunc;
+    DWORD dwRelativeAddr = dwNewEntryAddr - dwOrgEntryAddr;
+
+    newJump[0] = 0xE9; // JMP absolute
+    memcpy(&newJump[1], &dwRelativeAddr, sizeof(pNewFunc));
+    SIZE_T bytesWritten;
+    BOOL bRet = WriteProcessMemory(GetCurrentProcess(), pOrgEntry, newJump, sizeof(pNewFunc) + 1, &bytesWritten);
+    return bRet;
+}
+
+#endif
+
+#define PROCESS_CALLBACK_FILTER_ENABLED 0x1
+
+typedef BOOL(WINAPI* SetProcessUserModeExceptionPolicy)(__in DWORD dwFlags);
+typedef BOOL(WINAPI* GetProcessUserModeExceptionPolicy)(__out LPDWORD lpFlags);
+
+void fixExceptionsThroughKernel()
+{
+    // Try to make kernel not swallow exceptions on 64-bit os
+    // http://blog.paulbetts.org/index.php/2010/07/20/the-case-of-the-disappearing-onload-exception-user-mode-callback-exceptions-in-x64/
+
+    HMODULE hKernelDll = LoadLibrary("kernel32.dll");
+    if (hKernelDll)
+    {
+        SetProcessUserModeExceptionPolicy set = (SetProcessUserModeExceptionPolicy)GetProcAddress(hKernelDll, "SetProcessUserModeExceptionPolicy");
+        GetProcessUserModeExceptionPolicy get = (GetProcessUserModeExceptionPolicy)GetProcAddress(hKernelDll, "GetProcessUserModeExceptionPolicy");
+
+        if (set != NULL && get != NULL)
+        {
+            FASTLOG(FLog::Crash, "Found Get/SetProcessUserModeExceptionPolicy");
+            DWORD dwFlags;
+            if (get(&dwFlags))
+            {
+                FASTLOG1(FLog::Crash, "UserMode Exception Flags: %u. Setting CALLBACK_FILER off", dwFlags);
+                set(dwFlags & ~PROCESS_CALLBACK_FILTER_ENABLED); // turn off bit 1
+            }
+        }
+    }
+}
+
+void CrashReporter::Start()
+{
+    SetUnhandledExceptionFilter(CrashExceptionFilter);
+#ifndef _M_IX86
+    PreventSetUnhandledExceptionFilter();
+#endif
+
+    watcherThread.reset(new boost::thread(boost::bind(&CrashReporter::WatcherThreadFunc, CrashReporter::singleton)));
+}
+
+
+void CrashReporter::WatcherThreadFunc()
+{
+    Aya::set_thread_name("CrashReporter_WatcherThreadFunc");
+
+    Aya::Log::current()->writeEntry(Aya::Log::Information, "WatcherThread Started");
+
+    bool quit = false;
+    while (!quit && !destructing)
+    {
+        if (!reportCrashEvent.Wait(3 * 60 * 1000 /*3 minutes*/))
+        {
+            if (isAlive)
+            {
+                isAlive = false; // reset to test for next notify.
+            }
+            else if (hangReportingEnabled)
+            {
+                // not responsive!
+                if (deadlockCounter++ == 0) // only report first hang in session.
+                {
+                    /*
+                    logEvent("WatcherThread Detected hang.");
+                    FASTLOG(FLog::HangDetection, "WatcherThread Detected hang.");
+
+                    Aya::TaskScheduler::singleton().printJobs();
+
+                    LONG result;
+                    // Generate exception to get proper context in dump
+                    __try
+                    {
+                        RaiseException(EXCEPTION_BREAKPOINT, 0, 0, NULL);
+                    }
+                    __except (result = ProcessException(GetExceptionInformation(), true /*silent), EXCEPTION_EXECUTE_HANDLER)
+                    {
+                        LaunchUploadProcess();
+                    }*/
+
+                    // fuck up
+                    // random thread hangs every 5 minutes and it fucks everything up also theres a memory leak and shit
+                    // uhuahuhguhsudghsufhgUASHGHSDUGUSDUGHu
+
+                    //					_exit(EXIT_FAILURE);
+                }
+            }
+        }
+        else if (!destructing)
+        {
+            FASTLOG(FLog::Crash, "Processing non-hang");
+            threadResult = ProcessException(exceptionInfo, false);
+            quit = true;
+
+            LaunchUploadProcess();
+        }
+    }
+    logEvent("WatcherThread Exiting.");
+}
+
+void CrashReporter::EnableImmediateUpload(bool enabled)
+{
+    immediateUploadEnabled = enabled;
+}
+
+void CrashReporter::LaunchUploadProcess()
+{
+    if (!immediateUploadEnabled)
+        return;
+
+    // start new process to upload dmp
+    char filename[MAX_PATH];
+    DWORD size = GetModuleFileName(NULL, filename, MAX_PATH);
+    if (size && !strstr(filename, "RCCService"))
+    {
+        FASTLOG(FLog::Crash, "Launching process to upload dmp.");
+        STARTUPINFO si = {0};
+        si.cb = sizeof(si);
+        PROCESS_INFORMATION pi;
+        char cmd[256];
+        sprintf_s(cmd, 256, "\"%s\" -d", filename);
+        ::CreateProcess(NULL, cmd, NULL, NULL, false, NORMAL_PRIORITY_CLASS, NULL, NULL, &si, &pi);
+    }
+}
+void CrashReporter::DisableHangReporting()
+{
+    hangReportingEnabled = false;
+}
+
+
+void CrashReporter::NotifyAlive()
+{
+    FASTLOG(FLog::HangDetection, "Letting watcher thread know we're alive");
+    hangReportingEnabled = true;
+    isAlive = true;
+}

engine/network/src/CrashReporter.hpp

@@ -0,0 +1,77 @@
+#ifndef __CRASH_REPORTER_H
+#define __CRASH_REPORTER_H
+
+#include <assert.h>
+#include "boost/scoped_ptr.hpp"
+#include "boost/thread.hpp"
+#include "CEvent.hpp"
+
+
+
+/// Holds all the parameters to CrashReporter::Start
+struct CrashReportControls
+{
+    // Used to generate the dump filename.  Required with AOC_EMAIL_WITH_ATTACHMENT or AOC_WRITE_TO_DISK
+    char appName[512];
+    char appVersion[512];
+
+    // Used with AOC_WRITE_TO_DISK .  Path to write to.  Not the filename, just the path. Empty string means the current directory.
+    char pathToMinidump[300];
+    char crashExtention[64];
+
+    // How much memory to write. MiniDumpNormal is the least but doesn't seem to give correct globals. MiniDumpWithDataSegs gives more.
+    int minidumpType;
+};
+
+/// \brief On an unhandled exception, will save a minidump and email it.
+/// A minidump can be opened in visual studio to give the callstack and local variables at the time of the crash.
+/// It has the same amount of information as if you crashed while debugging in the relevant mode.  So Debug tends to give
+/// accurate stacks and info while Release does not.
+///
+/// Minidumps are only accurate for the code as it was compiled at the date of the release.  So you should label releases in source control
+/// and put that label number in the 'appVersion' field.
+
+void fixExceptionsThroughKernel();
+
+class CrashReporter
+{
+private:
+    LONG threadResult;
+    struct _EXCEPTION_POINTERS* exceptionInfo;
+    Aya::CEvent reportCrashEvent;
+    boost::scoped_ptr<boost::thread> watcherThread;
+    bool hangReportingEnabled;
+    bool isAlive;
+    LONG deadlockCounter;
+    bool destructing;
+    bool immediateUploadEnabled;
+
+    void LaunchUploadProcess();
+
+    LONG ProcessExceptionHelper(struct _EXCEPTION_POINTERS* ExceptionInfo, bool writeFullDmp, bool noMsg, char* dumpFilepath);
+
+protected:
+    bool silentCrashReporting;
+    virtual void logEvent(const char* msg) {};
+
+public:
+    static CrashReporter* singleton;
+    CrashReportControls controls;
+    CrashReporter();
+    ~CrashReporter();
+    void Start();
+    void WatcherThreadFunc();
+    virtual LONG ProcessException(struct _EXCEPTION_POINTERS* ExceptionInfo, bool noMsg);
+    LONG ProcessExceptionInThead(struct _EXCEPTION_POINTERS* ExceptionInfo);
+    void TheadFunc(struct _EXCEPTION_POINTERS* ExceptionInfo);
+    void DisableHangReporting();
+    void EnableImmediateUpload(bool enabled);
+
+    // call every second or so from FG thread to signal responsive app.
+    // must call at least once for hang reporting to be enabled.
+    void NotifyAlive();
+
+    HRESULT GenerateDmpFileName(__out_ecount(cchdumpFilepath) char* dumpFilepath, int cchdumpFilepath, bool fastLog = false, bool fullDmp = false);
+};
+
+#endif

engine/network/src/CrispProxy.hpp

@@ -0,0 +1,64 @@
+#pragma once
+
+#include <string>
+#include <map>
+#include <vector>
+
+#include "boost/shared_ptr.hpp"
+#include "boost/thread.hpp"
+
+namespace Crisp
+{
+namespace RMF3
+{
+typedef std::map<std::string, std::string> Data;
+}
+} // namespace Crisp
+
+namespace Aya
+{
+// This very closely mirrors Crisp::RMF3::Response, except we expose the fields directly
+// and remove the async-specific bits.
+struct CrispResponse
+{
+    const std::string id;
+    const int result;
+    const std::string originalContent;
+    const std::string filteredContent;
+    const int errorCode;
+    const std::string errorDescription;
+
+    CrispResponse(const std::string& id_, int result_, const std::string& originalContent_, const std::string& filteredContent_, int errorCode_,
+        const std::string& errorDescription_)
+        : id(id_)
+        , result(result_)
+        , originalContent(originalContent_)
+        , filteredContent(filteredContent_)
+        , errorCode(errorCode_)
+        , errorDescription(errorDescription_)
+    {
+    }
+
+    inline bool succeeded() const
+    {
+        return 0 == errorCode;
+    }
+};
+
+typedef boost::shared_ptr<const CrispResponse> CrispResponsePtr;
+
+// An interface of Crisp::RMF3 services.  The intention is for subclasses
+// to implement the behavior so that not all projects need the crisprmf3.dll.
+class CrispProxy
+{
+public:
+    typedef boost::function<void(CrispResponsePtr)> ResponseHandler;
+
+    virtual boost::thread checkContent(const std::string& id, const std::string& sender, const std::string& receiver, const std::string& content,
+        const Crisp::RMF3::Data* pData, const std::string& policy, ResponseHandler& callback) const = 0;
+
+    virtual boost::thread sendEvent(const std::string& id, const std::string& sender, const std::string& receiver, const std::string& event,
+        const Crisp::RMF3::Data* pData, ResponseHandler& callback) const = 0;
+};
+
+} // namespace Aya

engine/network/src/DataBlockEncryptor.cpp

@@ -0,0 +1,199 @@
+// Roblox: Moved here from RakNet 3.x
+
+/// \file
+///
+/// This file is part of RakNet Copyright 2003 Jenkins Software LLC
+///
+/// Usage of RakNet is subject to the appropriate license agreement.
+
+
+#include "DataBlockEncryptor.hpp"
+#include "RakNet/CheckSum.hpp"
+#include "RakNet/GetTime.hpp"
+#include "RakNet/Rand.hpp"
+#include "RakNet/RakAssert.hpp"
+#include <string.h>
+#include "Rijndael.hpp"
+// #include "Types.hpp"
+
+DataBlockEncryptor::DataBlockEncryptor()
+{
+    keySet = false;
+}
+
+DataBlockEncryptor::~DataBlockEncryptor() {}
+
+bool DataBlockEncryptor::IsKeySet(void) const
+{
+    return keySet;
+}
+
+void DataBlockEncryptor::SetKey(const unsigned char key[16])
+{
+    keySet = true;
+    // secretKeyAES128.set_key( key );
+    makeKey(&keyEncrypt, DIR_ENCRYPT, 16, (char*)key);
+    makeKey(&keyDecrypt, DIR_DECRYPT, 16, (char*)key);
+    cipherInit(&cipherInst, MODE_ECB, 0); // ECB is not secure except that I chain manually farther down.
+}
+
+void DataBlockEncryptor::UnsetKey(void)
+{
+    keySet = false;
+}
+
+void DataBlockEncryptor::Encrypt(
+    unsigned char* input, unsigned int inputLength, unsigned char* output, unsigned int* outputLength, RakNet::RakNetRandom* rnr)
+{
+    unsigned index, byteIndex, lastBlock;
+    unsigned int checkSum;
+    unsigned char paddingBytes;
+    unsigned char encodedPad;
+    unsigned char randomChar;
+    CheckSum checkSumCalculator;
+
+#ifdef _DEBUG
+
+    RakAssert(keySet);
+#endif
+
+    RakAssert(input && inputLength);
+
+
+    // randomChar will randomize the data so the same data sent twice will not look the same
+    randomChar = (unsigned char)rnr->RandomMT();
+
+    // 16-(((x-1) % 16)+1)
+
+    // # of padding bytes is 16 -(((input_length + extra_data -1) % 16)+1)
+    paddingBytes = (unsigned char)(16 - (((inputLength + sizeof(randomChar) + sizeof(checkSum) + sizeof(encodedPad) - 1) % 16) + 1));
+
+    // Randomize the pad size variable
+    encodedPad = (unsigned char)rnr->RandomMT();
+    encodedPad <<= 4;
+    encodedPad |= paddingBytes;
+
+    *outputLength = inputLength + sizeof(randomChar) + sizeof(checkSum) + sizeof(encodedPad) + paddingBytes;
+
+    // Write the data first, in case we are overwriting ourselves
+
+    if (input == output)
+        memmove(output + sizeof(checkSum) + sizeof(randomChar) + sizeof(encodedPad) + paddingBytes, input, inputLength);
+    else
+        memcpy(output + sizeof(checkSum) + sizeof(randomChar) + sizeof(encodedPad) + paddingBytes, input, inputLength);
+
+    // Write the random char
+    memcpy(output + sizeof(checkSum), (char*)&randomChar, sizeof(randomChar));
+
+    // Write the pad size variable
+    memcpy(output + sizeof(checkSum) + sizeof(randomChar), (char*)&encodedPad, sizeof(encodedPad));
+
+    // Write the padding
+    for (index = 0; index < paddingBytes; index++)
+        *(output + sizeof(checkSum) + sizeof(randomChar) + sizeof(encodedPad) + index) = (unsigned char)rnr->RandomMT();
+
+    // Calculate the checksum on the data
+    checkSumCalculator.Add(output + sizeof(checkSum), inputLength + sizeof(randomChar) + sizeof(encodedPad) + paddingBytes);
+
+    checkSum = checkSumCalculator.Get();
+
+    // Write checksum
+#ifdef HOST_ENDIAN_IS_BIG
+    output[0] = checkSum & 0xFF;
+    output[1] = (checkSum >> 8) & 0xFF;
+    output[2] = (checkSum >> 16) & 0xFF;
+    output[3] = (checkSum >> 24) & 0xFF;
+#else
+    memcpy(output, (char*)&checkSum, sizeof(checkSum));
+#endif
+
+    // AES on the first block
+    //	secretKeyAES128.encrypt16( output );
+    blockEncrypt(&cipherInst, &keyEncrypt, output, 16, output);
+
+    lastBlock = 0;
+
+    // Now do AES on every other block from back to front
+    for (index = *outputLength - 16; index >= 16; index -= 16)
+    {
+        for (byteIndex = 0; byteIndex < 16; byteIndex++)
+            output[index + byteIndex] ^= output[lastBlock + byteIndex];
+
+        // secretKeyAES128.encrypt16( output + index );
+        blockEncrypt(&cipherInst, &keyEncrypt, output + index, 16, output + index);
+
+        lastBlock = index;
+    }
+}
+
+bool DataBlockEncryptor::Decrypt(unsigned char* input, unsigned int inputLength, unsigned char* output, unsigned int* outputLength)
+{
+    unsigned index, byteIndex;
+    unsigned int checkSum;
+    unsigned char paddingBytes;
+    unsigned char encodedPad;
+    // NA: 2/26/2013 Remove warning for VS2012
+    // unsigned char randomChar;
+    CheckSum checkSumCalculator;
+#ifdef _DEBUG
+
+    RakAssert(keySet);
+#endif
+
+    if (input == 0 || inputLength < 16 || (inputLength % 16) != 0)
+    {
+        return false;
+    }
+
+    // Unchain in reverse order
+    for (index = 16; index <= inputLength - 16; index += 16)
+    {
+        //	secretKeyAES128.decrypt16( input + index );
+        blockDecrypt(&cipherInst, &keyDecrypt, input + index, 16, output + index);
+
+        for (byteIndex = 0; byteIndex < 16; byteIndex++)
+        {
+            if (index + 16 == (unsigned)inputLength)
+                output[index + byteIndex] ^= input[byteIndex];
+            else
+                output[index + byteIndex] ^= input[index + 16 + byteIndex];
+        }
+
+        //	lastBlock = index;
+    };
+
+    // Decrypt the first block
+    // secretKeyAES128.decrypt16( input );
+    blockDecrypt(&cipherInst, &keyDecrypt, input, 16, output);
+
+    // Read checksum
+#ifdef HOST_ENDIAN_IS_BIG
+    checkSum = (unsigned int)output[0] | (unsigned int)(output[1] << 8) | (unsigned int)(output[2] << 16) | (unsigned int)(output[3] << 24);
+#else
+    memcpy((char*)&checkSum, output, sizeof(checkSum));
+#endif
+
+    // Read the pad size variable
+    memcpy((char*)&encodedPad, output + sizeof(unsigned char) + sizeof(checkSum), sizeof(encodedPad));
+
+    // Ignore the high 4 bytes
+    paddingBytes = encodedPad & 0x0F;
+
+
+    // Get the data length
+    *outputLength = inputLength - sizeof(unsigned char) - sizeof(checkSum) - sizeof(encodedPad) - paddingBytes;
+
+    // Calculate the checksum on the data.
+    checkSumCalculator.Add(output + sizeof(checkSum), *outputLength + sizeof(unsigned char) + sizeof(encodedPad) + paddingBytes);
+
+    if (checkSum != checkSumCalculator.Get())
+        return false;
+
+    // Read the data
+    // if ( input == output )
+    memmove(output, output + sizeof(unsigned char) + sizeof(checkSum) + sizeof(encodedPad) + paddingBytes, *outputLength);
+    // else
+    //	memcpy( output, input + sizeof( randomChar ) + sizeof( checkSum ) + sizeof( encodedPad ) + paddingBytes, *outputLength );
+
+    return true;
+}

engine/network/src/DataBlockEncryptor.hpp

@@ -0,0 +1,71 @@
+// Roblox: Moved here from RakNet 3.x
+
+/// \file DataBlockEncryptor.h
+/// \internal
+/// \brief Encrypts and decrypts data blocks.  Used as part of secure connections.
+///
+/// This file is part of RakNet Copyright 2003 Jenkins Software LLC
+///
+/// Usage of RakNet is subject to the appropriate license agreement.
+
+
+#ifndef __DATA_BLOCK_ENCRYPTOR_H
+#define __DATA_BLOCK_ENCRYPTOR_H
+
+#include "Rijndael.hpp"
+#include "RakNet/RakMemoryOverride.hpp"
+
+namespace RakNet
+{
+class RakNetRandom;
+}
+
+
+/// Encrypts and decrypts data blocks.
+class DataBlockEncryptor
+{
+
+public:
+    // Constructor
+    DataBlockEncryptor();
+
+    // Destructor
+    ~DataBlockEncryptor();
+
+    /// \return true if SetKey has been called previously
+    bool IsKeySet(void) const;
+
+    /// \brief Set the encryption key
+    /// \param[in] key The new encryption key
+    void SetKey(const unsigned char key[16]);
+
+    /// \brief Unset the encryption key
+    void UnsetKey(void);
+
+    /// \brief Encryption adds 6 data bytes and then pads the number of bytes to be a multiple of 16.
+    /// \details Output should be large enough to hold this.
+    /// Output can be the same memory block as input
+    /// \param[in] input the input buffer to encrypt
+    /// \param[in] inputLength the size of the @em input buffer
+    /// \param[in] output the output buffer to store encrypted data
+    /// \param[in] outputLength the size of the output buffer
+    void Encrypt(unsigned char* input, unsigned int inputLength, unsigned char* output, unsigned int* outputLength, RakNet::RakNetRandom* rnr);
+
+    /// \brief Decryption removes bytes, as few as 6.
+    /// \details Output should be large enough to hold this.
+    /// Output can be the same memory block as input
+    /// \param[in] input the input buffer to decrypt
+    /// \param[in] inputLength the size of the @em input buffer
+    /// \param[in] output the output buffer to store decrypted data
+    /// \param[in] outputLength the size of the @em output buffer
+    /// \return False on bad checksum or input, true on success
+    bool Decrypt(unsigned char* input, unsigned int inputLength, unsigned char* output, unsigned int* outputLength);
+
+protected:
+    keyInstance keyEncrypt;
+    keyInstance keyDecrypt;
+    cipherInstance cipherInst;
+    bool keySet;
+};
+
+#endif

engine/network/src/Dictionary.cpp

@@ -0,0 +1,213 @@
+
+
+#include "Dictionary.hpp"
+#include "RakNet/StringCompressor.hpp"
+#include "Streaming.hpp"
+
+#include <boost/functional/hash/hash.hpp>
+
+namespace Aya
+{
+
+using namespace Reflection;
+
+namespace Network
+{
+
+template<>
+bool SenderDictionary<std::string>::isDefaultValue(const std::string& value)
+{
+    return value.empty();
+}
+
+template<>
+bool SenderDictionary<BinaryString>::isDefaultValue(const BinaryString& value)
+{
+    return value.value().empty();
+}
+
+void SenderDictionary<const Aya::Name*>::send(RakNet::BitStream& stream, const Aya::Name* value)
+{
+    if (isDefaultValue(value->toString()))
+    {
+        // id==0 is reserved for the empty item
+        unsigned char id = 0;
+        stream.Write(id);
+        return;
+    }
+
+    std::pair<Dictionary::iterator, bool> pair = dictionary.insert(Dictionary::value_type(value, nextIndex));
+    if (!pair.second)
+    {
+        // The item already exists in the dictionary. Simply send the current ID
+        unsigned char id = pair.first->second;
+        stream.Write(id);
+    }
+    else
+    {
+        // Remove the pre-existing entry from dictionary
+        dictionary.erase(items[nextIndex]);
+
+        // Put the new entry into the item table
+        items[nextIndex] = const_cast<Aya::Name*>(value);
+
+        // Send the id with 0x80 bit set to indicate this is a new item
+        unsigned char id = nextIndex | 0x80;
+        stream.Write(id); // TODO: We could get away with just sending a single bit. The receiver knows what the new index should be
+        stream << value->toString();
+
+        // Advance nextIndex == [1..127]
+        nextIndex %= (DICTIONARY_SIZE - 1);
+        ++nextIndex;
+    }
+}
+
+template<>
+void ReceiverDictionary<std::string>::setDefault(std::string& value)
+{
+    value.clear();
+}
+
+template<>
+void ReceiverDictionary<BinaryString>::setDefault(BinaryString& value)
+{
+    value = BinaryString();
+}
+
+void ReceiverStringDictionary::setDefault(std::string& value)
+{
+    value.clear();
+}
+
+void ReceiverStringDictionary::learn(unsigned char id, const std::string& value)
+{
+    dictionary[id] = value;
+    if (protection)
+    {
+        if (!hashTable.get())
+        {
+            hashTable.reset(new std::size_t[DICTIONARY_SIZE]);
+        }
+        boost::hash<std::string> stringHash;
+        hashTable.get()[id] = stringHash(std::string("a") + value + std::string("s"));
+    }
+}
+bool ReceiverStringDictionary::get(unsigned char id, std::string& value)
+{
+    value = dictionary[id];
+
+    if (protection)
+    {
+        AYAASSERT(hashTable.get());
+        boost::hash<std::string> stringHash;
+        if (stringHash(std::string("a") + value + std::string("s")) != hashTable.get()[id])
+        {
+            // Someone has been messing with our content... death to the infidels
+            value.clear();
+            return false;
+        }
+    }
+
+    // It's all good
+    return true;
+}
+
+void SharedStringDictionary::serializeString(const std::string& stringValue, RakNet::BitStream& bitStream)
+{
+    SenderDictionary<std::string>::send(bitStream, stringValue);
+}
+
+void SharedStringDictionary::serializeString(const ConstProperty& property, RakNet::BitStream& bitStream)
+{
+    const PropertyDescriptor& desc = property.getDescriptor();
+    std::string value = desc.getStringValue(property.getInstance());
+
+    serializeString(value, bitStream);
+}
+
+void SharedStringDictionary::deserializeString(std::string& valueString, RakNet::BitStream& bitStream)
+{
+    ReceiverDictionary<std::string>::receive(bitStream, valueString);
+}
+
+void SharedStringDictionary::deserializeString(Property& property, RakNet::BitStream& bitStream)
+{
+    std::string value;
+    deserializeString(value, bitStream);
+
+    if (property.getInstance())
+    {
+        const PropertyDescriptor& desc = property.getDescriptor();
+        desc.setStringValue(property.getInstance(), value);
+    }
+}
+
+SharedStringProtectedDictionary::SharedStringProtectedDictionary(bool protection)
+    : ReceiverStringDictionary(protection)
+{
+}
+
+void SharedStringProtectedDictionary::serializeString(const std::string& stringValue, RakNet::BitStream& bitStream)
+{
+    SenderDictionary<std::string>::send(bitStream, stringValue);
+}
+
+void SharedStringProtectedDictionary::serializeString(const ConstProperty& property, RakNet::BitStream& bitStream)
+{
+    const PropertyDescriptor& desc = property.getDescriptor();
+    std::string value = desc.getStringValue(property.getInstance());
+
+    serializeString(value, bitStream);
+}
+
+bool SharedStringProtectedDictionary::deserializeString(std::string& valueString, RakNet::BitStream& bitStream)
+{
+    return ReceiverStringDictionary::receive(bitStream, valueString);
+}
+
+bool SharedStringProtectedDictionary::deserializeString(Property& property, RakNet::BitStream& bitStream)
+{
+    std::string value;
+    if (!deserializeString(value, bitStream))
+    {
+        // They were cheating
+        return false;
+    }
+
+    if (property.getInstance())
+    {
+        const PropertyDescriptor& desc = property.getDescriptor();
+        desc.setStringValue(property.getInstance(), value);
+    }
+
+    return true;
+}
+
+void SharedBinaryStringDictionary::serializeString(const BinaryString& stringValue, RakNet::BitStream& bitStream)
+{
+    SenderDictionary<BinaryString>::send(bitStream, stringValue);
+}
+
+void SharedBinaryStringDictionary::serializeString(const ConstProperty& property, RakNet::BitStream& bitStream)
+{
+    BinaryString value = property.getValue<BinaryString>();
+
+    serializeString(value, bitStream);
+}
+
+void SharedBinaryStringDictionary::deserializeString(BinaryString& valueString, RakNet::BitStream& bitStream)
+{
+    ReceiverDictionary<BinaryString>::receive(bitStream, valueString);
+}
+
+void SharedBinaryStringDictionary::deserializeString(Property& property, RakNet::BitStream& bitStream)
+{
+    BinaryString value;
+    deserializeString(value, bitStream);
+
+    if (property.getInstance())
+        property.setValue<BinaryString>(value);
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Dictionary.hpp

@@ -0,0 +1,395 @@
+
+
+#pragma once
+
+#include <string>
+#include <map>
+#include "RakNet/BitStream.hpp"
+#include "boost/noncopyable.hpp"
+#include "Reflection/Property.hpp"
+#include "Utility/Guid.hpp"
+
+#include "Utility/BinaryString.hpp"
+
+#include "StreamingUtil.hpp"
+
+namespace Aya
+{
+
+namespace Network
+{
+// Must be a power of 2
+#define DICTIONARY_SIZE 128
+
+// A very simple dictionary of up to 127 items.
+// Sending the same item a 2nd time costs only 8 bits.
+// The algorithm is extremely simple - there's no fancy cache expiration.
+// When we run out of dictionary space we simply replace the oldest item
+// first.  (It's a cyclic buffer)
+// To use, pair a SenderDictionary with a ReceiverDictionary.
+template<class T>
+class SenderDictionary
+{
+    typedef std::map<T, unsigned char> Dictionary;
+    Dictionary dictionary;    // item --> id
+    T items[DICTIONARY_SIZE]; // id --> item
+    int nextIndex;
+    static bool isDefaultValue(const T& value);
+
+public:
+    SenderDictionary()
+        : nextIndex(1)
+    {
+    }
+    bool canSend(const T& value)
+    {
+        typename Dictionary::const_iterator iter = dictionary.find(value);
+        return (iter != dictionary.end());
+    }
+    bool trySend(RakNet::BitStream& stream, const T& value)
+    {
+        if (isDefaultValue(value))
+        {
+            // id==0 is reserved for the empty item
+            unsigned char id = 0;
+            stream.Write(id);
+            return true;
+        }
+
+        typename Dictionary::const_iterator iter = dictionary.find(value);
+        if (iter == dictionary.end())
+            return false;
+
+        // The item already exists in the dictionary. Simply send the ID
+        unsigned char id = iter->second;
+        AYAASSERT((id & 0x80) == 0);
+        AYAASSERT((id & 0x7F) < DICTIONARY_SIZE);
+
+        stream.Write(id);
+        return true;
+    }
+
+    void send(RakNet::BitStream& stream, const T& value)
+    {
+        if (isDefaultValue(value))
+        {
+            // id==0 is reserved for the empty item
+            unsigned char id = 0;
+            stream.Write(id);
+            return;
+        }
+
+        std::pair<typename Dictionary::iterator, bool> pair = dictionary.insert(typename Dictionary::value_type(value, nextIndex));
+        if (!pair.second)
+        {
+            // The item already exists in the dictionary. Simply send the current ID
+            unsigned char id = pair.first->second;
+            stream.Write(id);
+        }
+        else
+        {
+            // Remove the pre-existing entry from dictionary
+            dictionary.erase(items[nextIndex]);
+
+            // Put the new entry into the item table
+            items[nextIndex] = value;
+
+            // Send the id with 0x80 bit set to indicate this is a new item
+            unsigned char id = nextIndex | 0x80;
+            stream.Write(id); // TODO: We could get away with just sending a single bit. The receiver knows what the new index should be
+            stream << value;
+
+            // Advance nextIndex == [1..127]
+            nextIndex %= (DICTIONARY_SIZE - 1);
+            ++nextIndex;
+        }
+    }
+
+    void sendEmptyItem(RakNet::BitStream& stream)
+    {
+        // id==0 is reserved for the empty item
+        unsigned char id = 0;
+        stream.Write(id);
+        return;
+    }
+};
+
+template<>
+class SenderDictionary<const Aya::Name*>
+{
+    typedef boost::unordered_map<const Aya::Name*, unsigned char> Dictionary;
+    Dictionary dictionary;             // item --> id
+    Aya::Name* items[DICTIONARY_SIZE]; // id --> item
+    int nextIndex;
+    static bool isDefaultValue(const std::string& value)
+    {
+        return value.empty();
+    }
+
+public:
+    SenderDictionary()
+        : nextIndex(1)
+    {
+    }
+    bool canSend(const Aya::Name* value)
+    {
+        Dictionary::const_iterator iter = dictionary.find(value);
+        return (iter != dictionary.end());
+    }
+    bool trySend(RakNet::BitStream& stream, const Aya::Name* value)
+    {
+        if (isDefaultValue(value->toString()))
+        {
+            // id==0 is reserved for the empty item
+            unsigned char id = 0;
+            stream.Write(id);
+            return true;
+        }
+
+        Dictionary::const_iterator iter = dictionary.find(value);
+        if (iter == dictionary.end())
+            return false;
+
+        // The item already exists in the dictionary. Simply send the ID
+        unsigned char id = iter->second;
+        AYAASSERT((id & 0x80) == 0);
+        AYAASSERT((id & 0x7F) < DICTIONARY_SIZE);
+
+        stream.Write(id);
+        return true;
+    }
+
+    void send(RakNet::BitStream& stream, const Aya::Name* value);
+
+    void sendEmptyItem(RakNet::BitStream& stream)
+    {
+        // id==0 is reserved for the empty item
+        unsigned char id = 0;
+        stream.Write(id);
+        return;
+    }
+};
+
+template<class T>
+class ReceiverDictionary
+{
+    T dictionary[DICTIONARY_SIZE];
+    static void setDefault(T& t);
+
+public:
+    /*void skip(RakNet::BitStream& stream)
+    {
+            unsigned char id;
+            stream.Read(id);
+
+            if (id == 0)
+            {
+            }
+            else if (id & 0x80)
+            {
+                    T value;
+                    // The item is not in the dictionary
+                    stream >> value;
+            }
+    }*/
+    // d9mz - this is a hack to get around a compiler bug in VS2005
+    //        (BUT WE'RE NOT USING VS2005 ARE WE)
+    // error: '#pragma optimize' can only appear at file scope
+    // #pragma optimize( "", off )
+    void learn(unsigned char id, const T& value)
+    {
+        dictionary[id] = value;
+    }
+    bool get(unsigned char id, T& value)
+    {
+        value = dictionary[id];
+        return true;
+    }
+    bool receive(RakNet::BitStream& stream, T& value)
+    {
+        unsigned char id;
+        stream.Read(id);
+
+        if (id == 0)
+        {
+            setDefault(value);
+            return true;
+        }
+        else if (id & 0x80)
+        {
+            // The item is not in the dictionary
+            stream >> value;
+            learn(id & 0x7F, value);
+            return true;
+        }
+        else
+        {
+            // the item is in the dictionary
+            return get(id, value);
+        }
+    }
+    // d9mz - this is a hack to get around a compiler bug in VS2005
+    //        (BUT WE'RE NOT USING VS2005 ARE WE)
+    // error: '#pragma optimize' can only appear at file scope
+    // #pragma optimize( "", on )
+};
+
+class ReceiverStringDictionary
+{
+    std::string dictionary[DICTIONARY_SIZE];
+    std::auto_ptr<std::size_t> hashTable;
+    static void setDefault(std::string& t);
+
+    bool protection;
+
+public:
+    ReceiverStringDictionary(bool protection)
+        : protection(protection)
+    {
+    }
+    /*void skip(RakNet::BitStream& stream)
+    {
+            unsigned char id;
+            stream.Read(id);
+
+            if (id == 0)
+            {
+            }
+            else if (id & 0x80)
+            {
+                    T value;
+                    // The item is not in the dictionary
+                    stream >> value;
+            }
+    }*/
+    // d9mz - this is a hack to get around a compiler bug in VS2005
+    //        (BUT WE'RE NOT USING VS2005 ARE WE)
+    // error: '#pragma optimize' can only appear at file scope
+    // #pragma optimize( "", off )
+    void learn(unsigned char id, const std::string& value);
+    bool get(unsigned char id, std::string& value);
+    template<class T>
+    bool receive(RakNet::BitStream& stream, T& value)
+    {
+        unsigned char id;
+        stream.Read(id);
+
+        if (id == 0)
+        {
+            setDefault(value);
+            return true;
+        }
+        else if (id & 0x80)
+        {
+            // The item is not in the dictionary
+            stream >> value;
+            learn(id & 0x7F, value);
+            return true;
+        }
+        else
+        {
+            // the item is in the dictionary
+            return get(id, value);
+        }
+    }
+    // d9mz - this is a hack to get around a compiler bug in VS2005
+    //        (BUT WE'RE NOT USING VS2005 ARE WE)
+    // error: '#pragma optimize' can only appear at file scope
+    // #pragma optimize( "", on )
+};
+
+
+template<class T>
+class SharedDictionary
+    : public SenderDictionary<T>
+    , public ReceiverDictionary<T>
+    , boost::noncopyable
+{
+};
+
+class SharedStringDictionary
+    : public SenderDictionary<std::string>
+    , public ReceiverDictionary<std::string>
+    , boost::noncopyable
+{
+public:
+    void serializeString(const std::string& value, RakNet::BitStream& bitStream);
+    void serializeString(const Reflection::ConstProperty& property, RakNet::BitStream& bitStream);
+    void send(RakNet::BitStream& stream, const Aya::Name& value)
+    {
+        SenderDictionary<std::string>::send(stream, value.toString());
+    }
+    void send(RakNet::BitStream& stream, const char* value)
+    {
+        SenderDictionary<std::string>::send(stream, std::string(value));
+    }
+    bool trySend(RakNet::BitStream& stream, const Aya::Name& value)
+    {
+        return SenderDictionary<std::string>::trySend(stream, value.toString());
+    }
+    bool trySend(RakNet::BitStream& stream, const char* value)
+    {
+        return SenderDictionary<std::string>::trySend(stream, std::string(value));
+    }
+    void deserializeString(std::string& value, RakNet::BitStream& bitStream);
+    void deserializeString(Reflection::Property& property, RakNet::BitStream& bitStream);
+    void receive(RakNet::BitStream& stream, const Aya::Name*& value)
+    {
+        std::string s;
+        ReceiverDictionary<std::string>::receive(stream, s);
+        value = &Aya::Name::declare(s.c_str());
+    }
+};
+
+
+class SharedStringProtectedDictionary
+    : public SenderDictionary<std::string>
+    , public ReceiverStringDictionary
+    , boost::noncopyable
+{
+public:
+    SharedStringProtectedDictionary(bool protection);
+    void serializeString(const std::string& value, RakNet::BitStream& bitStream);
+    void serializeString(const Reflection::ConstProperty& property, RakNet::BitStream& bitStream);
+    void send(RakNet::BitStream& stream, const Aya::Name& value)
+    {
+        SenderDictionary<std::string>::send(stream, value.toString());
+    }
+    void send(RakNet::BitStream& stream, const char* value)
+    {
+        SenderDictionary<std::string>::send(stream, std::string(value));
+    }
+    bool trySend(RakNet::BitStream& stream, const Aya::Name& value)
+    {
+        return SenderDictionary<std::string>::trySend(stream, value.toString());
+    }
+    bool trySend(RakNet::BitStream& stream, const char* value)
+    {
+        return SenderDictionary<std::string>::trySend(stream, std::string(value));
+    }
+    bool deserializeString(std::string& value, RakNet::BitStream& bitStream);
+    bool deserializeString(Reflection::Property& property, RakNet::BitStream& bitStream);
+    void receive(RakNet::BitStream& stream, const Aya::Name*& value)
+    {
+        std::string s;
+        ReceiverStringDictionary::receive(stream, s);
+        // TODO: If there's a bug, then this could grow the Aya::Name database in
+        // and unbounded fashion. Use lookup() instead
+        value = &Aya::Name::declare(s.c_str());
+    }
+};
+
+class SharedBinaryStringDictionary
+    : public SenderDictionary<BinaryString>
+    , public ReceiverDictionary<BinaryString>
+    , boost::noncopyable
+{
+public:
+    void serializeString(const BinaryString& value, RakNet::BitStream& bitStream);
+    void serializeString(const Reflection::ConstProperty& property, RakNet::BitStream& bitStream);
+
+    void deserializeString(BinaryString& value, RakNet::BitStream& bitStream);
+    void deserializeString(Reflection::Property& property, RakNet::BitStream& bitStream);
+};
+} // namespace Network
+} // namespace Aya

engine/network/src/DirectPhysicsReceiver.cpp

@@ -0,0 +1,188 @@
+
+
+#include "DirectPhysicsReceiver.hpp"
+#include "Compressor.hpp"
+#include "RakNet/GetTime.hpp"
+#include "NetworkSettings.hpp"
+
+#include "Streaming.hpp"
+#include "Replicator.hpp"
+
+#include "DataModel/PartInstance.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+
+#include "World/World.hpp"
+#include "World/Primitive.hpp"
+#include "World/Assembly.hpp"
+#include "Utility/StandardOut.hpp"
+
+
+#include <boost/utility/enable_if.hpp>
+#include <boost/type_traits.hpp>
+#include "NetworkProfiler.hpp"
+
+DYNAMIC_FASTFLAG(PhysicsSenderUseOwnerTimestamp)
+DYNAMIC_FASTFLAG(CleanUpInterpolationTimestamps)
+SYNCHRONIZED_FASTFLAG(PhysicsPacketSendWorldStepTimestamp)
+
+using namespace Aya;
+using namespace Aya::Network;
+
+void DirectPhysicsReceiver::receivePacket(RakNet::BitStream& inBitstream, RakNet::Time timeStamp, ReplicatorStats::PhysicsReceiverStats* _stats)
+{
+    RakNet::Time interpolationTimestamp;
+    Aya::RemoteTime remoteSendTime;
+    if (SFFlag::getPhysicsPacketSendWorldStepTimestamp())
+    {
+        inBitstream >> interpolationTimestamp;
+        remoteSendTime = ((double)interpolationTimestamp / 1000.0f);
+    }
+    else
+    {
+        remoteSendTime = ((double)timeStamp / 1000.0f);
+    }
+
+    stats = _stats;
+
+    RakNet::Time now = RakNet::GetTime();
+
+    // AYAASSERT(now >= timeStamp); // Defect filed: DE6810
+    if (!SFFlag::getPhysicsPacketSendWorldStepTimestamp() && DFFlag::PhysicsSenderUseOwnerTimestamp && timeStamp > now)
+    {
+        timeStamp = now;
+    }
+
+    RakNet::Time deltaTime = SFFlag::getPhysicsPacketSendWorldStepTimestamp() ? now - interpolationTimestamp : now - timeStamp; //
+
+    // local Aya time when the packet was sent
+    Time localTime = replicator->raknetTimeToRbxTime(timeStamp);
+
+    while (true)
+    {
+        shared_ptr<PartInstance> part;
+
+        if (replicator->isStreamingEnabled())
+        {
+            bool done;
+            inBitstream >> done;
+            if (done)
+                break; // packet ended
+
+            bool cframeOnly;
+            inBitstream >> cframeOnly;
+            if (cframeOnly)
+            {
+                receiveMechanismCFrames(inBitstream, timeStamp, remoteSendTime);
+                continue;
+            }
+            else if (!receiveRootPart(part, inBitstream))
+            {
+                continue; // mechanism ended
+            }
+        }
+        else // non-streaming
+        {
+            if (!receiveRootPart(part, inBitstream))
+            {
+                break; // packet ended
+            }
+            BOOST_STATIC_ASSERT(sizeof(RakNet::Time) == sizeof(part->raknetTime));
+        }
+
+        // read mechanism if non-streaming OR (streaming and not cframe only)
+        if (part)
+        {
+            if (!iAmServer && localTime < part->getLastUpdateTime())
+            {
+                // old packet, discard
+                if (replicator->settings().printPhysicsErrors)
+                {
+                    Aya::StandardOut::singleton()->print(Aya::MESSAGE_INFO, "Discard old packet");
+                }
+                part.reset();
+            }
+
+            if (part)
+            {
+#if !defined(__linux) && !defined(__APPLE__)
+                RakNet::Time& partTime = part->raknetTime;
+                if (partTime > (SFFlag::getPhysicsPacketSendWorldStepTimestamp() ? interpolationTimestamp : timeStamp))
+                {
+                    if (replicator->settings().printPhysicsErrors)
+                    {
+                        Aya::StandardOut::singleton()->print(Aya::MESSAGE_INFO, "Physics-in old packet");
+                    }
+                    part.reset();
+                }
+                else
+                {
+                    if (SFFlag::getPhysicsPacketSendWorldStepTimestamp())
+                    {
+                        // RakNetTime will no longer hold some arbitrary send time, but instead
+                        // adjusted Physics Timestamp
+                        partTime = interpolationTimestamp;
+                    }
+                    else
+                    {
+                        partTime = timeStamp;
+                    }
+                }
+#else
+                RakNet::Time* partTime = &(part->raknetTime);
+                if (*partTime > (SFFlag::getPhysicsPacketSendWorldStepTimestamp() ? interpolationTimestamp : timeStamp))
+                {
+                    if (replicator->settings().printPhysicsErrors)
+                    {
+                        Aya::StandardOut::singleton()->print(Aya::MESSAGE_INFO, "Physics-in old packet");
+                    }
+                    part.reset();
+                }
+                else
+                {
+                    if (SFFlag::getPhysicsPacketSendWorldStepTimestamp())
+                    {
+                        // RakNetTime will no longer hold some arbitrary send time, but instead
+                        // adjusted Physics Timestamp
+                        *partTime = interpolationTimestamp;
+                    }
+                    else
+                    {
+                        *partTime = timeStamp;
+                    }
+                }
+#endif
+            }
+        }
+
+        RakNet::BitSize_t characterBits = inBitstream.GetReadOffset();
+        int numNodesInHistory;
+        receiveMechanism(inBitstream, part.get(), tempItem, remoteSendTime, numNodesInHistory);
+        if (stats && part && "Torso" == part->getName())
+        {
+            stats->details.characterAnim.increment();
+            stats->details.characterAnimSize.sample((inBitstream.GetReadOffset() - characterBits) / 8);
+        }
+
+        if (part)
+        {
+            setPhysics(tempItem, remoteSendTime, deltaTime, numNodesInHistory);
+
+            if (!iAmServer)
+            {
+                if (DFFlag::CleanUpInterpolationTimestamps)
+                {
+                    // Interpolation Delay has been re-purposed for use with relation to NetworkPing
+                    // Interpolation Delay depends greatly on NetworkPing and Frequency of Updates.
+                    part->setInterpolationDelay((double)deltaTime / 1000);
+                }
+                else
+                {
+                    part->setInterpolationDelay((localTime - part->getLastUpdateTime()).seconds());
+                }
+                part->setLastUpdateTime(localTime);
+            }
+        }
+    }
+}

engine/network/src/DirectPhysicsReceiver.hpp

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "PhysicsReceiver.hpp"
+#include "ReplicatorStats.hpp"
+
+namespace Aya
+{
+
+namespace Network
+{
+
+class DirectPhysicsReceiver : public PhysicsReceiver
+{
+private:
+    MechanismItem tempItem;
+
+public:
+    DirectPhysicsReceiver(Replicator* replicator, bool isServer)
+        : PhysicsReceiver(replicator, isServer)
+    {
+    }
+    virtual void receivePacket(RakNet::BitStream& bitsream, RakNet::Time timeStamp, ReplicatorStats::PhysicsReceiverStats* stats);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/ErrorCompPhysicsSender.cpp

@@ -0,0 +1,417 @@
+
+#if 1 // removing deprecated senders, set to 0 when FastFlag RemoveUnusedPhysicsSenders is set to true and removed (delete this file)
+#include "ErrorCompPhysicsSender.hpp"
+#include "Replicator.hpp"
+#include "Compressor.hpp"
+
+#include "Player.hpp"
+
+
+#include "World/World.hpp"
+#include "World/Primitive.hpp"
+#include "World/Assembly.hpp"
+#include "MechanismItem.hpp"
+
+#include "DataModel/ModelInstance.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+#include "DataModel/PhysicsService.hpp"
+
+
+#include "RakNet/GetTime.hpp"
+#include "RakNet/RakPeerInterface.hpp"
+#include "NetworkSettings.hpp"
+#include "ConcurrentRakPeer.hpp"
+#include "NetworkPacketCache.hpp"
+
+#include "Streaming.hpp"
+
+using namespace Aya;
+using namespace Aya::Network;
+
+
+
+ErrorCompPhysicsSender::ErrorCompPhysicsSender(Replicator& replicator)
+    : PhysicsSender(replicator)
+    , stepId(0)
+{
+    if (replicator.isProtocolCompatible())
+        physicsPacketCache = shared_from(ServiceProvider::find<PhysicsPacketCache>(&replicator));
+}
+
+ErrorCompPhysicsSender::~ErrorCompPhysicsSender() {}
+
+
+
+void ErrorCompPhysicsSender::step()
+{
+    /*
+
+            First, update the error on all items that had been sent
+            after the last step.  These will have a stepId of 0.
+
+            Also, visit some of the oldest items in the list as well.
+
+        TODO: Rather than picking a constant "countdown", pick a number that
+                  is a fraction of the items sent last frame (This will
+                      aid in auto-throttling how much we send).
+
+    */
+
+    if (!physicsService)
+    {
+        physicsService = shared_from(ServiceProvider::find<PhysicsService>(&replicator));
+
+        AYAASSERT(physicsService); // datamodel should always create this
+
+        // Get all current assemblies
+        std::for_each(physicsService->begin(), physicsService->end(), boost::bind(&ErrorCompPhysicsSender::addNugget, this, _1));
+
+        // Listen for assembly changes
+        addingAssemblyConnection = physicsService->assemblyAddingSignal.connect(boost::bind(&ErrorCompPhysicsSender::onAddingAssembly, this, _1));
+        // removedAssemblyConnection = physicsService->assemblyRemovedSignal.connect(boost::bind(&ErrorCompPhysicsSender::onRemovedAssembly, this,
+        // _1));
+    }
+
+    std::for_each(newMovingAssemblies.begin(), newMovingAssemblies.end(), boost::bind(&ErrorCompPhysicsSender::addNugget2, this, _1));
+    newMovingAssemblies.clear();
+
+    // Prepare the data needed to compute errors
+
+    // TODO: const
+    const ModelInstance* characterModel = NULL;
+    CoordinateFrame focus;
+    if (const Player* player = replicator.findTargetPlayer())
+    {
+        if (player->hasCharacterHead(focus))
+        {
+            characterModel = player->getConstCharacter();
+        }
+    }
+
+    stepId++; // can go for 100's of days at 30 fps until maxing out
+
+    {
+        int numOldVisits = 0;
+
+        while (numOldVisits < 100)
+        {
+            NuggetList::iterator iter = nuggetUpdateList.begin();
+            if (iter == nuggetUpdateList.end())
+                break;
+
+            Nugget* nugget = &(**iter);
+
+            AYAASSERT(nugget->errorStepId <= stepId);
+
+            // remove nugget if part is not in physics service sender list, it's not moving.
+            if (!nugget->part->PhysicsServiceHook::is_linked())
+            {
+                removeNugget(nugget->part);
+                continue;
+            }
+
+            if (nugget->errorStepId == stepId)
+            {
+                break; // we are done - first in the list is caught up
+            }
+            else
+            {
+                if (nugget->errorStepId > 0)
+                { // don't include just-sent items in the count of old items
+                    ++numOldVisits;
+                }
+                nugget->computeError(focus, characterModel, stepId);
+            }
+
+            // resort nugget into send list
+            nuggetSendList.erase(*iter);
+            SortedNuggetList::iterator newPos = nuggetSendList.insert(*nugget);
+
+            // update update iterator
+            *iter = newPos;
+
+            // move just updated item to end of update list
+            nuggetUpdateList.splice(nuggetUpdateList.end(), nuggetUpdateList, iter);
+
+            // update nugget iterator inside update list
+            nugget->updateListIter = --nuggetUpdateList.end();
+        }
+    }
+}
+
+int ErrorCompPhysicsSender::sendPacket(int maxPackets, PacketPriority packetPriority, ReplicatorStats::PhysicsSenderStats* stats)
+{
+    const unsigned int maxStreamSize(replicator.getPhysicsMtuSize());
+
+    boost::shared_ptr<RakNet::BitStream> bitStream;
+
+    int packetCount = 0;
+
+    SortedNuggetList::iterator iter = nuggetSendList.begin();
+
+    // Iterate by error until
+    while (iter != nuggetSendList.end())
+    {
+        if (!bitStream)
+        {
+            if (packetCount >= maxPackets)
+                break;
+
+            bitStream.reset(new RakNet::BitStream(maxStreamSize));
+
+            ++packetCount;
+
+            *bitStream << (unsigned char)ID_TIMESTAMP;
+            RakNet::Time now = RakNet::GetTime();
+#if !defined(__linux) && !defined(__APPLE__)
+            *bitStream << now;
+#else
+            *bitStream << static_cast<unsigned long long>(now);
+#endif
+            *bitStream << (unsigned char)ID_PHYSICS;
+        }
+
+        Nugget& nugget = *iter;
+
+        // remove nugget if part is not in physics service sender list, it's not moving
+        if (!nugget.part->PhysicsServiceHook::is_linked())
+        {
+            ++iter;
+            // remove will increment curBucket->iter
+            removeNugget(nugget.part);
+        }
+        else
+        {
+            if (!sendPhysicsData(*bitStream, nugget))
+            {
+                ++iter;
+                continue;
+            }
+
+            if (nugget.part->getConstPartPrimitive()->getConstAssembly())
+            {
+                Nugget::onSent(nugget);
+
+                // move just sent item to front of update list
+                nuggetUpdateList.splice(nuggetUpdateList.begin(), nuggetUpdateList, nugget.updateListIter);
+                nugget.updateListIter = nuggetUpdateList.begin();
+
+                AYAASSERT((*iter).lastSent == nugget.part->getCoordinateFrame());
+                AYAASSERT((*iter).biggestSize == Nugget::minSize());
+                AYAASSERT((*iter).errorStepId == 0);
+                ++iter;
+            }
+            else
+            {
+                AYAASSERT(0);
+            }
+        }
+
+        if (bitStream->GetNumberOfBytesUsed() > static_cast<int>(maxStreamSize))
+        {
+            // Packet "end" tag
+            if (replicator.isStreamingEnabled())
+                *bitStream << true; // done
+            else
+                replicator.serializeId(*bitStream, NULL);
+
+            // Send ID_PHYSICS
+            replicator.rakPeer->Send(bitStream, packetPriority, UNRELIABLE, PHYSICS_CHANNEL, replicator.remotePlayerId, false);
+            stats->physicsPacketsSent.sample();
+            stats->physicsPacketsSentSmooth.sample();
+            stats->physicsPacketsSentSize.sample(bitStream->GetNumberOfBytesUsed());
+
+            bitStream.reset();
+        }
+    }
+
+    if (bitStream)
+    {
+        // Packet "end" tag
+        if (replicator.isStreamingEnabled())
+            *bitStream << true; // done
+        else
+            replicator.serializeId(*bitStream, NULL);
+
+        // Send ID_PHYSICS
+        replicator.rakPeer->Send(bitStream, packetPriority, UNRELIABLE, PHYSICS_CHANNEL, replicator.remotePlayerId, false);
+        stats->physicsPacketsSent.sample();
+        stats->physicsPacketsSentSmooth.sample();
+        stats->physicsPacketsSentSize.sample(bitStream->GetNumberOfBytesUsed());
+    }
+
+    return packetCount;
+};
+
+
+void ErrorCompPhysicsSender::onAddingAssembly(shared_ptr<Instance> assembly)
+{
+    shared_ptr<PartInstance> part = Instance::fastSharedDynamicCast<PartInstance>(assembly);
+    AYAASSERT(part);
+
+    newMovingAssemblies.push_back(part);
+}
+
+void ErrorCompPhysicsSender::addNugget(PartInstance& part)
+{
+    addNugget2(shared_from(&part));
+}
+
+void ErrorCompPhysicsSender::addNugget2(shared_ptr<PartInstance> part)
+{
+    // try add item to map
+    std::pair<NuggetMap::iterator, bool> result = nuggetMap.insert(std::make_pair(part, Nugget(part)));
+
+    // add succeeded
+    if (result.second)
+    {
+        // put new nugget to end of send list, it'll get updated next step()
+        SortedNuggetList::iterator iter = nuggetSendList.insert(nuggetSendList.end(), result.first->second);
+        nuggetUpdateList.push_front(iter);
+
+        iter->updateListIter = nuggetUpdateList.begin();
+        iter->radius = part->getPartPrimitive()->getAssembly()->getLastComputedRadius();
+    }
+}
+
+void ErrorCompPhysicsSender::removeNugget(shared_ptr<const PartInstance> part)
+{
+    NuggetMap::iterator iter = nuggetMap.find(part);
+    AYAASSERT(iter != nuggetMap.end());
+
+    nuggetSendList.erase(*iter->second.updateListIter);
+    nuggetUpdateList.erase(iter->second.updateListIter);
+    nuggetMap.erase(iter);
+}
+
+void ErrorCompPhysicsSender::onRemovedAssembly(shared_ptr<Instance> assembly)
+{
+    shared_ptr<const PartInstance> part = Instance::fastSharedDynamicCast<PartInstance>(assembly);
+    AYAASSERT(part);
+
+    removeNugget(part);
+}
+
+bool ErrorCompPhysicsSender::sendPhysicsData(RakNet::BitStream& bitStream, const Nugget& nugget)
+{
+    sendDetailed = nugget.sendDetailed;
+    CoordinateFrame cFrame = nugget.lastSent;
+    float accumulatedError;
+    return PhysicsSender::sendPhysicsData(bitStream, nugget.part.get(), sendDetailed, currentStepTimestamp, cFrame, accumulatedError, NULL);
+}
+
+void ErrorCompPhysicsSender::writeAssembly(
+    RakNet::BitStream& bitStream, const Assembly* assembly, Compressor::CompressionType compressionType, bool crossPacketCompression)
+{
+    if (physicsPacketCache)
+    {
+        // look for this packet in the cache
+        if (!physicsPacketCache->fetchIfUpToDate(assembly, (unsigned char)sendDetailed, bitStream))
+        {
+            // cache miss, recreate bit stream and update cache
+            unsigned int startBit = bitStream.GetWriteOffset();
+
+            PhysicsSender::writeAssembly(bitStream, assembly, compressionType);
+
+            bitStream.SetReadOffset(startBit);
+
+            if (!physicsPacketCache->update(assembly, (unsigned char)sendDetailed, bitStream, startBit, bitStream.GetNumberOfBitsUsed() - startBit))
+            {
+                // this item should be in the cache
+                StandardOut::singleton()->print(MESSAGE_INFO, "cache update failed");
+                AYAASSERT(0);
+            }
+        }
+    }
+    else
+        PhysicsSender::writeAssembly(bitStream, assembly, compressionType);
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+float ErrorCompPhysicsSender::Nugget::minDistance()
+{
+    return 3.0f;
+}
+float ErrorCompPhysicsSender::Nugget::maxDistance()
+{
+    return 1000.0f;
+}
+float ErrorCompPhysicsSender::Nugget::minSize()
+{
+    return 2.0f;
+}
+float ErrorCompPhysicsSender::Nugget::maxSize()
+{
+    return 50.0f;
+}
+
+ErrorCompPhysicsSender::Nugget::Nugget(shared_ptr<PartInstance> part)
+    : part(part) // indexes
+    , error(0.0f)
+    , errorStepId(-1)
+    , lastSent(part->getCoordinateFrame())
+    , biggestSize(minSize())
+    , sendDetailed(true)
+    , radius(0.0f)
+{
+}
+
+// static
+void ErrorCompPhysicsSender::Nugget::onSent(Nugget& nugget)
+{
+    // part - no change
+    // error - will be computed
+    nugget.errorStepId = 0;
+    nugget.lastSent = nugget.part->getCoordinateFrame();
+    nugget.biggestSize = minSize(); // resets
+}
+
+
+void ErrorCompPhysicsSender::Nugget::computeError(const CoordinateFrame& focus, const ModelInstance* focusModel, int stepId)
+{
+    AYAASSERT(part->getConstPartPrimitive()->getConstAssembly());
+
+    if ((stepId % 20) == 0)
+    {
+        const Assembly* assembly = part->getConstPartPrimitive()->getConstAssembly();
+        AYAASSERT(assembly);
+
+        radius = assembly->getLastComputedRadius();
+    }
+
+    const CoordinateFrame& cf = part->getCoordinateFrame();
+
+    // 1. Biggest Size
+    biggestSize = G3D::clamp(2 * radius, biggestSize, maxSize());
+
+    // current
+    float currentDistance = minDistance(); // There is no center of interest.  Location doesn't affect error
+    if (focusModel)
+    {
+        currentDistance = Math::taxiCabMagnitude(cf.translation - focus.translation) - radius;
+        currentDistance = G3D::clamp(currentDistance, minDistance(), maxDistance());
+    }
+
+    // 2. Error
+    // 3. Send Detailed
+    if (part->isDescendantOf(focusModel))
+    { // the character!
+        error = std::numeric_limits<double>::max();
+        sendDetailed = true;
+    }
+    else
+    {
+        const float positionalError = Math::taxiCabMagnitude(cf.translation - lastSent.translation);
+        const float rotationError = Math::sumDeltaAxis(cf.rotation, lastSent.rotation) * radius * 0.2f; // factor down - we are summing the axis
+        error = biggestSize * biggestSize * (rotationError + positionalError) / currentDistance;
+        sendDetailed = currentDistance < 20.0f;
+    }
+
+    // 4. errorStepId
+    errorStepId = stepId;
+}
+#endif // removing deprecated senders

engine/network/src/ErrorCompPhysicsSender.hpp

@@ -0,0 +1,125 @@
+
+#if 1 // removing deprecated senders, set to 0 when FastFlag RemoveUnusedPhysicsSenders is set to true and removed (delete this file)
+#pragma once
+
+#include "PhysicsSender.hpp"
+#include "BoostAppend.hpp"
+
+#include "DataModel/PartInstance.hpp"
+
+
+#include <boost/multi_index_container.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/hashed_index.hpp>
+
+#include "signal.hpp"
+
+
+#include "boost/pool/pool_alloc.hpp"
+#include "boost/intrusive/set.hpp"
+using boost::multi_index_container;
+using namespace boost::multi_index;
+
+
+namespace Aya
+{
+
+class Instance;
+class PhysicsService;
+class ModelInstance;
+
+namespace Network
+{
+
+class Replicator;
+class PhysicsPacketCache;
+
+class ErrorCompPhysicsSender : public PhysicsSender
+{
+
+    class Nugget;
+
+    // custom containers
+    typedef std::list<shared_ptr<PartInstance>, boost::fast_pool_allocator<shared_ptr<PartInstance>>> NewMovingAssemblies;
+    NewMovingAssemblies newMovingAssemblies; // new moving assemblies for current step
+
+    typedef boost::intrusive::set_base_hook<boost::intrusive::tag<Nugget>, boost::intrusive::link_mode<boost::intrusive::normal_link>> NuggetHook;
+    typedef boost::intrusive::multiset<Nugget, boost::intrusive::compare<std::greater<Nugget>>, boost::intrusive::base_hook<NuggetHook>>
+        SortedNuggetList;
+
+    typedef std::list<SortedNuggetList::iterator> NuggetList;
+    typedef boost::unordered_map<shared_ptr<const PartInstance>, Nugget> NuggetMap;
+
+    NuggetMap nuggetMap;
+    NuggetList nuggetUpdateList;
+    SortedNuggetList nuggetSendList;
+
+    class Nugget : public NuggetHook
+    {
+    public:
+        shared_ptr<const PartInstance> part;
+        double error;
+        int errorStepId; // "stepId" of the last send  (-1 means uninitialized. 0 means we just got sent and need updating)
+
+        Nugget(shared_ptr<PartInstance> part);
+
+        static void onSent(Nugget& nugget);
+
+        void computeError(const CoordinateFrame& focus, const ModelInstance* focusModel, int timestamp);
+
+        CoordinateFrame lastSent; // the CF last sent
+        bool sendDetailed;        // Should this send detailed (uncompressed) physics info
+        float biggestSize;        // May change for an articulated assembly....
+        float radius;             // cached assembly radius
+
+        static float minDistance();
+        static float maxDistance();
+        static float minSize();
+        static float maxSize();
+
+        friend bool operator>(const Nugget& a, const Nugget& b)
+        {
+            return a.error > b.error;
+        }
+
+        NuggetList::iterator updateListIter; // iterator into update list
+    };
+
+    int stepId;
+    shared_ptr<PhysicsService> physicsService;
+    shared_ptr<PhysicsPacketCache> physicsPacketCache;
+
+    bool sendDetailed;
+
+    Aya::signals::scoped_connection addingAssemblyConnection;
+    Aya::signals::scoped_connection removedAssemblyConnection;
+
+public:
+    ErrorCompPhysicsSender(Replicator& replicator);
+    ~ErrorCompPhysicsSender();
+
+    virtual void step();
+    virtual int sendPacket(int maxPackets, PacketPriority packetPriority, ReplicatorStats::PhysicsSenderStats* stats);
+
+protected:
+    virtual void writeAssembly(
+        RakNet::BitStream& bitStream, const Assembly* assembly, Compressor::CompressionType compressionType, bool crossPacketCompression = false);
+
+private:
+    void updateErrors();
+    bool sendPhysicsData(RakNet::BitStream& bitStream, const Nugget& nugget);
+
+    void addNugget(PartInstance& part);
+    void addNugget2(shared_ptr<PartInstance> part);
+
+    void onAddingAssembly(shared_ptr<Instance> assembly);
+    void onRemovedAssembly(shared_ptr<Instance> assembly);
+    void removeNugget(shared_ptr<const PartInstance> part);
+};
+
+
+} // namespace Network
+} // namespace Aya
+
+#endif // removing deprecated senders

engine/network/src/ErrorCompPhysicsSender2.cpp

@@ -0,0 +1,623 @@
+
+#if 1 // removing deprecated senders, set to 0 when FastFlag RemoveUnusedPhysicsSenders is set to true and removed (delete this file)
+
+#include "ErrorCompPhysicsSender2.hpp"
+#include "Replicator.hpp"
+#include "Compressor.hpp"
+
+#include "Player.hpp"
+
+
+#include "World/World.hpp"
+#include "World/Primitive.hpp"
+#include "World/Assembly.hpp"
+#include "MechanismItem.hpp"
+
+#include "DataModel/ModelInstance.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+#include "DataModel/PhysicsService.hpp"
+
+
+#include "RakNet/GetTime.hpp"
+#include "RakNet/RakPeerInterface.hpp"
+#include "NetworkSettings.hpp"
+#include "ConcurrentRakPeer.hpp"
+#include "NetworkPacketCache.hpp"
+
+#include "Streaming.hpp"
+
+using namespace Aya;
+using namespace Aya::Network;
+
+#define NUM_GROUPS 4
+
+ErrorCompPhysicsSender2::ErrorCompPhysicsSender2(Replicator& replicator)
+    : PhysicsSender(replicator)
+    , bucketIndex(0)
+    , stepId(0)
+{
+
+    if (replicator.isProtocolCompatible())
+        physicsPacketCache = shared_from(ServiceProvider::find<PhysicsPacketCache>(&replicator));
+
+    for (int i = 0; i < NUM_GROUPS; i++)
+    {
+        nuggetSendGroups.push_back(Bucket());
+    }
+}
+
+ErrorCompPhysicsSender2::~ErrorCompPhysicsSender2() {}
+
+
+
+void ErrorCompPhysicsSender2::step()
+{
+    /*
+
+            First, update the error on all items that had been sent
+            after the last step.  These will have a stepId of 0.
+
+            Also, visit some of the oldest items in the list as well.
+
+        TODO: Rather than picking a constant "countdown", pick a number that
+                  is a fraction of the items sent last frame (This will
+                      aid in auto-throttling how much we send).
+
+    */
+
+    if (!physicsService)
+    {
+        physicsService = shared_from(ServiceProvider::find<PhysicsService>(&replicator));
+
+        AYAASSERT(physicsService); // datamodel should always create this
+
+        // Get all current assemblies
+        std::for_each(physicsService->begin(), physicsService->end(), boost::bind(&ErrorCompPhysicsSender2::addNugget, this, _1));
+
+        // Listen for assembly changes
+        // Optimization: onAddingAssemly simply adds new item to a STL list, more expensive operations are deferred til later
+        addingAssemblyConnection = physicsService->assemblyAddingSignal.connect(boost::bind(&ErrorCompPhysicsSender2::onAddingAssembly, this, _1));
+        // removedAssemblyConnection = physicsService->assemblyRemovedSignal.connect(boost::bind(&ErrorCompPhysicsSender2::onRemovedAssembly, this,
+        // _1));
+    }
+
+    // Optimization: adding and removing nugget is expensive, by deferring it to here we allow it to run concurrently
+    // process new moving assemblies
+    std::for_each(newMovingAssemblies.begin(), newMovingAssemblies.end(), boost::bind(&ErrorCompPhysicsSender2::addNugget2, this, _1));
+    newMovingAssemblies.clear();
+
+    // Prepare the data needed to compute errors
+    const ModelInstance* characterModel = NULL;
+    CoordinateFrame focus;
+    if (const Player* player = replicator.findTargetPlayer())
+    {
+        if (player->hasCharacterHead(focus))
+        {
+            characterModel = player->getConstCharacter();
+        }
+    }
+
+    stepId++; // can go for 100's of days at 30 fps until maxing out
+
+    // Compute errors
+    {
+        int numOldVisits = 0;
+
+        NuggetUpdateList::iterator iter = nuggetUpdateList.begin();
+        while (numOldVisits <= 0)
+        {
+            if (iter == nuggetUpdateList.end())
+                break;
+
+            Nugget* nugget = &(**iter);
+            AYAASSERT(nugget->errorStepId <= stepId);
+
+            // remove nugget if part is not in physics service sender list, it's not moving.
+            if (!nugget->part->PhysicsServiceHook::is_linked())
+            {
+                iter++;
+                removeNugget(nugget->part);
+                continue;
+            }
+
+            int oldGroup = nugget->groupId;
+
+            if (nugget->errorStepId == stepId)
+            {
+                break; // we are done - first in the list is caught up
+            }
+            else
+            {
+                if (nugget->errorStepId > 0)
+                { // don't include just-sent items in the count of old items
+                    ++numOldVisits;
+                }
+                nugget->computeDeltaError(focus, characterModel, stepId);
+            }
+
+            // add to group
+            NuggetIterator sendIter =
+                nuggetSendGroups[nugget->groupId].splice(nuggetSendGroups[nugget->groupId].iter, &nuggetSendGroups[oldGroup], *iter);
+
+            AYAASSERT(nuggetSendGroups[nugget->groupId].size() > 0
+                          ? nuggetSendGroups[nugget->groupId].iter != nuggetSendGroups[nugget->groupId].nuggetList.end()
+                          : true);
+            AYAASSERT(nuggetSendGroups[oldGroup].size() > 0 ? nuggetSendGroups[oldGroup].iter != nuggetSendGroups[oldGroup].nuggetList.end() : true);
+
+            *nugget->mapIter->second = sendIter;
+
+            iter++;
+        }
+    }
+}
+
+void ErrorCompPhysicsSender2::calculateSendCount()
+{
+    bool last = true;
+    int prev = nuggetSendGroups.size() - 1;
+    for (int i = nuggetSendGroups.size() - 1; i >= 0; i--)
+    {
+        if (nuggetSendGroups[i].size() == 0)
+        {
+            nuggetSendGroups[i].targetSentCount = 0;
+            nuggetSendGroups[i].sendCount = 0;
+            continue;
+        }
+        // last non empty bucket always send 1
+        if (last)
+        {
+            nuggetSendGroups[i].targetSentCount = 1;
+            nuggetSendGroups[i].sendCount = 0;
+            prev = i;
+            last = false;
+            continue;
+        }
+
+        // how many to send from this bucket before sending 1 item in next bucket, multiply by actual send count of next bucket count to keep ratio at
+        // 2:1
+        float count = nuggetSendGroups[i].size() * 2.0f / nuggetSendGroups[prev].size() * nuggetSendGroups[prev].targetSentCount;
+
+        nuggetSendGroups[i].targetSentCount = G3D::iCeil(count);
+
+        prev = i;
+    }
+}
+
+
+int ErrorCompPhysicsSender2::sendPacket(int maxPackets, PacketPriority packetPriority, ReplicatorStats::PhysicsSenderStats* stats)
+{
+    const unsigned int maxStreamSize(replicator.getPhysicsMtuSize());
+
+    boost::shared_ptr<RakNet::BitStream> bitStream;
+
+    int packetCount = 0;
+
+    bool done = false;
+
+    calculateSendCount();
+
+    // flag is a set of bits where each one representing a bucket, 1 == has stuff to send
+    int bucketFlag = (1 << nuggetSendGroups.size()) - 1;
+    while (!done && bucketFlag)
+    {
+        // reset bucket iterator
+        if (bucketIndex >= nuggetSendGroups.size())
+            bucketIndex = 0;
+
+        Bucket* curBucket = &nuggetSendGroups[bucketIndex];
+
+        if (curBucket->nuggetList.size() == 0)
+        {
+            bucketFlag &= ~(1 << bucketIndex);
+            bucketIndex++;
+            continue;
+        }
+        else if (curBucket->iter->sendStepId == stepId)
+        {
+            bucketFlag &= ~(1 << bucketIndex);
+            bucketIndex++;
+            continue;
+        }
+
+        AYAASSERT(curBucket->iter != curBucket->nuggetList.end());
+
+        while (curBucket->sendCount < curBucket->targetSentCount)
+        {
+            if (!bitStream)
+            {
+                if (packetCount >= maxPackets)
+                {
+                    done = true;
+                    break;
+                }
+
+                bitStream.reset(new RakNet::BitStream(maxStreamSize));
+
+                ++packetCount;
+
+                *bitStream << (unsigned char)ID_TIMESTAMP;
+
+                RakNet::Time timestamp = RakNet::GetTime();
+                if (curBucket->iter->part->computeNetworkOwnerIsSomeoneElse())
+                {
+                    // retrieve the original packet send time of the part
+                    AYAASSERT(timestamp >= curBucket->iter->part->raknetTime);
+                    if (curBucket->iter->part->raknetTime)
+                        timestamp = curBucket->iter->part->raknetTime;
+                }
+
+#if !defined(__linux) && !defined(__APPLE__)
+                *bitStream << timestamp;
+#else
+                *bitStream << static_cast<unsigned long long>(timestamp);
+#endif
+                *bitStream << (unsigned char)ID_PHYSICS;
+            }
+
+            AYAASSERT(curBucket->iter != curBucket->nuggetList.end());
+
+            // remove nugget if part is not in physics service sender list, it's not moving
+            if (!curBucket->iter->part->PhysicsServiceHook::is_linked())
+            {
+                removeNugget(curBucket->iter->part);
+
+                if (curBucket->size() == 0)
+                    break;
+            }
+            else
+            {
+                AYAASSERT(curBucket->iter->sendStepId <= stepId);
+                writeNugget(*bitStream, *curBucket->iter);
+
+                if (bitStream->GetNumberOfBytesUsed() > static_cast<int>(maxStreamSize))
+                {
+                    // Packet "end" tag
+                    if (replicator.isStreamingEnabled())
+                        *bitStream << true; // done
+                    else
+                        replicator.serializeId(*bitStream, NULL);
+
+                    // Send ID_PHYSICS
+                    replicator.rakPeer->Send(bitStream, packetPriority, UNRELIABLE, PHYSICS_CHANNEL, replicator.remotePlayerId, false);
+                    stats->physicsPacketsSent.sample();
+                    stats->physicsPacketsSentSmooth.sample();
+                    stats->physicsPacketsSentSize.sample(bitStream->GetNumberOfBytesUsed());
+
+                    bitStream.reset();
+                }
+            }
+
+            curBucket->sendCount++;
+            curBucket->iter++;
+
+            // reset iterator for next time in loop
+            if (curBucket->iter == curBucket->nuggetList.end())
+            {
+                curBucket->iter = curBucket->nuggetList.begin();
+                break;
+            }
+
+            AYAASSERT(curBucket->iter != curBucket->nuggetList.end());
+        }
+
+        if (curBucket->sendCount >= curBucket->targetSentCount)
+            curBucket->sendCount = 0;
+
+        bucketIndex++;
+    }
+
+    if (bitStream)
+    {
+        // Packet "end" tag
+        if (replicator.isStreamingEnabled())
+            *bitStream << true; // done
+        else
+            replicator.serializeId(*bitStream, NULL);
+
+        // Send ID_PHYSICS
+        replicator.rakPeer->Send(bitStream, packetPriority, UNRELIABLE, PHYSICS_CHANNEL, replicator.remotePlayerId, false);
+        stats->physicsPacketsSent.sample();
+        stats->physicsPacketsSentSmooth.sample();
+        stats->physicsPacketsSentSize.sample(bitStream->GetNumberOfBytesUsed());
+    }
+
+    return packetCount;
+};
+
+bool ErrorCompPhysicsSender2::writeNugget(RakNet::BitStream& bitStream, Nugget& nugget)
+{
+    if (!sendPhysicsData(bitStream, nugget))
+    {
+        nugget.sendStepId = stepId;
+        return false;
+    }
+
+    if (nugget.part->getConstPartPrimitive()->getConstAssembly())
+    {
+        nugget.onSent(stepId);
+        AYAASSERT(nugget.biggestSize == Nugget::minSize());
+        AYAASSERT(nugget.errorStepId == 0);
+
+        // move just sent item to front of update list
+        nuggetUpdateList.splice(nuggetUpdateList.begin(), nuggetUpdateList, nugget.mapIter->second);
+
+        // update iterators in map
+        nugget.mapIter->second = nuggetUpdateList.begin();
+
+        return true;
+    }
+    else
+        AYAASSERT(0);
+
+    return false;
+}
+
+void ErrorCompPhysicsSender2::onAddingAssembly(shared_ptr<Instance> assembly)
+{
+    shared_ptr<PartInstance> part = Instance::fastSharedDynamicCast<PartInstance>(assembly);
+    AYAASSERT(part);
+
+    newMovingAssemblies.push_back(part);
+}
+
+void ErrorCompPhysicsSender2::addNugget(PartInstance& part)
+{
+    addNugget2(shared_from(&part));
+}
+
+void ErrorCompPhysicsSender2::addNugget2(shared_ptr<PartInstance> part)
+{
+    // try add item to map
+    std::pair<NuggetMap::iterator, bool> result = nuggetMap.insert(std::make_pair(part, nuggetUpdateList.end()));
+
+    // add succeeded
+    if (result.second)
+    {
+        int group = 0;
+        NuggetIterator iter = nuggetSendGroups[group].push_back(part);
+        iter->groupId = group;
+
+        // add to send iterator to front of update list, so the nugget will be updated in step()
+        nuggetUpdateList.push_front(iter);
+
+        // set update iterator inside map
+        result.first->second = nuggetUpdateList.begin();
+
+        // cache map iterator inside nugget
+        iter->mapIter = result.first;
+    }
+}
+
+void ErrorCompPhysicsSender2::removeNugget(shared_ptr<const PartInstance> part)
+{
+    NuggetMap::iterator iter = nuggetMap.find(part);
+    AYAASSERT(iter != nuggetMap.end());
+
+    NuggetIterator nuggetIter = *iter->second;
+
+    int groupID = nuggetIter->groupId;
+
+    nuggetSendGroups[groupID].erase(nuggetIter);
+
+    nuggetUpdateList.erase(iter->second);
+
+    nuggetMap.erase(iter);
+}
+
+void ErrorCompPhysicsSender2::onRemovedAssembly(shared_ptr<Instance> assembly)
+{
+    shared_ptr<const PartInstance> part = Instance::fastSharedDynamicCast<PartInstance>(assembly);
+    AYAASSERT(part);
+
+    NuggetMap::iterator iter = nuggetMap.find(part);
+    AYAASSERT(iter != nuggetMap.end());
+
+    NuggetIterator nuggetIter = *iter->second;
+
+    int groupID = nuggetIter->groupId;
+
+    nuggetSendGroups[groupID].erase(nuggetIter);
+
+    nuggetUpdateList.erase(iter->second);
+
+    nuggetMap.erase(iter);
+}
+
+bool ErrorCompPhysicsSender2::sendPhysicsData(RakNet::BitStream& bitStream, const Nugget& nugget)
+{
+    sendDetailed = nugget.sendDetailed;
+    CoordinateFrame cFrame = nugget.part->getCoordinateFrame();
+    float accumulatedError;
+    return PhysicsSender::sendPhysicsData(bitStream, nugget.part.get(), sendDetailed, currentStepTimestamp, cFrame, accumulatedError, NULL);
+}
+
+void ErrorCompPhysicsSender2::writeAssembly(
+    RakNet::BitStream& bitStream, const Assembly* assembly, Compressor::CompressionType compressionType, bool crossPacketCompression)
+{
+    if (physicsPacketCache)
+    {
+        // look for this packet in the cache
+        if (!physicsPacketCache->fetchIfUpToDate(assembly, (unsigned char)sendDetailed, bitStream))
+        {
+            // cache miss, recreate bit stream and update cache
+            RakNet::BitStream stream;
+
+            unsigned int startBit = bitStream.GetWriteOffset();
+
+            PhysicsSender::writeAssembly(bitStream, assembly, compressionType);
+
+            bitStream.SetReadOffset(startBit);
+
+            if (!physicsPacketCache->update(assembly, (unsigned char)sendDetailed, bitStream, startBit, bitStream.GetNumberOfBitsUsed() - startBit))
+            {
+                // this item should be in the cache
+                StandardOut::singleton()->print(MESSAGE_INFO, "cache update failed");
+                AYAASSERT(0);
+            }
+
+            stream.SetReadOffset(0);
+            bitStream.Write(stream);
+        }
+    }
+    else
+        PhysicsSender::writeAssembly(bitStream, assembly, compressionType);
+}
+
+ErrorCompPhysicsSender2::NuggetIterator ErrorCompPhysicsSender2::Bucket::push_back(shared_ptr<PartInstance> part)
+{
+    nuggetList.push_back(ErrorCompPhysicsSender2::Nugget(part));
+
+    // set iterator at start of list if 1st item
+    if (nuggetList.size() == 1)
+        iter = nuggetList.begin();
+
+    return --nuggetList.end();
+}
+
+ErrorCompPhysicsSender2::NuggetIterator ErrorCompPhysicsSender2::Bucket::splice(const NuggetIterator where, Bucket* from, const NuggetIterator first)
+{
+    if (this == from && where == first)
+        return where;
+
+    // splicing within self does not cause iterators to become invalid
+    if (this != from && from->iter == first)
+    {
+        from->iter++;
+
+        if (from->iter == from->nuggetList.end())
+            from->iter = from->nuggetList.begin();
+    }
+
+    if (nuggetList.size() > 0)
+    {
+        nuggetList.splice(where, from->nuggetList, first);
+        NuggetIterator temp = where;
+        return --temp;
+    }
+    else
+    {
+        nuggetList.splice(nuggetList.begin(), from->nuggetList, first);
+        iter = nuggetList.begin();
+        return nuggetList.begin();
+    }
+}
+
+void ErrorCompPhysicsSender2::Bucket::erase(const NuggetIterator where)
+{
+    if (iter == where)
+    {
+        iter = nuggetList.erase(where);
+        if (iter == nuggetList.end())
+            iter = nuggetList.begin();
+    }
+    else
+        nuggetList.erase(where);
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+float ErrorCompPhysicsSender2::Nugget::minDistance()
+{
+    return 3.0f;
+}
+float ErrorCompPhysicsSender2::Nugget::maxDistance()
+{
+    return 800.0f;
+}
+float ErrorCompPhysicsSender2::Nugget::minSize()
+{
+    return 2.0f;
+}
+float ErrorCompPhysicsSender2::Nugget::maxSize()
+{
+    return 25.0f;
+}
+float ErrorCompPhysicsSender2::Nugget::midArea()
+{
+    return 312.5f;
+}
+
+ErrorCompPhysicsSender2::Nugget::Nugget(shared_ptr<PartInstance> part)
+    : part(part) // indexes
+    , errorStepId(-1)
+    , sendStepId(-1)
+    , biggestSize(minSize())
+    , sendDetailed(true)
+    , groupId(NUM_GROUPS - 1)
+{
+}
+
+ErrorCompPhysicsSender2::Nugget::~Nugget() {}
+
+// static
+void ErrorCompPhysicsSender2::Nugget::onSent(Nugget& nugget)
+{
+    // part - no change
+    nugget.errorStepId = 0;
+    nugget.biggestSize = minSize(); // resets
+}
+
+void ErrorCompPhysicsSender2::Nugget::onSent(int stepId)
+{
+    // part - no change
+    errorStepId = 0;
+    sendStepId = stepId;
+    biggestSize = minSize(); // resets
+}
+
+void ErrorCompPhysicsSender2::Nugget::computeDeltaError(const CoordinateFrame& focus, const ModelInstance* focusModel, int stepId)
+{
+    const Assembly* assembly = part->getConstPartPrimitive()->getConstAssembly();
+    AYAASSERT(assembly);
+
+    float radius = assembly->getLastComputedRadius();
+
+    // 1. Biggest Size
+    biggestSize = G3D::clamp(2 * radius, biggestSize, maxSize());
+
+    // current
+    float currentDistance = minDistance(); // There is no center of interest.  Location doesn't affect error
+    if (focusModel)
+    {
+        currentDistance = Math::taxiCabMagnitude(part->getCoordinateFrame().translation - focus.translation) - radius;
+        currentDistance = G3D::clamp(currentDistance, minDistance(), maxDistance());
+    }
+
+    float velocityError;
+    float sizeError;
+
+    // 2. Error
+    // 3. Send Detailed
+    if (part->isDescendantOf(focusModel))
+    { // the character!
+        groupId = 0;
+        sendDetailed = true;
+    }
+    else
+    {
+        // which group this object belongs to is initially set by the distance to player, then modified by the velocity and size
+        float distance = currentDistance * NUM_GROUPS / maxDistance();
+
+        float lVel = G3D::clamp(part->getLinearVelocity().squaredMagnitude(), 0.0f, 10000.0f);
+        float rVel = G3D::clamp(part->getRotationalVelocity().squaredMagnitude(), 0.0f, 10000.0f);
+
+        velocityError = -1 + ((lVel / 5000.0f * 0.80f) + (rVel / 5000.0f * 0.25f));
+
+        // -1 + (0 to 2), 0 == small object, 2 == big object
+        sizeError = -1 + biggestSize * biggestSize / midArea();
+
+        groupId = G3D::iRound(distance - sizeError - velocityError);
+        groupId = G3D::clamp((double)groupId, 0, NUM_GROUPS - 1);
+
+        sendDetailed = currentDistance < 20.0f;
+    }
+
+    // 4. errorStepId
+    errorStepId = stepId;
+
+    // const_cast<PartInstance*>(part.get())->setColor3(Color::colorFromError(groupId));
+}
+#endif // removing deprecated senders

engine/network/src/ErrorCompPhysicsSender2.hpp

@@ -0,0 +1,139 @@
+#if 1 // removing deprecated senders, set to 0 when FastFlag RemoveUnusedPhysicsSenders is set to true and removed (delete this file)
+
+#pragma once
+
+#include "PhysicsSender.hpp"
+#include "BoostAppend.hpp"
+#include "DataModel/PartInstance.hpp"
+
+#include "signal.hpp"
+
+
+#include "boost/pool/pool_alloc.hpp"
+
+namespace Aya
+{
+
+class Instance;
+class PhysicsService;
+class ModelInstance;
+
+namespace Network
+{
+
+class Replicator;
+class PhysicsPacketCache;
+
+class ErrorCompPhysicsSender2 : public PhysicsSender
+{
+    class Nugget;
+
+    typedef std::list<Nugget> NuggetList;
+    typedef NuggetList::iterator NuggetIterator;
+    typedef std::list<NuggetIterator> NuggetUpdateList;
+    typedef boost::unordered_map<shared_ptr<const PartInstance>, NuggetUpdateList::iterator> NuggetMap;
+
+    class Nugget
+    {
+    public:
+        shared_ptr<const PartInstance> part;
+        int errorStepId;             // "stepId" of the last error calculation  (-1 means uninitialized. 0 means we just got sent and need updating)
+        int sendStepId;              // "stepId" of the last send
+        int groupId;                 // which bucket this item belong to
+        NuggetMap::iterator mapIter; // iterator into hash map, for fast update without having to do look up
+
+        Nugget(shared_ptr<PartInstance> part);
+        ~Nugget();
+
+        static void onSent(Nugget& nugget);
+        void onSent(int stepId);
+
+        void computeDeltaError(const CoordinateFrame& focus, const ModelInstance* focusModel, int timestamp);
+
+        bool sendDetailed; // Should this send detailed (uncompressed) physics info
+        float biggestSize; // May change for an articulated assembly....
+
+        static float minDistance();
+        static float maxDistance();
+        static float minSize();
+        static float maxSize();
+        static float midArea();
+    };
+
+    struct Bucket
+    {
+        NuggetIterator iter;
+        NuggetList nuggetList;
+
+        int targetSentCount;
+        int sendCount;
+
+        Bucket()
+            : targetSentCount(0)
+            , sendCount(0)
+        {
+        }
+
+        NuggetIterator push_back(shared_ptr<PartInstance> part);
+        void erase(const NuggetIterator where);
+
+        // moves a node from one bucket to another.
+        // if first == iter, iter gets moved to next node if not at end, otherwise it moves to beginning
+        NuggetIterator splice(const NuggetIterator where, Bucket* from, const NuggetIterator first);
+
+        inline unsigned int size()
+        {
+            return nuggetList.size();
+        }
+    };
+
+    typedef std::vector<Bucket> NuggetSendGroups;
+
+    NuggetMap nuggetMap;
+    NuggetUpdateList nuggetUpdateList;
+    NuggetSendGroups nuggetSendGroups;
+    unsigned int bucketIndex;
+
+    int stepId;
+    shared_ptr<PhysicsService> physicsService;
+    shared_ptr<PhysicsPacketCache> physicsPacketCache;
+
+    bool sendDetailed;
+
+    Aya::signals::scoped_connection addingAssemblyConnection;
+    Aya::signals::scoped_connection removedAssemblyConnection;
+
+    typedef std::list<shared_ptr<PartInstance>, boost::fast_pool_allocator<shared_ptr<PartInstance>>> NewMovingAssemblies;
+    NewMovingAssemblies newMovingAssemblies; // new moving assemblies for current step
+
+public:
+    ErrorCompPhysicsSender2(Replicator& replicator);
+    ~ErrorCompPhysicsSender2();
+
+    virtual void step();
+    virtual int sendPacket(int maxPackets, PacketPriority packetPriority, ReplicatorStats::PhysicsSenderStats* stats);
+
+protected:
+    virtual void writeAssembly(
+        RakNet::BitStream& bitStream, const Assembly* assembly, Compressor::CompressionType compressionType, bool crossPacketCompression = false);
+
+private:
+    void calculateSendCount();
+
+    bool sendPhysicsData(RakNet::BitStream& bitStream, const Nugget& nugget);
+
+    void addNugget(PartInstance& part);
+    void addNugget2(shared_ptr<PartInstance> part);
+    void removeNugget(shared_ptr<const PartInstance> part);
+
+    void onAddingAssembly(shared_ptr<Instance> assembly);
+    void onRemovedAssembly(shared_ptr<Instance> assembly);
+
+    bool writeNugget(RakNet::BitStream& bitStream, Nugget& nugget);
+};
+
+
+} // namespace Network
+} // namespace Aya
+
+#endif // removing deprecated senders

engine/network/src/GameConfigurer.cpp

@@ -0,0 +1,945 @@
+#include "DataModel/GameBasicSettings.hpp"
+#include "Players.hpp"
+
+#include "GameConfigurer.hpp"
+
+#include "DataModel/DataModel.hpp"
+
+#include "DataModel/UserInputService.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+#include "DataModel/PhysicsSettings.hpp"
+
+
+#include "DataModel/DebugSettings.hpp"
+
+#include "DataModel/ChangeHistory.hpp"
+
+#include "DataModel/InsertService.hpp"
+
+
+#include "DataModel/SocialService.hpp"
+
+#include "DataModel/GamePassService.hpp"
+
+#include "DataModel/MarketplaceService.hpp"
+
+#include "DataModel/TimerService.hpp"
+
+#include "DataModel/Visit.hpp"
+
+#include "DataModel/DebugSettings.hpp"
+
+#include "DataModel/GuiService.hpp"
+
+#include "DataModel/Value.hpp"
+
+#include "DataModel/PlayerGui.hpp"
+
+#include "DataModel/HttpService.hpp"
+
+#include "DataModel/Folder.hpp"
+
+#include "DataModel/ScreenGui.hpp"
+
+#include "DataModel/ContentProvider.hpp"
+
+#include "DataModel/MegaCluster.hpp"
+
+#include "DataModel/TeleportService.hpp"
+
+#include "DataModel/HttpRbxApiService.hpp"
+
+#include "Utility/AyaService.hpp"
+#include "Utility/Statistics.hpp"
+
+#include "Utility/RobloxServicesTools.hpp"
+#include "Script/script.hpp"
+#include "Script/ScriptContext.hpp"
+#include "Script/ModuleScript.hpp"
+#include "Script/CoreScript.hpp"
+#include "Utility/StandardOut.hpp"
+#include "Xml/WebParser.hpp"
+#include "Utility/ScriptInformationProvider.hpp"
+
+
+#include "Client.hpp"
+#include "ClientReplicator.hpp"
+#include "Marker.hpp"
+#include "GamePerfMonitor.hpp"
+#include "Players.hpp"
+
+#include "Utility/rbxrandom.hpp"
+
+#include <vector>
+
+#include "StringConv.hpp"
+
+#include "Script/LuaVM.hpp"
+
+#include "boost/tokenizer.hpp"
+#include "boost/algorithm/string.hpp"
+#include "XStudioBuild.hpp"
+
+FASTSTRINGVARIABLE(SocialServiceFriendUrl, "Game/LuaWebService/HandleSocialRequest.ashx?method=IsFriendsWith&playerid=%d&userid=%d")
+FASTSTRINGVARIABLE(SocialServiceBestFriendUrl, "Game/LuaWebService/HandleSocialRequest.ashx?method=IsBestFriendsWith&playerid=%d&userid=%d")
+FASTSTRINGVARIABLE(SocialServiceGroupUrl, "Game/LuaWebService/HandleSocialRequest.ashx?method=IsInGroup&playerid=%d&groupid=%d")
+FASTSTRINGVARIABLE(SocialServiceGroupRankUrl, "Game/LuaWebService/HandleSocialRequest.ashx?method=GetGroupRank&playerid=%d&groupid=%d")
+FASTSTRINGVARIABLE(SocialServiceGroupRoleUrl, "Game/LuaWebService/HandleSocialRequest.ashx?method=GetGroupRole&playerid=%d&groupid=%d")
+FASTSTRINGVARIABLE(GamePassServicePlayerHasPassUrl, "Game/GamePass/GamePassHandler.ashx?Action=HasPass&UserID=%d&PassID=%d")
+FASTSTRINGVARIABLE(MobileJoinRateFormatUrl, "Game/JoinRate.ashx?st=%d&i=%d&p=%d&c=%s&r=%s&d=%d&b=%d&platform=%s")
+FASTFLAG(UseBuildGenericGameUrl)
+
+
+FASTFLAGVARIABLE(ClientABTestingEnabled, true)
+FASTFLAG(LegacyMessageBox)
+
+DYNAMIC_FASTFLAG(UseR15Character)
+
+using namespace Aya;
+
+void GameConfigurer::parseArgs(const std::string& args)
+{
+    parameters = Aya::make_shared<const Reflection::ValueTable>();
+    WebParser::parseJSONTable(args, parameters);
+}
+
+int GameConfigurer::getParamInt(const std::string& key)
+{
+    Aya::Reflection::ValueTable::const_iterator iter = parameters->find(key);
+    if (iter != parameters->end())
+        return iter->second.get<int>();
+
+    return 0;
+}
+
+std::string GameConfigurer::getParamString(const std::string& key)
+{
+    Aya::Reflection::ValueTable::const_iterator iter = parameters->find(key);
+    if (iter != parameters->end())
+        return iter->second.get<std::string>();
+
+    return std::string();
+}
+
+bool GameConfigurer::getParamBool(const std::string& key)
+{
+    Aya::Reflection::ValueTable::const_iterator iter = parameters->find(key);
+    if (iter != parameters->end())
+        return iter->second.get<bool>();
+
+    return false;
+}
+
+void GameConfigurer::registerPlay(const std::string& key, int userId, int placeId)
+{
+    if (!getParamBool("CookieStoreEnabled"))
+        return;
+}
+
+void GameConfigurer::setupUrls()
+{
+    std::string baseUrl = getParamString("BaseUrl");
+    dataModel->create<InsertService>();
+
+    if (FFlag::UseBuildGenericGameUrl)
+    {
+        SocialService* socialService = dataModel->create<SocialService>();
+        socialService->setFriendUrl(BuildGenericGameUrl(baseUrl, FString::SocialServiceFriendUrl));
+        socialService->setBestFriendUrl(BuildGenericGameUrl(baseUrl, FString::SocialServiceBestFriendUrl));
+        socialService->setGroupUrl(BuildGenericGameUrl(baseUrl, FString::SocialServiceGroupUrl));
+        socialService->setGroupRankUrl(BuildGenericGameUrl(baseUrl, FString::SocialServiceGroupRankUrl));
+        socialService->setGroupRoleUrl(BuildGenericGameUrl(baseUrl, FString::SocialServiceGroupRoleUrl));
+        dataModel->create<GamePassService>()->setPlayerHasPassUrl(BuildGenericGameUrl(baseUrl, FString::GamePassServicePlayerHasPassUrl));
+    }
+    else
+    {
+        SocialService* socialService = dataModel->create<SocialService>();
+        socialService->setFriendUrl(baseUrl + FString::SocialServiceFriendUrl);
+        socialService->setBestFriendUrl(baseUrl + FString::SocialServiceBestFriendUrl);
+        socialService->setGroupUrl(baseUrl + FString::SocialServiceGroupUrl);
+        socialService->setGroupRankUrl(baseUrl + FString::SocialServiceGroupRankUrl);
+        socialService->setGroupRoleUrl(baseUrl + FString::SocialServiceGroupRoleUrl);
+        dataModel->create<GamePassService>()->setPlayerHasPassUrl(baseUrl + FString::GamePassServicePlayerHasPassUrl);
+    }
+}
+
+PlayerConfigurer::PlayerConfigurer()
+    : testing(true)
+    , isTouchDevice(false)
+    , connectResolved(false)
+    , connectionFailed(false)
+    , loadResolved(false)
+    , joinResolved(false)
+    , playResolved(true)
+    , waitingForCharacter(false)
+    , startTime(0)
+    , playStartTime(0)
+    , launchMode(-1)
+{
+}
+
+PlayerConfigurer::~PlayerConfigurer()
+{
+    for (auto& c : connections)
+        c.disconnect();
+}
+
+void PlayerConfigurer::ifSeleniumThenSetCookie(const std::string& key, const std::string& value) {}
+
+void PlayerConfigurer::showErrorWindow(const std::string& message, const std::string& errorType, const std::string& errorCategory)
+{
+    if (GuiService* gs = dataModel->create<GuiService>())
+    {
+        if (errorType != "Kick" || gs->getErrorMessage() == "")
+            gs->setErrorMessage(message);
+    }
+
+    dataModel->setUiMessage(message);
+}
+
+static void ignoreResponse(std::string*, std::exception*) {}
+
+void PlayerConfigurer::reportError(const std::string& error, const std::string& msg)
+{
+    StandardOut::singleton()->printf(MESSAGE_INFO, "***ERROR*** %s %s", error.c_str(), msg.c_str());
+    if (!testing)
+    {
+        Aya::Visit* visit = dataModel->create<Aya::Visit>();
+        visit->setUploadUrl("");
+    }
+
+    Network::Client* client = dataModel->create<Network::Client>();
+    client->disconnect();
+
+    if (TimerService* timer = dataModel->create<TimerService>())
+    {
+        std::string errorMsg = Aya::format("Error: %s", error.c_str());
+        timer->delay(boost::bind(&PlayerConfigurer::showErrorWindow, this, errorMsg, msg, "Other"), 4.0);
+    }
+}
+
+void PlayerConfigurer::reportCounter(const std::string& counterNamesCSV, bool blocking) {}
+
+void PlayerConfigurer::requestCharacter(shared_ptr<Network::Replicator> replicator, shared_ptr<bool> isWaiting)
+{
+    (*isWaiting) = false;
+
+    dataModel->setUiMessage("Requesting character");
+
+    if (!loadResolved)
+    {
+        loadResolved = true;
+        double duration = G3D::System::time() - startTime;
+    }
+
+    try
+    {
+        replicator->requestCharacter();
+        dataModel->setUiMessage("Waiting for character");
+        waitingForCharacter = true;
+    }
+    catch (Aya::base_exception& e)
+    {
+        reportError(e.what(), "W4C");
+    }
+}
+
+void periodicallyZoomExtents(weak_ptr<DataModel> dmWeak, shared_ptr<bool> isWaiting)
+{
+    shared_ptr<DataModel> dm = dmWeak.lock();
+
+    if (dm && *isWaiting)
+    {
+        dm->getWorkspace()->zoomToExtents();
+
+        dm->create<TimerService>()->delay(boost::bind(&periodicallyZoomExtents, dmWeak, isWaiting), 0.5);
+    }
+}
+
+void PlayerConfigurer::onGameClose()
+{
+
+    //
+}
+
+void PlayerConfigurer::onDisconnection(const std::string& peer, bool lostConnection)
+{
+    if (peer.length() > 0)
+    {
+        if (lostConnection)
+        {
+            if (!connectionFailed)
+                showErrorWindow("You have lost the connection to the game", "LostConnection", "LostConnection");
+        }
+        else
+        {
+            if (!connectionFailed)
+            {
+                showErrorWindow("This game has shut down", "Kick", "Kick");
+                if (dataModel)
+                {
+                    Network::Players* players = dataModel->find<Network::Players>();
+                    if (players)
+                    {
+                        Network::Player* p = players->getLocalPlayer();
+                        if (p)
+                        {
+                            Instance* ps = p->findFirstChildByName("PlayerScripts");
+                            if (ps)
+                                ps->destroy();
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    try
+    {
+        std::string url = Aya::format("%s&disconnect=true", getParamString("PingUrl").c_str());
+#if defined(AYA_PLATFORM_DURANGO)
+        HttpAsync::get(url);
+#else
+        dataModel->httpGet(url, true);
+#endif
+    }
+    catch (Aya::base_exception&)
+    {
+        // don't care
+    }
+}
+
+void PlayerConfigurer::onConnectionAccepted(std::string url, shared_ptr<Instance> replicator)
+{
+    connectResolved = true;
+
+    shared_ptr<bool> waitingForMarker(new bool(true));
+
+    try
+    {
+        if (!testing)
+        {
+            Aya::Visit* visit = dataModel->create<Aya::Visit>();
+            visit->setPing(getParamString("PingUrl"), getParamInt("PingInterval"));
+        }
+
+        if (!getParamBool("GenerateTeleportJoin"))
+            dataModel->setUiMessageBrickCount();
+        else
+            dataModel->setUiMessage("Teleporting...");
+
+        boost::shared_ptr<Network::ClientReplicator> rep = Instance::fastSharedDynamicCast<Network::ClientReplicator>(replicator);
+
+        connections.push_back(rep->disconnectionSignal.connect(boost::bind(&PlayerConfigurer::onDisconnection, this, _1, _2)));
+        connections.push_back(rep->receivedGlobalsSignal.connect(boost::bind(&PlayerConfigurer::onReceivedGlobals, this)));
+
+        connections.push_back(rep->gameLoadedSignal.connect(boost::bind(&PlayerConfigurer::onGameLoaded, this, waitingForMarker)));
+    }
+    catch (Aya::base_exception& e)
+    {
+        StandardOut::singleton()->printf(Aya::MESSAGE_ERROR, "onConnectionAccepted failed: %s", e.what());
+        reportError(e.what(), "ConnectionAccepted");
+    }
+
+    dataModel->create<TimerService>()->delay(boost::bind(&periodicallyZoomExtents, weak_from(dataModel), waitingForMarker), 0);
+}
+
+void PlayerConfigurer::onConnectionFailed(const std::string& remoteAddress, int errorCode, const std::string& errorMsg)
+{
+    connectionFailed = true;
+    std::string msg;
+    if (FFlag::LegacyMessageBox)
+    {
+        msg = Aya::format("Failed to connect to the Game. (ID=%d)", errorCode);
+    }
+    else
+    {
+        msg = Aya::format("Failed to connect to the Game. (ID = %d: %s)", errorCode, errorMsg.c_str());
+    }
+
+    std::string errorType = Aya::format("ID%d", errorCode);
+    showErrorWindow(msg, errorType, "Other");
+}
+
+void PlayerConfigurer::onConnectionRejected()
+{
+    showErrorWindow("This game is not available. Please try another", "WrongVersion", "WrongVersion");
+}
+
+void PlayerConfigurer::onReceivedGlobals() {}
+
+void PlayerConfigurer::onGameLoaded(boost::shared_ptr<bool> isWaiting)
+{
+    *isWaiting = false;
+
+    if (!loadResolved)
+    {
+        loadResolved = true;
+        double duration = G3D::System::time() - startTime;
+    }
+
+    try
+    {
+        dataModel->setUiMessage("Waiting for character");
+        waitingForCharacter = true;
+
+        // see if our virtualversion is within bounds for the place
+        StarterPlayerService* sps = dataModel->find<StarterPlayerService>();
+        GameBasicSettings::VirtualVersion placeMinVersion = static_cast<GameBasicSettings::VirtualVersion>(sps->getMinVirtualVersion());
+        GameBasicSettings::VirtualVersion placeMaxVersion = static_cast<GameBasicSettings::VirtualVersion>(sps->getMaxVirtualVersion());
+        GameBasicSettings::VirtualVersion clientVersion = Aya::GameBasicSettings::singleton().getVirtualVersion();
+
+        if (clientVersion < placeMinVersion)
+        {
+            clientVersion = placeMinVersion;
+            Aya::GameBasicSettings::singleton().setVirtualVersionInternal(clientVersion);
+        }
+        else if (clientVersion > placeMaxVersion)
+        {
+            clientVersion = placeMaxVersion;
+            Aya::GameBasicSettings::singleton().setVirtualVersionInternal(clientVersion);
+        }
+
+        if (Network::Players::frontendProcessing(dataModel))
+        {
+            std::string starterScript = "StarterScript";
+            if (ScriptContext* scriptContext = dataModel->create<ScriptContext>())
+            {
+                scriptContext->addCoreScriptLocal(starterScript, shared_ptr<Instance>());
+            }
+        }
+    }
+    catch (Aya::base_exception& e)
+    {
+        reportError(e.what(), "W4C");
+    }
+}
+
+void PlayerConfigurer::onPlayerIdled(double time)
+{
+    if (time > 1200)
+    {
+        showErrorWindow(Aya::format("You were disconnected for being idle %d minutes", (int)(time / 60)), "Idle", "Idle");
+
+        if (Network::Client* client = dataModel->find<Network::Client>())
+            client->disconnect();
+    }
+}
+
+void PlayerConfigurer::setMessage(const std::string& msg)
+{
+    if (!getParamBool("GenerateTeleportJoin"))
+        dataModel->setUiMessage(msg);
+    else
+        dataModel->setUiMessage("Teleporting...");
+}
+
+void PlayerConfigurer::configure(Aya::Security::Identities identity, DataModel* dm, const std::string& args, int lm)
+{
+    startTime = G3D::System::time();
+
+    dataModel = dm;
+    launchMode = lm;
+
+    // Client-side A/B testing setup
+    // urls to try:
+    // https://api.gametest1.robloxlabs.com/users/get-experiment-enrollments
+    // https://api.gametest1.robloxlabs.com/users/get-studio-experiment-enrollments
+
+    std::string baseUrl = Aya::ContentProvider::getUnsecureApiBaseUrl(GetBaseURL());
+
+    // begin fetching now
+    Aya::HttpFuture abTest1, abTest2;
+    if (FFlag::ClientABTestingEnabled)
+    {
+        abTest1 = FetchABTestDataAsync(baseUrl + "users/get-experiment-enrollments");
+        abTest2 = FetchABTestDataAsync(baseUrl + "users/get-studio-experiment-enrollments");
+    }
+
+    Aya::Security::Impersonator impersonate(identity);
+
+    parseArgs(args);
+
+    // virtual version
+    int virtualVersion = getParamInt("VirtualVersion");
+    GameBasicSettings::VirtualVersion vv = static_cast<GameBasicSettings::VirtualVersion>(virtualVersion);
+    dataModel->setInitialVersion(vv);
+
+    Aya::GameBasicSettings::singleton().setVirtualVersionInternal(vv);
+
+    Http::gameID = getParamString("GameId");
+
+    testing = false; // (getParamString("ClientTicket").length() == 0);
+
+    if (DFFlag::UseR15Character)
+        dataModel->create<Network::Client>();
+
+    dataModel->setPlaceID(getParamInt("PlaceId"), getParamBool("IsRobloxPlace"));
+    int universeId = getParamInt("UniverseId");
+    dataModel->setUniverseId(universeId);
+    dataModel->create<HttpService>();
+    isTouchDevice = dm->create<UserInputService>()->getTouchEnabled();
+
+    StandardOut::singleton()->printf(MESSAGE_SENSITIVE, "! Joining game '%s' place %d at %s", getParamString("GameId").c_str(),
+        getParamInt("PlaceId"), getParamString("MachineAddress").c_str());
+
+    connections.push_back(dataModel->closingSignal.connect(boost::bind(&PlayerConfigurer::onGameClose, this)));
+
+    dataModel->create<ChangeHistoryService>()->setEnabled(false);
+    dataModel->create<ContentProvider>()->setThreadPool(16);
+
+    setupUrls();
+
+    DataModel::CreatorType creatorType;
+    Reflection::EnumDesc<DataModel::CreatorType>::singleton().convertToValue(getParamString("CreatorTypeEnum").c_str(), creatorType);
+    dataModel->setCreatorID(getParamInt("CreatorId"), creatorType);
+
+    Network::Players* players = dataModel->create<Network::Players>();
+    Network::Players::ChatOption chatOption;
+    Reflection::EnumDesc<Network::Players::ChatOption>::singleton().convertToValue(getParamString("ChatStyle").c_str(), chatOption);
+    players->setChatOption(chatOption);
+
+    if (!DFFlag::UseR15Character)
+        dataModel->create<Network::Client>();
+    dataModel->create<Visit>();
+
+    ifSeleniumThenSetCookie("SeleniumTest1", "Started join script");
+
+    try
+    {
+        setMessage("Connecting to Server");
+
+        Network::Client* client = dm->create<Network::Client>();
+
+        connections.push_back(client->connectionAcceptedSignal.connect(boost::bind(&PlayerConfigurer::onConnectionAccepted, this, _1, _2)));
+        connections.push_back(client->connectionRejectedSignal.connect(boost::bind(&PlayerConfigurer::onConnectionRejected, this)));
+        connections.push_back(client->connectionFailedSignal.connect(boost::bind(&PlayerConfigurer::onConnectionFailed, this, _1, _2, _3)));
+
+        client->setTicket(getParamString("ClientTicket"));
+
+        ifSeleniumThenSetCookie("SeleniumTest2", "Successfully connected to server");
+
+        client->setGameSessionID(getParamString("SessionId"));
+
+        shared_ptr<Network::Player> player = Instance::fastSharedDynamicCast<Network::Player>(
+            client->playerConnect(getParamInt("UserId"), getParamString("MachineAddress"), getParamInt("ServerPort"), getParamInt("ClientPort"), -1));
+
+        // prepare callback for when the Character appears
+        playerChangedConnection = player->propertyChangedSignal.connect(boost::bind(&PlayerConfigurer::onPlayerChanged, this, _1));
+
+        registerPlay(getParamString("CookieStoreFirstTimePlayKey"), getParamInt("UserId"), getParamInt("PlaceId"));
+        if (TimerService* timer = dataModel->create<TimerService>())
+        {
+            timer->delay(boost::bind(&PlayerConfigurer::registerPlay, this, getParamString("CookieStoreFiveMinutePlayKey"), getParamInt("UserId"),
+                             getParamInt("PlaceId")),
+                60 * 5.0);
+        }
+
+        player->setSuperSafeChat(getParamBool("SuperSafeChat"));
+        player->setUnder13(getParamBool("IsUnknownOrUnder13"));
+        Network::Player::MembershipType membershipType;
+        Reflection::EnumDesc<Network::Player::MembershipType>::singleton().convertToValue(getParamString("MembershipType").c_str(), membershipType);
+        player->setMembershipType(membershipType);
+        player->setAccountAge(getParamInt("AccountAge"));
+
+        connections.push_back(player->idledSignal.connect(boost::bind(&PlayerConfigurer::onPlayerIdled, this, _1)));
+
+
+        try
+        {
+            player->setName(getParamString("UserName"));
+        }
+        catch (Aya::base_exception&)
+        {
+            // don't care, happens when called from studio cmd bar
+        }
+
+        // I don't know why UserName needs to do this, so I'm copying this behavior for DisplayName -sorket
+        try
+        {
+            player->setDisplayName(getParamString("DisplayName"));
+        }
+        catch (Aya::base_exception&)
+        {
+            // don't care, happens when called from studio cmd bar
+        }
+
+        player->setCharacterAppearance(getParamString("CharacterAppearance"));
+        player->setFollowUserId(getParamInt("FollowUserId"));
+
+        if (!testing)
+        {
+            Aya::Visit* visit = dataModel->create<Aya::Visit>();
+            visit->setUploadUrl("");
+        }
+    }
+    catch (Aya::base_exception& e)
+    {
+        reportError(e.what(), "CreatePlayer");
+    }
+
+
+    if (!testing)
+    {
+        gamePerfMonitor.reset(
+            new GamePerfMonitor(getParamString("BaseUrl"), getParamString("GameId"), getParamInt("PlaceId"), getParamInt("UserId")));
+        gamePerfMonitor->start(dataModel);
+    }
+
+
+    if (FFlag::ClientABTestingEnabled)
+    {
+        try
+        {
+            LoadABTestFromString(abTest1.get());
+        }
+        catch (const std::exception& e1)
+        {
+            try
+            {
+                LoadABTestFromString(abTest2.get());
+            }
+            catch (const std::exception& e2)
+            {
+                FASTLOG2(FLog::Error, "Failed to load AB test data from both URLS: [%s] [%s]", e1.what(), e2.what());
+            }
+        }
+    }
+}
+
+void PlayerConfigurer::onPlayerChanged(const Reflection::PropertyDescriptor* propertyDescriptor)
+{
+    if (propertyDescriptor->name == "Character")
+    {
+        dataModel->clearUiMessage();
+        waitingForCharacter = false;
+
+        playerChangedConnection.disconnect();
+
+        if (!joinResolved)
+        {
+            joinResolved = true;
+
+            MegaClusterInstance* megaCluster = Instance::fastDynamicCast<MegaClusterInstance>(dataModel->getWorkspace()->getTerrain());
+            bool hasTerrain = megaCluster && megaCluster->isAllocated();
+
+            playStartTime = G3D::System::time();
+            playResolved = false;
+        }
+
+        if (gamePerfMonitor)
+        {
+            // delay collecting network stats by 2 mins after character is resolved
+            if (TimerService* timer = dataModel->create<TimerService>())
+            {
+                timer->delay(boost::bind(&GamePerfMonitor::setPostDiagStats, gamePerfMonitor, true), 2 * 60);
+            }
+        }
+    }
+}
+
+bool isHidden(const boost::filesystem::path& p)
+{
+    boost::filesystem::path name = p.filename();
+    if (name != ".." && name != "." && name.c_str()[0] == '.')
+    {
+        return true;
+    }
+
+    return false;
+}
+
+
+bool StudioConfigurer::findModulesAndLoad(
+    const std::string& baseModulePath, const boost::filesystem::path& dir_path, boost::unordered_map<std::string, ProtectedString>& coreModules)
+{
+    if (!boost::filesystem::exists(dir_path))
+        return false;
+
+    boost::filesystem::directory_iterator end_itr; // default construction yields past-the-end
+    for (boost::filesystem::directory_iterator itr(dir_path); itr != end_itr; ++itr)
+    {
+        if (is_directory(itr->status()) && !isHidden(itr->path()))
+        {
+            if (findModulesAndLoad(baseModulePath, itr->path(), coreModules))
+                return true;
+        }
+
+        else if (is_regular_file(itr->status()))
+        {
+            boost::filesystem::path currentPath = itr->path();
+            std::string filePath = currentPath.string();
+
+            if (filePath.find(".lua") != std::string::npos)
+            {
+                size_t pathLocation = filePath.find(baseModulePath);
+                if (pathLocation != std::string::npos)
+                {
+                    filePath = filePath.substr(pathLocation + baseModulePath.length() + 1);
+                    filePath = filePath.substr(0, filePath.length() - 4);
+                }
+
+                // load module
+                if (!filePath.empty())
+                {
+                    ProtectedString source = Aya::CoreScript::fetchSource("/Modules/" + filePath).get();
+                    if (!source.empty())
+                    {
+                        coreModules[filePath] = source;
+                    }
+                }
+            }
+        }
+    }
+    return false;
+}
+
+shared_ptr<Aya::Folder> moduleScriptFolder;
+
+void StudioConfigurer::loadCoreModules()
+{
+    Aya::CoreGuiService* coreGuiService = Aya::ServiceProvider::find<Aya::CoreGuiService>(dataModel);
+
+    if (!coreGuiService)
+    {
+        return;
+    }
+
+    shared_ptr<Aya::ScreenGui> robloxScreenGui = coreGuiService->getRobloxScreenGui();
+    if (!robloxScreenGui)
+    {
+        return;
+    }
+
+    moduleScriptFolder = Aya::Creatable<Instance>::create<Aya::Folder>();
+    moduleScriptFolder->setName("Modules");
+    moduleScriptFolder->setRobloxLocked(true);
+    moduleScriptFolder->setParent(Aya::Instance::fastDynamicCast<Aya::Instance>(robloxScreenGui.get()));
+
+    boost::unordered_map<std::string, ProtectedString> coreModules;
+
+    if (LuaVM::canCompileScripts())
+    {
+        const std::string path = Aya::BaseScript::hasCoreScriptReplacements() ? Aya::BaseScript::adminScriptsPath + "/Modules"
+                                                                              : ContentProvider::assetFolder() + "scripts/Modules";
+
+        boost::filesystem::path filePath(path);
+        if (!boost::filesystem::exists(filePath))
+        {
+            return;
+        }
+
+        findModulesAndLoad(filePath.string(), filePath, coreModules);
+    }
+    else
+    {
+        boost::unordered_map<std::string, std::string> byteCodeModules = LuaVM::getBytecodeCoreModules();
+        for (boost::unordered_map<std::string, std::string>::iterator iter = byteCodeModules.begin(); iter != byteCodeModules.end(); ++iter)
+        {
+            coreModules[Aya::rot13((*iter).first)] = ProtectedString::fromBytecode((*iter).second);
+        }
+    }
+
+    for (boost::unordered_map<std::string, ProtectedString>::iterator iter = coreModules.begin(); iter != coreModules.end(); ++iter)
+    {
+        shared_ptr<ModuleScript> moduleScript = Aya::Creatable<Instance>::create<ModuleScript>();
+
+        std::string name = (*iter).first;
+        shared_ptr<Aya::Folder> lastFolder = moduleScriptFolder;
+
+        boost::filesystem::path namePath(name);
+        if (namePath.has_parent_path())
+        {
+            const std::string delimiter = "\\";
+
+            while (name.find("/") != std::string::npos)
+            {
+                name.replace(name.find("/"), 1, delimiter);
+            }
+
+            size_t pos = 0;
+            int level = 0;
+            std::string token;
+            while ((pos = name.find(delimiter)) != std::string::npos)
+            {
+                token = name.substr(0, pos);
+                name.erase(0, pos + delimiter.length());
+
+                if (Instance* findFolder = moduleScriptFolder->findFirstChildByNameRecursive(token))
+                {
+                    if (Aya::Folder* folderInstance = Instance::fastDynamicCast<Aya::Folder>(findFolder))
+                    {
+                        lastFolder = shared_from(folderInstance);
+                    }
+                }
+                else
+                {
+                    shared_ptr<Aya::Folder> newFolder = Aya::Creatable<Instance>::create<Aya::Folder>();
+                    newFolder->setName(token);
+                    newFolder->setRobloxLocked(true);
+                    if (level == 0)
+                    {
+                        newFolder->setParent(moduleScriptFolder.get());
+                    }
+                    else
+                    {
+                        newFolder->setParent(lastFolder.get());
+                    }
+                    lastFolder = newFolder;
+                }
+                level++;
+            }
+        }
+
+        moduleScript->setName(name);
+        moduleScript->setSource((*iter).second);
+        moduleScript->setRobloxLocked(true);
+
+        moduleScript->setParent(lastFolder.get());
+    }
+}
+
+void StudioConfigurer::configure(Aya::Security::Identities identity, DataModel* dm, const std::string& args, int launchMode)
+{
+    dataModel = dm;
+    parseArgs(args);
+
+    std::string baseUrl = getParamString("BaseUrl");
+
+    dataModel->create<InsertService>();
+
+    setupUrls();
+
+    if (Network::Players::frontendProcessing(dataModel))
+    {
+        loadCoreModules();
+
+        // Really ugly hack since getting the datamodel is a pain in the fucking ass
+        Aya::GameBasicSettings::singleton().virtualVersionChangedSignal.connect(
+            [&](const GameBasicSettings::VirtualVersion version)
+            {
+                if (version == GameBasicSettings::VERSION_2016 && !moduleScriptFolder->getChildren2())
+                {
+                    boost::unordered_map<std::string, ProtectedString> coreModules;
+
+                    if (LuaVM::canCompileScripts())
+                    {
+                        const std::string path = Aya::BaseScript::hasCoreScriptReplacements() ? Aya::BaseScript::adminScriptsPath + "/Modules"
+                                                                                              : ContentProvider::assetFolder() + "scripts/Modules";
+
+                        boost::filesystem::path filePath(path);
+                        if (!boost::filesystem::exists(filePath))
+                        {
+                            return;
+                        }
+
+                        findModulesAndLoad(filePath.string(), filePath, coreModules);
+                    }
+                    else
+                    {
+                        boost::unordered_map<std::string, std::string> byteCodeModules = LuaVM::getBytecodeCoreModules();
+                        for (boost::unordered_map<std::string, std::string>::iterator iter = byteCodeModules.begin(); iter != byteCodeModules.end();
+                            ++iter)
+                        {
+                            coreModules[Aya::rot13((*iter).first)] = ProtectedString::fromBytecode((*iter).second);
+                        }
+                    }
+
+                    for (boost::unordered_map<std::string, ProtectedString>::iterator iter = coreModules.begin(); iter != coreModules.end(); ++iter)
+                    {
+                        shared_ptr<ModuleScript> moduleScript = Aya::Creatable<Instance>::create<ModuleScript>();
+
+                        std::string name = (*iter).first;
+                        shared_ptr<Aya::Folder> lastFolder = moduleScriptFolder;
+
+                        boost::filesystem::path namePath(name);
+                        if (namePath.has_parent_path())
+                        {
+                            const std::string delimiter = "\\";
+
+                            while (name.find("/") != std::string::npos)
+                            {
+                                name.replace(name.find("/"), 1, delimiter);
+                            }
+
+                            size_t pos = 0;
+                            int level = 0;
+                            std::string token;
+                            while ((pos = name.find(delimiter)) != std::string::npos)
+                            {
+                                token = name.substr(0, pos);
+                                name.erase(0, pos + delimiter.length());
+
+                                if (Instance* findFolder = moduleScriptFolder->findFirstChildByNameRecursive(token))
+                                {
+                                    if (Aya::Folder* folderInstance = Instance::fastDynamicCast<Aya::Folder>(findFolder))
+                                    {
+                                        lastFolder = shared_from(folderInstance);
+                                    }
+                                }
+                                else
+                                {
+                                    shared_ptr<Aya::Folder> newFolder = Aya::Creatable<Instance>::create<Aya::Folder>();
+                                    newFolder->setName(token);
+                                    newFolder->setRobloxLocked(true);
+                                    if (level == 0)
+                                    {
+                                        newFolder->setParent(moduleScriptFolder.get());
+                                    }
+                                    else
+                                    {
+                                        newFolder->setParent(lastFolder.get());
+                                    }
+                                    lastFolder = newFolder;
+                                }
+                                level++;
+                            }
+                        }
+
+                        moduleScript->setName(name);
+                        moduleScript->setSource((*iter).second);
+                        moduleScript->setRobloxLocked(true);
+                        Aya::StandardOut::singleton()->printf(MESSAGE_INFO, "loading %s", name.c_str());
+                        moduleScript->setParent(lastFolder.get());
+                    }
+                }
+            });
+    }
+
+#if defined(AYA_PLATFORM_DURANGO)
+    if (ScriptContext* scriptContext = dataModel->create<ScriptContext>())
+    {
+        if (starterScript.empty())
+            starterScript = "StarterScript";
+        scriptContext->addCoreScriptLocal(starterScript, shared_ptr<Instance>());
+        return;
+    }
+#endif
+
+#if defined(AYA_STUDIO) && ENABLE_XBOX_STUDIO_BUILD
+    if (ScriptContext* scriptContext = dataModel->create<ScriptContext>())
+    {
+        starterScript = "XStarterScript";
+        scriptContext->addCoreScriptLocal(starterScript, shared_ptr<Instance>());
+        return;
+    }
+#endif
+
+
+    // this will be called in case of old play solo
+    if (ScriptContext* scriptContext = dataModel->create<ScriptContext>())
+    {
+        AyaService* ayaService = ServiceProvider::create<AyaService>(dataModel);
+
+        {
+            starterScript = "StarterScript";
+
+            if (Network::Players::backendProcessing(dataModel))
+                starterScript = "ServerCoreScripts/" + starterScript;
+
+            scriptContext->addCoreScriptLocal(starterScript, shared_ptr<Instance>());
+        }
+    }
+}

engine/network/src/GameConfigurer.hpp

@@ -0,0 +1,121 @@
+#pragma once
+
+#include <string>
+#include <boost/scoped_ptr.hpp>
+#include "Reflection/Type.hpp"
+#include "Reflection/Property.hpp"
+#include "signal.hpp"
+
+#include "Security/SecurityContext.hpp"
+
+
+namespace boost
+{
+namespace filesystem
+{
+class path;
+}
+} // namespace boost
+
+
+namespace Aya
+{
+
+class DataModel;
+class Instance;
+class GamePerfMonitor;
+class ProtectedString;
+
+namespace Network
+{
+class Replicator;
+}
+
+class GameConfigurer
+{
+public:
+    GameConfigurer() {};
+    virtual ~GameConfigurer() {}
+
+protected:
+    DataModel* dataModel;
+    boost::shared_ptr<const Reflection::ValueTable> parameters;
+
+    void parseArgs(const std::string& args);
+    int getParamInt(const std::string& key);
+    std::string getParamString(const std::string& key);
+    bool getParamBool(const std::string& key);
+    void registerPlay(const std::string& key, int userId, int placeId);
+
+    void setupUrls();
+
+public:
+    virtual void configure(Aya::Security::Identities identity, DataModel* dm, const std::string& args, int launchMode = -1) = 0;
+};
+
+class PlayerConfigurer : public GameConfigurer
+{
+    bool testing;
+    bool isTouchDevice;
+    bool connectResolved;
+    bool connectionFailed;
+    bool loadResolved;
+    bool joinResolved;
+    bool playResolved;
+    G3D::RealTime startTime;
+    G3D::RealTime playStartTime;
+    bool waitingForCharacter;
+    int launchMode;
+
+    boost::shared_ptr<GamePerfMonitor> gamePerfMonitor;
+
+    Aya::signals::scoped_connection playerChangedConnection;
+
+    std::vector<Aya::signals::connection> connections;
+
+    void ifSeleniumThenSetCookie(const std::string& key, const std::string& value);
+    void showErrorWindow(const std::string& message, const std::string& errorType, const std::string& errorCategory);
+
+    void reportError(const std::string& error, const std::string& msg);
+    void reportCounter(const std::string& counterNamesCSV, bool blocking);
+    void reportStats(const std::string& category, float value);
+    void reportDuration(const std::string& category, const std::string& result, double duration, bool blocking);
+
+    void requestCharacter(boost::shared_ptr<Network::Replicator> replicator, boost::shared_ptr<bool> isWaiting);
+
+    void onGameClose();
+    void onDisconnection(const std::string& peer, bool lostConnection);
+    void onConnectionAccepted(std::string url, boost::shared_ptr<Instance> replicator);
+    void onConnectionFailed(const std::string& remoteAddress, int errorCode, const std::string& errorMsg);
+    void onConnectionRejected();
+    void onReceivedGlobals();
+    void onGameLoaded(boost::shared_ptr<bool> isWaiting);
+    void onPlayerIdled(double time);
+    void onPlayerChanged(const Reflection::PropertyDescriptor* propertyDescriptor);
+
+    void setMessage(const std::string& msg);
+
+public:
+    PlayerConfigurer();
+    ~PlayerConfigurer();
+
+    /*override*/ void configure(Aya::Security::Identities identity, DataModel* dm, const std::string& args, int launchMode = -1);
+};
+
+class StudioConfigurer : public GameConfigurer
+{
+private:
+    bool findModulesAndLoad(
+        const std::string& baseModulePath, const boost::filesystem::path& dir_path, boost::unordered_map<std::string, ProtectedString>& coreModules);
+    void loadCoreModules();
+
+public:
+    StudioConfigurer() {}
+    ~StudioConfigurer() {}
+
+    std::string starterScript;
+    /*override*/ void configure(Aya::Security::Identities identity, DataModel* dm, const std::string& args, int launchMode = -1);
+};
+
+
+} // namespace Aya

engine/network/src/GamePerfMonitor.cpp

@@ -0,0 +1,81 @@
+#include "GamePerfMonitor.hpp"
+
+#include "DataModel/DataModel.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+#include "DataModel/Stats.hpp"
+
+#include "DataModel/DebugSettings.hpp"
+
+#include "DataModel/TimerService.hpp"
+
+#include "World/World.hpp"
+#include "Utility/RunStateOwner.hpp"
+
+
+#include "Players.hpp"
+
+
+using namespace Aya;
+
+
+FASTINTVARIABLE(GamePerfMonitorPercentage, 2)
+FASTINTVARIABLE(GamePerfMonitorReportTimer, 10) // minutes
+
+GamePerfMonitor::~GamePerfMonitor() {}
+
+void GamePerfMonitor::collectAndPostStats(boost::weak_ptr<DataModel> dataModel)
+{
+    boost::shared_ptr<DataModel> sharedDM = dataModel.lock();
+    if (!sharedDM)
+        return;
+
+    if (sharedDM->isClosed())
+        return;
+
+    postDiagStats(sharedDM);
+
+    if (TimerService* timer = sharedDM->create<TimerService>())
+    {
+        timer->delay(boost::bind(&GamePerfMonitor::collectAndPostStats, this, dataModel), FInt::GamePerfMonitorReportTimer * 60);
+    }
+}
+
+void GamePerfMonitor::postDiagStats(boost::shared_ptr<DataModel> dataModel)
+{
+    //
+}
+
+void GamePerfMonitor::onGameClose(boost::weak_ptr<DataModel> dataModel)
+{
+    boost::shared_ptr<DataModel> sharedDM = dataModel.lock();
+    if (!sharedDM)
+        return;
+}
+
+void GamePerfMonitor::start(DataModel* dataModel)
+{
+    if (!dataModel)
+        return;
+
+    Stats::StatsService* statsService = dataModel->create<Stats::StatsService>();
+    if (statsService)
+        frmStats = shared_from(statsService->findFirstChildByName("FrameRateManager"));
+
+    bool shouldPost = (abs(userId) % 100) < FInt::GamePerfMonitorPercentage;
+    dataModel->setJobsExtendedStatsWindow(30);
+
+    if (shouldPost)
+    {
+        if (TimerService* timer = dataModel->create<TimerService>())
+        {
+            timer->delay(boost::bind(&GamePerfMonitor::collectAndPostStats, this, weak_from(dataModel)), 60);
+        }
+    }
+
+    if (shouldPost)
+    {
+        dataModel->closingSignal.connect(boost::bind(&GamePerfMonitor::onGameClose, this, weak_from(dataModel)));
+    }
+}

engine/network/src/GamePerfMonitor.hpp

@@ -0,0 +1,51 @@
+#pragma once
+
+#include "intrusive_ptr_target.hpp"
+
+#include <boost/shared_ptr.hpp>
+#include <boost/scoped_ptr.hpp>
+#include <boost/thread/thread.hpp>
+
+namespace Aya
+{
+
+class DataModel;
+class Instance;
+
+class GamePerfMonitor
+{
+    std::string baseUrl;
+    std::string jobId;
+    int placeId;
+    int userId;
+    bool shouldPostDiagStats;
+
+    boost::shared_ptr<Instance> frmStats; // keep a shared pointer here so we can still use it after RenderView shuts down
+
+    void collectAndPostStats(boost::weak_ptr<DataModel> dataModel);
+    void postDiagStats(boost::shared_ptr<DataModel> dataModel);
+
+    void onGameClose(boost::weak_ptr<DataModel> dataModel);
+
+
+public:
+    GamePerfMonitor(const std::string& baseUrl, const std::string& jobId, int placeId, int userId)
+        : baseUrl(baseUrl)
+        , jobId(jobId)
+        , placeId(placeId)
+        , userId(userId)
+        , shouldPostDiagStats(false) {};
+    ~GamePerfMonitor();
+
+    void start(DataModel* dataModel);
+
+    void setPostDiagStats(bool shouldPost)
+    {
+        shouldPostDiagStats = shouldPost;
+    }
+};
+
+
+
+
+} // namespace Aya

engine/network/src/GuidRegistryService.cpp

@@ -0,0 +1,10 @@
+#include "GuidRegistryService.hpp"
+
+const char* const Aya::Network::sGuidRegistryService = "GuidRegistryService";
+
+Aya::Network::GuidRegistryService::GuidRegistryService(void)
+    : registry(Registry::create())
+{
+}
+
+Aya::Network::GuidRegistryService::~GuidRegistryService(void) {}

engine/network/src/GuidRegistryService.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "Tree/Service.hpp"
+
+
+namespace Aya
+{
+
+namespace Network
+{
+
+extern const char* const sGuidRegistryService;
+class GuidRegistryService
+    : public DescribedNonCreatable<GuidRegistryService, Instance, sGuidRegistryService>
+    , public Service
+{
+public:
+    boost::intrusive_ptr<GuidItem<Instance>::Registry> const registry;
+    GuidRegistryService(void);
+    ~GuidRegistryService(void);
+};
+} // namespace Network
+} // namespace Aya

engine/network/src/InterpolatingPhysicsReceiver.cpp

@@ -0,0 +1,336 @@
+
+#if 1 // remove this file when fast flag RemoveInterpolationReciever is accepted and removed
+
+#include "InterpolatingPhysicsReceiver.hpp"
+#include "Compressor.hpp"
+#include "RakNet/GetTime.hpp"
+#include "RakNet/RakPeerInterface.hpp"
+#include "NetworkSettings.hpp"
+
+#include "Replicator.hpp"
+
+#include "Player.hpp"
+
+
+#include "Utility/ObscureValue.hpp"
+
+
+#include "DataModel/PartInstance.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+#include "DataModel/DataModel.hpp"
+
+
+#include "World/World.hpp"
+#include "World/Primitive.hpp"
+#include "World/Assembly.hpp"
+
+using namespace Aya;
+using namespace Aya::Network;
+
+FASTFLAG(RemoveInterpolationReciever)
+
+class InterpolatingPhysicsReceiver::Job : public ReplicatorJob
+{
+    weak_ptr<InterpolatingPhysicsReceiver> receiver;
+    ObscureValue<double> receiveRate;
+
+public:
+    Job(shared_ptr<InterpolatingPhysicsReceiver> receiver)
+        : ReplicatorJob("Net InterpolatePhysics", *(receiver->replicator), DataModelJob::PhysicsIn)
+        , receiver(receiver)
+        , receiveRate(replicator->settings().getReceiveRate())
+    {
+        cyclicExecutive = true;
+    }
+
+private:
+    virtual Time::Interval sleepTime(const Stats& stats)
+    {
+        return computeStandardSleepTime(stats, receiveRate);
+    }
+    virtual Error error(const Stats& stats)
+    {
+        return computeStandardError(stats, receiveRate);
+    }
+
+    virtual TaskScheduler::StepResult stepDataModelJob(const Stats& stats)
+    {
+        if (shared_ptr<InterpolatingPhysicsReceiver> safeReceiver = receiver.lock())
+        {
+            safeReceiver->step(RakNet::GetTimeMS());
+            return TaskScheduler::Stepped;
+        }
+        return TaskScheduler::Done;
+    }
+};
+
+static double lerpValue = 0.05;
+
+InterpolatingPhysicsReceiver::InterpolatingPhysicsReceiver(Replicator* replicator, bool isServer)
+    : PhysicsReceiver(replicator, isServer)
+    , outOfOrderMechanisms(lerpValue)
+{
+    if (FFlag::RemoveInterpolationReciever)
+    {
+        AYAASSERT(false); // nobody should be creating this
+    }
+}
+
+void InterpolatingPhysicsReceiver::start(shared_ptr<PhysicsReceiver> physicsReceiver)
+{
+    shared_ptr<InterpolatingPhysicsReceiver> interpolatingPhysicsReceiver =
+        boost::dynamic_pointer_cast<InterpolatingPhysicsReceiver>(physicsReceiver);
+    if (interpolatingPhysicsReceiver.get() != this)
+        AYACRASH();
+
+    tryToCreateJob(interpolatingPhysicsReceiver);
+    if (!job)
+        serviceProviderConnection = replicator->ancestryChangedSignal.connect(
+            boost::bind(&InterpolatingPhysicsReceiver::onAncestryChanged, this, interpolatingPhysicsReceiver));
+}
+
+void InterpolatingPhysicsReceiver::tryToCreateJob(shared_ptr<InterpolatingPhysicsReceiver> interpolatingPhysicsReceiver)
+{
+    AYAASSERT(interpolatingPhysicsReceiver.get() == this);
+    AYAASSERT(!job);
+
+    // We can't create the job until we're in the DataModel
+    if (!DataModel::get(replicator))
+        return;
+
+    job.reset(new Job(interpolatingPhysicsReceiver));
+    TaskScheduler::singleton().add(job);
+}
+
+void InterpolatingPhysicsReceiver::onAncestryChanged(shared_ptr<InterpolatingPhysicsReceiver> interpolatingPhysicsReceiver)
+{
+    tryToCreateJob(interpolatingPhysicsReceiver);
+    if (job)
+        // We've succeeded in creating the job
+        serviceProviderConnection.disconnect();
+}
+
+InterpolatingPhysicsReceiver::~InterpolatingPhysicsReceiver()
+{
+    TaskScheduler::singleton().remove(job);
+}
+
+
+void InterpolatingPhysicsReceiver::Nugget::receive(
+    RakNet::BitStream& bitstream, RakNet::Time timeStamp, const ModelInstance* noLagModel, InterpolatingPhysicsReceiver* receiver)
+{
+    if (history->count > 0)
+    {
+        // Check for out-of-order data
+        if (timeStamp < history->data[history->last].networkTime)
+        {
+            MechanismItem dummy;
+            int numNodesInHistory;
+            receiver->receiveMechanism(bitstream, part.get(), dummy, timeStamp, numNodesInHistory);
+            receiver->outOfOrderMechanisms.sample(1);
+            return;
+        }
+        else
+            receiver->outOfOrderMechanisms.sample(0);
+    }
+
+    history->last = (history->last + 1 + bufferSize) % bufferSize;
+    if (history->count < bufferSize)
+        history->count++;
+
+    MechanismItem& item = history->data[history->last];
+
+    int numNodesInHistory;
+    receiver->receiveMechanism(bitstream, part.get(), item, timeStamp, numNodesInHistory);
+    item.networkTime = timeStamp;
+
+    // Now compute the lag
+    if (noLagModel && part && part->isDescendantOf(noLagModel))
+    {
+        lag = 0;
+    }
+    else if (history->count >= 2)
+    {
+        const int index2 = (history->last - 1 + bufferSize) % bufferSize;
+        double newLag = (double)(item.networkTime - history->data[index2].networkTime);
+
+        // Add a lag buffer
+        const double physicsReceiveDelayFactor = 1.2;
+        newLag *= physicsReceiveDelayFactor;
+
+        // Use an asymetric log average that favors large values.
+        // Small values will be large in number, so they will win out
+        // over time.
+#if 1
+        lag = (lag * lag + newLag * newLag) / (lag + newLag);
+#else
+        if (newLag > lag)
+            lag = 0.8 * newLag + 0.2 * lag;
+        else
+            lag = 0.2 * newLag + 0.8 * lag;
+#endif
+    }
+    receiver->sampleLag(lag);
+}
+
+
+void InterpolatingPhysicsReceiver::setLerpedPhysics(const MechanismItem& itemBefore, const MechanismItem& itemAfter, float lerpAlpha)
+{
+    // For efficiency we don't bother lerping near maxima
+    if (lerpAlpha <= 0.001)
+        setPhysics(itemBefore);
+    else if (lerpAlpha >= 0.99)
+        setPhysics(itemAfter);
+    else if (!MechanismItem::consistent(&itemBefore, &itemAfter))
+        setPhysics(itemAfter);
+    else
+    {
+        MechanismItem::lerp(&itemBefore, &itemAfter, &reusableMechanismItem, lerpAlpha);
+        setPhysics(reusableMechanismItem);
+    }
+}
+
+
+bool InterpolatingPhysicsReceiver::Nugget::step(RakNet::Time time, InterpolatingPhysicsReceiver* receiver) const
+{
+    if (part->getDragging())
+    {
+        return false;
+    }
+
+    if (!part->getPartPrimitive()->getAssembly())
+    {
+        return false;
+    }
+
+    if (!receiver->okDistributedReceivePart(part))
+    {
+        return false;
+    }
+
+    if (history->count == 0)
+        return true;
+
+    if (history->count == 1)
+    {
+        receiver->setPhysics(history->data[history->last]);
+        return true;
+    }
+
+    const double laggedTime = time - lag;
+
+    const MechanismItem* itemAfter = 0;
+    int i;
+    for (i = 0; i < history->count; ++i)
+    {
+        const int index = (history->last - i + bufferSize) % bufferSize;
+
+        const MechanismItem& item(history->data[index]);
+
+        if (item.networkTime > laggedTime)
+        {
+            itemAfter = &item;
+        }
+        else if (itemAfter == 0)
+        {
+            itemAfter = &item;
+            break;
+        }
+        else
+        {
+            float lerpAlpha = (float)(laggedTime - item.networkTime) / (float)(itemAfter->networkTime - item.networkTime);
+            // TODO: Allow for forward-looking lerpAlpha up to a point
+            receiver->setLerpedPhysics(item, *itemAfter, lerpAlpha);
+            receiver->sampleBufferSeek(i + 1);
+            return true;
+        }
+    }
+
+    receiver->sampleBufferSeek(i);
+    receiver->setPhysics(*itemAfter);
+
+    return true;
+}
+
+void InterpolatingPhysicsReceiver::step(RakNet::Time time)
+{
+    lagStats.clearBufferAccumulator();
+    Nuggets::index<Nugget::part_tag>::type& index = nuggets.get<Nugget::part_tag>();
+    Nuggets::index_iterator<Nugget::part_tag>::type iter = index.begin();
+    Nuggets::index_iterator<Nugget::part_tag>::type end = index.end();
+    while (iter != end)
+    {
+        // TODO: Erase if no longer relevant
+        if (!(*iter).step(time, this))
+            iter = index.erase(iter);
+        else
+            ++iter;
+    }
+    lagStats.sampleBufferAccumulator();
+}
+
+void InterpolatingPhysicsReceiver::receivePacket(RakNet::BitStream& inBitstream, RakNet::Time timeStamp, ReplicatorStats::PhysicsReceiverStats* stats)
+{
+    lagStats.clearLagAccumulator();
+    Nuggets::index<Nugget::part_tag>::type& index = nuggets.get<Nugget::part_tag>();
+
+    const ModelInstance* noLagModel = 0;
+
+    while (true)
+    {
+        shared_ptr<PartInstance> part;
+
+        if (!receiveRootPart(part, inBitstream))
+        {
+            break; // no instance - done
+        }
+
+        Nuggets::index_iterator<Nugget::part_tag>::type it = index.find(part);
+
+        if (it != index.end())
+        {
+            index.modify(it, boost::bind(&Nugget::receive, _1, boost::ref(inBitstream), timeStamp, noLagModel, this));
+        }
+        else
+        {
+            Nugget nugget(part);
+            nugget.receive(inBitstream, timeStamp, noLagModel, this);
+            if (part)
+                index.insert(nugget);
+        }
+    }
+    lagStats.sampleLagAccumulator();
+}
+
+
+InterpolatingPhysicsReceiver::LagStats::LagStats()
+    : averageBufferSeek(lerpValue)
+    , averageLag(lerpValue)
+    , maxBufferSeek(0)
+{
+}
+
+void InterpolatingPhysicsReceiver::LagStats::clearBufferAccumulator()
+{
+    bufferSeekAccumulator = AccumulatorSet();
+}
+
+void InterpolatingPhysicsReceiver::LagStats::sampleBufferAccumulator()
+{
+    if (boost::accumulators::count(bufferSeekAccumulator) > 0)
+        averageBufferSeek.sample(boost::accumulators::mean(bufferSeekAccumulator));
+}
+void InterpolatingPhysicsReceiver::LagStats::clearLagAccumulator()
+{
+    lagAccumulator = AccumulatorSet();
+}
+
+void InterpolatingPhysicsReceiver::LagStats::sampleLagAccumulator()
+{
+    if (boost::accumulators::count(lagAccumulator) > 0)
+        averageLag.sample(boost::accumulators::mean(lagAccumulator));
+}
+
+#endif // remove this file when fast flag RemoveInterpolationReciever is accepted and removed

engine/network/src/InterpolatingPhysicsReceiver.hpp

@@ -0,0 +1,187 @@
+#if 1 // remove this file when fast flag RemoveInterpolationReciever is accepted and removed
+
+#pragma once
+#include "PhysicsReceiver.hpp"
+#include "ReplicatorStats.hpp"
+#include "BoostAppend.hpp"
+#include "RunningAverage.hpp"
+
+#include "Nil.hpp"
+
+
+#include <boost/multi_index_container.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/hashed_index.hpp>
+
+#include <boost/accumulators/accumulators.hpp>
+#include <boost/accumulators/statistics/stats.hpp>
+#include <boost/accumulators/statistics/mean.hpp>
+
+
+using boost::multi_index_container;
+using namespace boost::multi_index;
+
+namespace Aya
+{
+
+class ModelInstance;
+
+namespace Network
+{
+
+/// Keeps a history of prior data and interpolates position data by introducing lag
+class InterpolatingPhysicsReceiver : public PhysicsReceiver
+{
+    class Nugget
+    {
+    public:
+        struct part_tag
+        {
+        };
+        struct lastUpdate_tag
+        {
+        };
+
+        // Index fields
+        shared_ptr<PartInstance> part;
+        RakNet::Time lastUpdate;
+
+        // Non-indexed fields
+        double lag;
+
+        static const int bufferSize = 40;
+        struct History
+        {
+            // TODO: Use boost::circular_buffer
+            int last;
+            int count;
+            MechanismItem data[bufferSize];
+            History()
+                : last(0)
+                , count(0)
+            {
+            }
+        };
+        shared_ptr<History> history;
+
+        Nugget(const shared_ptr<PartInstance>& part)
+            : part(part)
+            , lag(0)
+            , history(new History())
+        {
+        }
+
+        void receive(RakNet::BitStream& bitsream, RakNet::Time timeStamp, const ModelInstance* noLagModel, InterpolatingPhysicsReceiver* receiver);
+        bool step(RakNet::Time time, InterpolatingPhysicsReceiver* receiver) const; // returns false if the item should be removed from the list
+    };
+
+    typedef multi_index_container<Nugget,
+        indexed_by<hashed_unique<tag<Nugget::part_tag>, BOOST_MULTI_INDEX_MEMBER(Nugget, shared_ptr<PartInstance>, part)>,
+            ordered_non_unique<tag<Nugget::lastUpdate_tag>, BOOST_MULTI_INDEX_MEMBER(Nugget, RakNet::Time, lastUpdate)>>>
+        Nuggets;
+    Nuggets nuggets;
+
+    class Job;
+    shared_ptr<Job> job;
+    Aya::signals::scoped_connection serviceProviderConnection;
+
+    class LagStats
+    {
+    private:
+        typedef boost::accumulators::accumulator_set<double, boost::accumulators::stats<boost::accumulators::tag::mean>> AccumulatorSet;
+        AccumulatorSet bufferSeekAccumulator;
+        AccumulatorSet lagAccumulator;
+        static void gccWorkaround()
+        {
+            // These statements eliminate "defined but not used" warnings
+            /*
+                            boost::accumulators::extract::quantile;
+                            boost::accumulators::extract::sum;
+                            boost::accumulators::extract::sum_of_weights;
+                            boost::accumulators::extract::sum_of_variates;
+                            boost::accumulators::extract::mean;
+                            boost::accumulators::extract::mean_of_weights;
+                            boost::accumulators::extract::tail_mean;
+            */
+        }
+
+        RunningAverage<double> averageBufferSeek;
+        RunningAverage<double> averageLag;
+
+    public:
+        unsigned int maxBufferSeek;
+        LagStats();
+
+        void clearBufferAccumulator();
+        void accumulateBufferSeek(unsigned int bufferSeek)
+        {
+            if (bufferSeek > maxBufferSeek)
+                maxBufferSeek = bufferSeek;
+            bufferSeekAccumulator(bufferSeek);
+        }
+        void sampleBufferAccumulator();
+
+        void clearLagAccumulator();
+        void accumulateLag(double lag)
+        {
+            lagAccumulator(lag);
+        }
+        void sampleLagAccumulator();
+
+        unsigned int getMaxBufferSeek() const
+        {
+            return maxBufferSeek;
+        }
+        double getAverageBufferSeek() const
+        {
+            return averageBufferSeek.value();
+        }
+        double getAverageLag() const
+        {
+            return averageLag.value();
+        }
+    };
+    LagStats lagStats;
+
+    MechanismItem reusableMechanismItem;
+
+public:
+    RunningAverage<double> outOfOrderMechanisms;
+    InterpolatingPhysicsReceiver(Replicator* replicator, bool isServer);
+    ~InterpolatingPhysicsReceiver();
+    /*override*/ void start(shared_ptr<PhysicsReceiver> physicsReceiver);
+    void setLerpedPhysics(const MechanismItem& itemBefore, const MechanismItem& itemAfter, float lerpAlpha);
+    virtual void receivePacket(RakNet::BitStream& bitsream, RakNet::Time timeStamp, ReplicatorStats::PhysicsReceiverStats* stats);
+    void sampleBufferSeek(unsigned int bufferSeek)
+    {
+        lagStats.accumulateBufferSeek(bufferSeek);
+    }
+    void sampleLag(double lag)
+    {
+        lagStats.accumulateLag(lag);
+    }
+    unsigned int getMaxBufferSeek() const
+    {
+        return lagStats.maxBufferSeek;
+    }
+    double getAverageBufferSeek() const
+    {
+        return lagStats.getAverageBufferSeek();
+    }
+    double getAverageLag() const
+    {
+        return lagStats.getAverageLag();
+    }
+
+    void step(RakNet::Time time);
+
+private:
+    void onAncestryChanged(shared_ptr<InterpolatingPhysicsReceiver> interpolatingPhysicsReceiver);
+    void tryToCreateJob(shared_ptr<InterpolatingPhysicsReceiver> interpolatingPhysicsReceiver);
+};
+
+} // namespace Network
+} // namespace Aya
+
+#endif // remove this file when fast flag RemoveInterpolationReciever is accepted and removed

engine/network/src/Item.cpp

@@ -0,0 +1,163 @@
+
+#include "Item.hpp"
+#include "Streaming.hpp"
+
+namespace Aya
+{
+namespace Network
+{
+
+void Item::writeItemType(RakNet::BitStream& stream, Item::ItemType value)
+{
+    // using a 5b item id.
+    BOOST_STATIC_ASSERT(ItemTypeMaxValue < 19);
+
+    if (value >= 1 && value <= 3)
+    {
+        // Compact form
+        stream.WriteBits((const unsigned char*)&value, 2);
+    }
+    else
+    {
+        const char zero = 0;
+        stream.WriteBits((const unsigned char*)&zero, 2);
+        // Long form
+        stream.WriteBits((const unsigned char*)&value, 5);
+    }
+}
+
+void Item::readItemType(RakNet::BitStream& stream, Item::ItemType& value)
+{
+    unsigned int t;
+    readFastN<2>(stream, t);
+    value = static_cast<ItemType>(t);
+
+    if (value != 0)
+    {
+        // Compact form
+    }
+    else
+    {
+        unsigned int t;
+        readFastN<5>(stream, t);
+        value = static_cast<ItemType>(t);
+    }
+}
+
+
+
+bool ItemQueue::validate() const
+{
+    return true;
+}
+
+bool ItemQueue::preValidate(int i)
+{
+    AYAASSERT(inCode == 0);
+    inCode = i;
+    AYAASSERT(validate());
+    return true;
+}
+
+bool ItemQueue::postValidate(int i)
+{
+    AYAASSERT(validate());
+    AYAASSERT(inCode == i);
+    inCode = 0;
+    return true;
+}
+
+ItemQueue::ItemQueue()
+{
+    inCode = 0;
+}
+
+ItemQueue::~ItemQueue()
+{
+    AYAASSERT(inCode == 0);
+}
+
+bool ItemQueue::empty() const
+{
+    return itemList.empty();
+}
+
+size_t ItemQueue::size() const
+{
+    return itemList.size();
+}
+
+Aya::Time::Interval ItemQueue::head_wait() const
+{
+    if (itemList.empty())
+        return Aya::Time::Interval::zero();
+
+    return Aya::Time::now<Aya::Time::Fast>() - itemList.front().timestamp;
+}
+
+Aya::Time ItemQueue::head_time() const
+{
+    if (itemList.empty())
+        return Aya::Time();
+
+    return itemList.front().timestamp;
+}
+
+void ItemQueue::deleteAll()
+{
+    Item* item;
+    while (pop_if_present(item))
+        delete item;
+}
+
+void ItemQueue::clear()
+{
+    itemList.clear();
+}
+
+bool ItemQueue::pop_if_present(Item*& item)
+{
+    AYAASSERT(preValidate(3));
+
+    if (itemList.empty())
+    {
+        AYAASSERT(postValidate(3));
+        return false;
+    }
+
+    item = &itemList.front();
+    itemList.pop_front();
+
+    AYAASSERT(postValidate(3));
+    return true;
+}
+
+
+void ItemQueue::push_back(Item* item)
+{
+    item->timestamp = Aya::Time::now<Aya::Time::Fast>();
+    AYAASSERT(preValidate(4));
+    itemList.push_back(*item);
+    AYAASSERT(postValidate(4));
+}
+
+
+void ItemQueue::push_front(Item* item)
+{
+    item->timestamp = Aya::Time::now<Aya::Time::Fast>();
+    AYAASSERT(preValidate(5));
+    itemList.push_front(*item);
+    AYAASSERT(postValidate(5));
+}
+
+void ItemQueue::push_front_preserve_timestamp(Item* item)
+{
+    AYAASSERT(preValidate(5));
+    itemList.push_front(*item);
+    AYAASSERT(postValidate(5));
+}
+
+
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Item.hpp

@@ -0,0 +1,157 @@
+#pragma once
+
+#include "Declarations.hpp"
+
+#include "boost.hpp"
+#include "Debug.hpp"
+
+#include "time.hpp"
+
+#include "Memory.hpp"
+
+#include "boost/intrusive/list.hpp"
+
+namespace RakNet
+{
+class BitStream;
+}
+
+namespace Aya
+{
+namespace Network
+{
+
+class Replicator;
+
+struct ItemTag;
+typedef boost::intrusive::list_base_hook<boost::intrusive::tag<ItemTag>> ItemHook;
+
+class AyaBaseClass Item
+    : boost::noncopyable
+    , public ItemHook
+{
+public:
+    enum ItemType
+    {
+        ItemTypeEnd = 0,
+        ItemTypeDelete = 1,
+        ItemTypeNew = 2,
+        ItemTypeChangeProperty = 3,
+        ItemTypeMarker = 4,
+        ItemTypePing = 5,
+        ItemTypePingBack = 6,
+        ItemTypeEventInvocation = 7,
+        ItemTypeRequestCharacter = 8,
+        ItemTypeRocky = 9, // Cheat reporting
+        ItemTypePropAcknowledgement = 10,
+        ItemTypeJoinData = 11,
+        ItemTypeUpdateClientQuota = 12,
+        ItemTypeStreamData = 13,
+        ItemTypeRegionRemoval = 14,
+        ItemTypeInstanceRemoval = 15,
+        ItemTypeTag = 16,
+        ItemTypeStats = 17,
+        ItemTypeHash = 18,
+        ItemTypeMaxValue = 18,
+    };
+
+protected:
+    Replicator& replicator;
+    Item(Replicator& replicator)
+        : replicator(replicator)
+    {
+    }
+
+public:
+    Aya::Time timestamp;
+
+    virtual ~Item() {}
+    virtual bool write(RakNet::BitStream& bitStream) = 0;
+    static void writeItemType(RakNet::BitStream& stream, Item::ItemType value);
+    static void readItemType(RakNet::BitStream& stream, Item::ItemType& value);
+};
+
+// Memory pool enabled Item
+class PooledItem
+    : public Item
+    , public AutoMemPool::Object
+{
+public:
+    PooledItem(Replicator& replicator)
+        : Item(replicator)
+    {
+    }
+    virtual ~PooledItem() {}
+};
+
+// Debugs and checks for re-entrant calls.  Right now NOT thread safe.   Just here for safety watch.
+
+class AyaBaseClass ItemQueue : boost::noncopyable
+{
+public:
+    typedef boost::intrusive::list<Item, boost::intrusive::base_hook<ItemHook>> ItemList;
+
+private:
+    ItemList itemList;
+
+    volatile int inCode;
+
+    bool validate() const;
+
+    bool preValidate(int i);
+
+    bool postValidate(int i);
+
+public:
+    ItemQueue();
+
+    ~ItemQueue();
+
+    bool empty() const;
+    size_t size() const;
+    Aya::Time::Interval head_wait() const;
+    Aya::Time head_time() const;
+
+    void deleteAll();
+
+    void clear();
+
+    bool pop_if_present(Item*& item);
+
+    void push_back(Item* item);
+
+    void push_front(Item* item);
+
+    void push_front_preserve_timestamp(Item* item);
+
+    ItemList::iterator begin()
+    {
+        return itemList.begin();
+    }
+    ItemList::iterator end()
+    {
+        return itemList.end();
+    }
+};
+
+class AyaBaseClass DeserializedItem
+{
+protected:
+    Item::ItemType type;
+
+public:
+    DeserializedItem()
+        : type(Item::ItemTypeEnd) {};
+    virtual ~DeserializedItem() {};
+
+    Item::ItemType getType()
+    {
+        return type;
+    }
+
+    virtual void process(Replicator& replicator) = 0;
+};
+
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Marker.cpp

@@ -0,0 +1,31 @@
+#include "./Marker.hpp"
+#include "atomic.hpp"
+
+
+namespace Aya
+{
+namespace Network
+{
+
+const char* const sMarker = "NetworkMarker";
+
+Reflection::EventDesc<Marker, void()> event_Returned(&Marker::receivedSignal, "Received");
+
+static Aya::atomic<int> ctr;
+
+Marker::Marker()
+    : returned(false)
+    , id(++ctr)
+{
+    // Markers should never be put under another object
+    this->lockParent();
+}
+
+void Marker::fireReturned()
+{
+    // TODO: Memoize the result and handle new connections using a Future system
+    receivedSignal();
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Marker.hpp

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "Tree/Instance.hpp"
+
+
+namespace Aya
+{
+namespace Network
+{
+
+extern const char* const sMarker;
+
+class Marker : public Aya::DescribedNonCreatable<Marker, Instance, sMarker>
+{
+    bool returned; // == the marker has made the round-trip
+public:
+    Marker();
+    const long id;
+    Aya::signal<void()> receivedSignal;
+    void fireReturned();
+};
+
+
+} // namespace Network
+} // namespace Aya

engine/network/src/MechanismItem.cpp

@@ -0,0 +1,87 @@
+
+
+#include "MechanismItem.hpp"
+#include "Utility/PV.hpp"
+
+#include "Utility/Math.hpp"
+
+#include "Debug.hpp"
+
+
+using namespace Aya;
+
+MechanismItem::~MechanismItem()
+{
+    for (int i = 0; i < buffer.size(); ++i)
+        delete buffer[i];
+}
+
+void MechanismItem::reset(int numElements)
+{
+    networkHumanoidState = 0;
+    hasVelocity = false;
+    networkTime = 0;
+    currentElements = numElements;
+    while (buffer.size() < numElements)
+        buffer.append(new AssemblyItem());
+}
+
+AssemblyItem& MechanismItem::appendAssembly()
+{
+    AYAASSERT(buffer.size() >= currentElements);
+
+    if (buffer.size() == currentElements)
+        buffer.append(new AssemblyItem());
+    else
+    {
+        AYAASSERT(buffer[currentElements] != NULL);
+        buffer[currentElements]->reset();
+    }
+
+    AssemblyItem& answer = *buffer[currentElements];
+
+    currentElements++;
+
+    return answer;
+}
+
+/*
+        For now - no lerp on mechanisms (>1 assembly)
+*/
+
+bool MechanismItem::consistent(const MechanismItem* before, const MechanismItem* after)
+{
+    AYAASSERT(before && after);
+    AYAASSERT(before->networkTime < after->networkTime);
+
+    return ((before->hasVelocity == after->hasVelocity) && (before->numAssemblies() == 1) && (after->numAssemblies() == 1) &&
+            (before->getAssemblyItem(0).motorAngles.size() == after->getAssemblyItem(0).motorAngles.size()));
+}
+
+
+void MechanismItem::lerp(const MechanismItem* before, const MechanismItem* after, MechanismItem* out, float lerpAlpha)
+{
+    AYAASSERT(consistent(before, after));
+    AYAASSERT(before->numAssemblies() == 1);
+
+    out->reset(before->numAssemblies());
+
+    out->networkTime = 0; // shouldn't be used
+    out->networkHumanoidState = after->networkHumanoidState;
+    out->hasVelocity = after->hasVelocity;
+
+    AssemblyItem& beforeA = before->getAssemblyItem(0);
+    AssemblyItem& afterA = after->getAssemblyItem(0);
+    AssemblyItem& outA = out->getAssemblyItem(0);
+
+    outA.rootPart = afterA.rootPart;
+
+    outA.pv = beforeA.pv.lerp(afterA.pv, lerpAlpha);
+
+    outA.motorAngles.resize(beforeA.motorAngles.size()); // should prevent reallocs
+    for (int i = 0; i < beforeA.motorAngles.size(); ++i)
+    {
+        outA.motorAngles[i] = CompactCFrame(beforeA.motorAngles[i].translation.lerp(afterA.motorAngles[i].translation, lerpAlpha),
+            beforeA.motorAngles[i].getAxisAngle().lerp(afterA.motorAngles[i].getAxisAngle(), lerpAlpha));
+    }
+}

engine/network/src/MechanismItem.hpp

@@ -0,0 +1,79 @@
+#pragma once
+
+#include "DataModel/PartInstance.hpp"
+
+#include "Utility/PV.hpp"
+
+#include "Memory.hpp"
+
+#include "Debug.hpp"
+
+#include "Array.hpp"
+#include "CompactCFrame.hpp"
+#include "RakNet/RakNetTime.hpp"
+
+namespace Aya
+{
+
+class Primitive;
+
+class AssemblyItem : public boost::noncopyable
+{
+public:
+    shared_ptr<PartInstance> rootPart;
+    Aya::PV pv;
+    G3D::Array<CompactCFrame> motorAngles;
+
+    void reset()
+    {
+        rootPart.reset();
+        motorAngles.fastClear();
+    }
+
+    AssemblyItem()
+    {
+        reset();
+    }
+};
+
+
+class MechanismItem : boost::noncopyable
+{
+private:
+    // TODO: Replace with intrusive list of AssemblyItem
+    G3D::Array<AssemblyItem*> buffer;
+
+    int currentElements;
+
+public:
+    RakNet::Time networkTime;
+    unsigned char networkHumanoidState;
+    bool hasVelocity;
+
+    void reset(int numElements = 0);
+
+    MechanismItem()
+    {
+        reset();
+    }
+
+    ~MechanismItem();
+
+    AssemblyItem& appendAssembly();
+
+    int numAssemblies() const
+    {
+        return currentElements;
+    }
+
+    AssemblyItem& getAssemblyItem(int i) const
+    {
+        AYAASSERT(i < currentElements);
+        return *buffer[i];
+    }
+
+    static bool consistent(const MechanismItem* before, const MechanismItem* after);
+
+    static void lerp(const MechanismItem* before, const MechanismItem* after, MechanismItem* out, float lerpAlpha);
+};
+} // namespace Aya

engine/network/src/MovementHistoryJob.cpp

@@ -0,0 +1,50 @@
+
+#include "MovementHistoryJob.hpp"
+#include "NetworkSettings.hpp"
+#include "DataModel/DataModel.hpp"
+
+#include "DataModel/PartInstance.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+#include "Util.hpp"
+
+
+using namespace Aya;
+using namespace Aya::Network;
+
+MovementHistoryJob::MovementHistoryJob(shared_ptr<DataModel> dataModel)
+    : DataModelJob("Movement History Job", DataModelJob::Write, false, dataModel, Time::Interval(0))
+    , dataModel(dataModel)
+    , movementHistoryRate(20)
+{
+    cyclicExecutive = true;
+    cyclicPriority = CyclicExecutiveJobPriority_Network_ProcessIncoming;
+}
+
+Time::Interval MovementHistoryJob::sleepTime(const Stats& stats)
+{
+    return computeStandardSleepTime(stats, movementHistoryRate);
+}
+
+TaskScheduler::Job::Error MovementHistoryJob::error(const Stats& stats)
+{
+    return TaskScheduler::Job::computeStandardErrorCyclicExecutiveSleeping(stats, movementHistoryRate);
+}
+
+
+TaskScheduler::StepResult MovementHistoryJob::stepDataModelJob(const Stats& stats)
+{
+    bool deprecated = true;
+    AYAASSERT(!deprecated);
+
+    if (shared_ptr<DataModel> safeDataModel = dataModel.lock())
+    {
+        DataModel::scoped_write_request request(safeDataModel.get());
+
+        safeDataModel->getWorkspace()->updateHistory();
+
+        return TaskScheduler::Stepped;
+    }
+    return TaskScheduler::Done;
+}

engine/network/src/MovementHistoryJob.hpp

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "DataModel/DataModelJob.hpp"
+
+
+namespace Aya
+{
+
+class PartInstance;
+class DataModel;
+
+namespace Network
+{
+
+class Server;
+class ServerReplicator;
+
+class MovementHistoryJob : public DataModelJob
+{
+private:
+    boost::weak_ptr<DataModel> dataModel;
+    float movementHistoryRate;
+
+    // Job overrides
+    /*override*/ virtual Error error(const Stats& stats);
+    /*override*/ Time::Interval sleepTime(const Stats& stats);
+    /*override*/ virtual TaskScheduler::StepResult stepDataModelJob(const Stats& stats);
+
+public:
+    MovementHistoryJob(shared_ptr<DataModel> dataModel);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/NetworkClusterPacketCache.cpp

@@ -0,0 +1,144 @@
+#include "NetworkClusterPacketCache.hpp"
+
+#include "DataModel/MegaCluster.hpp"
+
+
+#include "Voxel/Cell.hpp"
+
+#include "Voxel2/Grid.hpp"
+
+
+// raknet
+#include "RakNet/BitStream.hpp"
+
+using namespace Aya;
+using namespace Aya::Network;
+
+LOGGROUP(TerrainCellListener)
+
+const char* const Aya::Network::sClusterPacketCacheBase = "ClusterPacketCacheBase";
+const char* const Aya::Network::sClusterPacketCache = "ClusterPacketCache";
+const char* const Aya::Network::sOneQuarterClusterPacketCache = "OneQuarterClusterPacketCacheBase";
+
+template<class Key>
+ClusterPacketCacheBase<Key>::ClusterPacketCacheBase()
+{
+}
+
+template<class Key>
+unsigned int ClusterPacketCacheBase<Key>::getCachedBitStreamBytesUsed(const Key& regionId)
+{
+    boost::shared_lock<boost::shared_mutex> lock(sharedMutex);
+
+    CachedBitStream& cachedStream = streamCache[regionId];
+
+    if (!cachedStream.dirty)
+    {
+        return cachedStream.bitStream->GetNumberOfBytesUsed();
+    }
+
+    return 0;
+}
+
+template<class Key>
+bool ClusterPacketCacheBase<Key>::fetchIfUpToDate(const Key& regionId, RakNet::BitStream& outBitStream)
+{
+    boost::shared_lock<boost::shared_mutex> lock(sharedMutex);
+
+    CachedBitStream& cachedStream = streamCache[regionId];
+
+    if (!cachedStream.dirty)
+    {
+        outBitStream.WriteBits(cachedStream.bitStream->GetData(), cachedStream.bitStream->GetNumberOfBitsUsed(), false);
+        return true;
+    }
+
+    return false;
+}
+
+// copy all data from bitStream to cache
+template<class Key>
+bool ClusterPacketCacheBase<Key>::update(const Key& regionId, RakNet::BitStream& bitStream, unsigned int numBits)
+{
+    boost::unique_lock<boost::shared_mutex> lock(sharedMutex);
+
+    CachedBitStream& cachedStream = streamCache[regionId];
+
+    if (!cachedStream.bitStream)
+    {
+        cachedStream.bitStream.reset(new RakNet::BitStream());
+    }
+
+    cachedStream.bitStream->Reset();
+    cachedStream.bitStream->Write(bitStream, numBits);
+    cachedStream.dirty = false;
+
+    return true;
+}
+
+template<class Key>
+void ClusterPacketCacheBase<Key>::setupListener(MegaClusterInstance* megaClusterInstance)
+{
+    if (!clusterInstance)
+    {
+        if (megaClusterInstance->isSmooth())
+            megaClusterInstance->getSmoothGrid()->connectListener(this);
+        else
+            megaClusterInstance->getVoxelGrid()->connectListener(this);
+
+        clusterInstance = shared_from(megaClusterInstance);
+    }
+}
+
+template<class Key>
+void ClusterPacketCacheBase<Key>::onServiceProvider(ServiceProvider* oldProvider, ServiceProvider* newProvider)
+{
+    if (oldProvider)
+    {
+        streamCache.clear();
+
+        if (clusterInstance)
+        {
+            if (clusterInstance->isSmooth())
+                clusterInstance->getSmoothGrid()->disconnectListener(this);
+            else
+                clusterInstance->getVoxelGrid()->disconnectListener(this);
+
+            clusterInstance.reset();
+        }
+    }
+}
+
+// d9mz - error: explicit instantiation of 'Aya::Network::ClusterPacketCacheBase' must occur in namespace 'Network'
+namespace Aya
+{
+namespace Network
+{
+// Explicit template instantiation
+template class ClusterPacketCacheBase<StreamRegion::Id>;
+template class ClusterPacketCacheBase<SpatialRegion::Id>;
+} // namespace Network
+} // namespace Aya
+
+void ClusterPacketCache::terrainCellChanged(const Voxel::CellChangeInfo& cell)
+{
+    streamCache[SpatialRegion::regionContainingVoxel(cell.position)].dirty = true;
+}
+
+void ClusterPacketCache::onTerrainRegionChanged(const Voxel2::Region& region)
+{
+    AYAASSERT(false);
+}
+
+void OneQuarterClusterPacketCache::terrainCellChanged(const Voxel::CellChangeInfo& cell)
+{
+    streamCache[StreamRegion::regionContainingVoxel(cell.position)].dirty = true;
+}
+
+void OneQuarterClusterPacketCache::onTerrainRegionChanged(const Voxel2::Region& region)
+{
+    std::vector<Vector3int32> ids = region.getChunkIds(StreamRegion::_PrivateConstants::kRegionSizeInVoxelsAsBitShift);
+
+    for (auto i : ids)
+        streamCache[StreamRegion::Id(i)].dirty = true;
+}

engine/network/src/NetworkClusterPacketCache.hpp

@@ -0,0 +1,109 @@
+#pragma once
+
+#include "Tree/Service.hpp"
+
+#include "signal.hpp"
+
+#include "Utility/StreamRegion.hpp"
+
+
+#include "Voxel/CellChangeListener.hpp"
+
+#include "Voxel2/GridListener.hpp"
+
+
+namespace RakNet
+{
+class BitStream;
+}
+
+namespace Aya
+{
+
+class MegaClusterInstance;
+
+namespace Network
+{
+// This class stores the final raknet bit stream of each cluster chunk.
+// Cache operations are guarded by a shared mutex that uses readers-writer lock,
+// this allows multiple send jobs to fetch from cache concurrently, but only one can update at any given time
+extern const char* const sClusterPacketCacheBase;
+
+template<class Key>
+class ClusterPacketCacheBase
+    : public Voxel::CellChangeListener
+    , public Voxel2::GridListener
+{
+protected:
+    struct CachedBitStream
+    {
+        bool dirty;
+        boost::shared_ptr<RakNet::BitStream> bitStream;
+
+        CachedBitStream()
+            : dirty(true)
+        {
+        }
+    };
+
+    typedef boost::unordered_map<Key, CachedBitStream> StreamCacheList;
+    StreamCacheList streamCache;
+
+    boost::shared_mutex sharedMutex;
+    boost::shared_ptr<MegaClusterInstance> clusterInstance;
+
+public:
+    ClusterPacketCacheBase();
+    virtual ~ClusterPacketCacheBase() {};
+
+    // Search for cached bitstream by index. This function uses a shared locked to allow multiple reads at same time.
+    // Returns true if fetched successfully.
+    bool fetchIfUpToDate(const Key& regionId, RakNet::BitStream& outBitStream);
+
+    // Copy all data from bitStream to cache. This function upgrades the shard mutex into an unique lock that blocks all other
+    // read and write operations.
+    bool update(const Key& regionId, RakNet::BitStream& bitStream, unsigned int numBits);
+
+    unsigned int getCachedBitStreamBytesUsed(const Key& regionId);
+
+    void setupListener(MegaClusterInstance* megaClusterInstance);
+
+protected:
+    void onServiceProvider(ServiceProvider* oldProvider, ServiceProvider* newProvider);
+};
+
+extern const char* const sClusterPacketCache;
+class ClusterPacketCache
+    : public ClusterPacketCacheBase<SpatialRegion::Id>
+    , public DescribedNonCreatable<ClusterPacketCache, Instance, sClusterPacketCache>
+    , public Service
+{
+public:
+    ClusterPacketCache()
+    {
+        setName(sClusterPacketCache);
+    };
+    ~ClusterPacketCache() {};
+
+    void terrainCellChanged(const Voxel::CellChangeInfo& cell) override;
+    void onTerrainRegionChanged(const Voxel2::Region& region) override;
+};
+
+extern const char* const sOneQuarterClusterPacketCache;
+class OneQuarterClusterPacketCache
+    : public ClusterPacketCacheBase<StreamRegion::Id>
+    , public DescribedNonCreatable<OneQuarterClusterPacketCache, Instance, sOneQuarterClusterPacketCache>
+    , public Service
+{
+public:
+    OneQuarterClusterPacketCache()
+    {
+        setName(sOneQuarterClusterPacketCache);
+    };
+    ~OneQuarterClusterPacketCache() {};
+
+    void terrainCellChanged(const Voxel::CellChangeInfo& cell) override;
+    void onTerrainRegionChanged(const Voxel2::Region& region) override;
+};
+} // namespace Network
+} // namespace Aya

engine/network/src/NetworkFilter.cpp

@@ -0,0 +1,425 @@
+#include "NetworkFilter.hpp"
+
+#include "Player.hpp"
+
+#include "Replicator.hpp"
+#include "Humanoid/Humanoid.hpp"
+#include "DataModel/PartInstance.hpp"
+
+#include "DataModel/PlayerGui.hpp"
+
+#include "DataModel/Hopper.hpp"
+
+#include "DataModel/Teams.hpp"
+
+#include "DataModel/Accoutrement.hpp"
+
+#include "DataModel/SkateboardPlatform.hpp"
+
+#include "DataModel/Seat.hpp"
+
+#include "DataModel/StarterPlayerService.hpp"
+
+#include "Players.hpp"
+
+
+namespace Aya
+{
+namespace Reflection
+{
+template<>
+EnumDesc<Network::FilterResult>::EnumDesc()
+    : EnumDescriptor("FilterResult")
+{
+    addPair(Network::Reject, "Rejected");
+    addPair(Network::Accept, "Accepted");
+}
+
+template<>
+Network::FilterResult& Variant::convert<Network::FilterResult>(void)
+{
+    return genericConvert<Network::FilterResult>();
+}
+
+} // namespace Reflection
+
+template<>
+bool StringConverter<Network::FilterResult>::convertToValue(const std::string& text, Network::FilterResult& value)
+{
+    return Reflection::EnumDesc<Network::FilterResult>::singleton().convertToValue(text.c_str(), value);
+}
+} // namespace Aya
+
+using namespace Aya;
+using namespace Reflection;
+using namespace Network;
+
+
+NetworkFilter::NetworkFilter(Replicator* replicator)
+    : replicator(replicator)
+{
+}
+
+NetworkFilter::~NetworkFilter(void) {}
+
+template<class T, FilterResult R>
+static bool filterInside(Instance* parent, FilterResult& result)
+{
+    if (!parent)
+        return false;
+
+    if (Instance::fastDynamicCast<T>(parent))
+    {
+        result = R;
+        return true;
+    }
+
+    return filterInside<T, R>(parent->getParent(), result);
+}
+
+
+bool Aya::Network::NetworkFilter::filterChangedProperty(Instance* instance, const Reflection::PropertyDescriptor& desc, FilterResult& result)
+{
+    // if desc==Instance::propParent, then filterParent will be called, so those rules apply, too
+
+    // Can't change items in these services:
+    if (filterInside<StarterGuiService, Reject>(instance, result))
+        return true;
+    if (filterInside<StarterPackService, Reject>(instance, result))
+        return true;
+    if (filterInside<StarterPlayerService, Reject>(instance, result))
+        return true;
+    if (filterInside<Teams, Reject>(instance, result))
+        return true;
+
+    // Allow/Reject some physics properties
+    if (desc == PartInstance::prop_NetworkIsSleeping)
+    {
+        result = Accept;
+        return true;
+    }
+
+#if 0
+	// DE3665
+	if (desc == PartInstance::prop_NetworkOwner)
+	{
+		result = Reject;
+		return true;
+	}
+#endif
+
+    if (desc == Player::prop_SimulationRadius)
+    {
+        if (replicator->getPlayer().get() == instance)
+            result = Accept;
+        else
+            result = Reject;
+        return true;
+    }
+
+    if (Humanoid* humanoid = Instance::fastDynamicCast<Humanoid>(instance))
+    {
+        Network::Player* player = Network::Players::getPlayerFromCharacter(Humanoid::getCharacterFromHumanoid(humanoid));
+
+        if (player == replicator->getPlayer().get())
+            if (humanoid->isLegalForClientToChange(desc))
+            {
+                result = Accept;
+                return true;
+            }
+    }
+
+    return false;
+}
+
+bool Aya::Network::NetworkFilter::filterNew(Instance* instance, Instance* parent, FilterResult& result)
+{
+    if (filterParent(instance, parent, result))
+        return true;
+
+    return false;
+}
+
+bool Aya::Network::NetworkFilter::filterDelete(Instance* instance, FilterResult& result)
+{
+    if (filterParent(instance, NULL, result))
+        return true;
+
+    return false;
+}
+
+bool Aya::Network::NetworkFilter::filterParent(Instance* instance, Instance* parent, FilterResult& result)
+{
+    if (Player* player = Instance::fastDynamicCast<Player>(instance))
+    {
+        if (!player->getParent())
+        {
+            // This Player is just being inserted into the world, so it is presumably the local player.
+            // Let it go if it is going to the right place:
+            result = Instance::fastDynamicCast<Players>(parent) ? Accept : Reject;
+            return true;
+        }
+    }
+
+    // Can't move to these services:
+    if (filterInside<StarterGuiService, Reject>(parent, result))
+        return true;
+    if (filterInside<StarterPackService, Reject>(parent, result))
+        return true;
+    if (filterInside<StarterPlayerService, Reject>(parent, result))
+        return true;
+    if (filterInside<Teams, Reject>(parent, result))
+        return true;
+
+    // Can't move when in these services:
+    if (filterInside<StarterGuiService, Reject>(instance, result))
+        return true;
+    if (filterInside<StarterPackService, Reject>(instance, result))
+        return true;
+    if (filterInside<StarterPlayerService, Reject>(instance, result))
+        return true;
+    if (filterInside<Teams, Reject>(instance, result))
+        return true;
+
+    // Can't set if parent is locked
+    if (instance && instance->getIsParentLocked())
+    {
+        StandardOut::singleton()->printf(MESSAGE_WARNING, "trying to set locked parent!");
+        result = Reject;
+        return true;
+    }
+
+    return false;
+}
+
+bool Aya::Network::NetworkFilter::filterEvent(Instance* instance, const Reflection::EventDescriptor& desc, FilterResult& result)
+{
+    return false;
+}
+
+boost::unordered_set<std::string> StrictNetworkFilter::propertyWhiteList;
+boost::unordered_set<std::string> StrictNetworkFilter::eventWhiteList;
+
+StrictNetworkFilter::StrictNetworkFilter(Replicator* replicator)
+    : replicator(replicator)
+    , instanceBeingRemovedFromLocalPlayer(NULL)
+{
+    // these lists should only contain scriptable properties or events, all non-scriptable (REPLICATE_ONLY) items are not filtered
+
+    if (propertyWhiteList.size() == 0)
+    {
+        std::string props[] = {"Jump", "Sit", "Throttle", "Steer", "SimulationRadius"};
+        propertyWhiteList.insert(props, props + sizeof(props) / sizeof(std::string));
+    }
+
+    if (eventWhiteList.size() == 0)
+    {
+        std::string events[] = {"OnServerEvent", "PromptProductPurchaseFinished", "PromptPurchaseFinished", "PromptPurchaseRequested",
+            "ClientPurchaseSuccess", "ServerPurchaseVerification", "Activated", "Deactivated", "SimulationRadiusChanged", "LuaDialogCallbackSignal",
+            "ServerAdVerification", "ClientAdVerificationResults"};
+        eventWhiteList.insert(events, events + sizeof(events) / sizeof(std::string));
+    }
+}
+
+void StrictNetworkFilter::onChildRemoved(Instance* removed, const Instance* oldParent)
+{
+    instanceBeingRemovedFromLocalPlayer = NULL;
+
+    if (Player* player = replicator->findTargetPlayer())
+    {
+        if (ModelInstance* playerModel = player->getCharacter())
+        {
+            if (oldParent == playerModel || oldParent->isDescendantOf(playerModel))
+            {
+                instanceBeingRemovedFromLocalPlayer = removed;
+                return;
+            }
+        }
+
+        if ((oldParent == player) || (oldParent->isDescendantOf(player)))
+        {
+            instanceBeingRemovedFromLocalPlayer = removed;
+        }
+    }
+}
+
+FilterResult StrictNetworkFilter::filterChangedProperty(Instance* instance, const Reflection::PropertyDescriptor& desc)
+{
+    if (desc == Instance::propParent)
+        return Accept; // should be handled by filterParent
+
+    if (desc.isScriptable() || desc.canXmlWrite() || desc.canXmlRead())
+    {
+        if (propertyWhiteList.find(desc.name.toString()) != propertyWhiteList.end())
+            return Accept;
+
+        return Reject;
+    }
+
+    // accept all non-scriptable properties
+    return Accept;
+}
+
+bool isOrUnderAncestor(const Instance* instance, const Instance* newParent, const Instance* ancestor)
+{
+    const Instance* oldParent = instance->getParent();
+
+    // originating parent must be player or player character
+    if (oldParent != newParent)
+    {
+        // only server code should enter here
+        AYAASSERT(Players::backendProcessing(instance));
+
+        if (!oldParent)
+            return false;
+
+        // originating parent must be player or player character
+        if (oldParent == ancestor || oldParent->isDescendantOf(ancestor))
+            return true;
+    }
+    else if (newParent == ancestor || newParent->isDescendantOf(ancestor))
+    {
+        // only client should enter here
+        AYAASSERT(Players::frontendProcessing(instance));
+        return true;
+    }
+
+    return false;
+}
+
+FilterResult StrictNetworkFilter::filterParent(Instance* instance, Instance* newParent)
+{
+    FilterResult result = Reject;
+
+    Aya::Weld* weld = NULL;
+    weld = Instance::fastDynamicCast<Aya::Weld>(instance);
+
+    if (instance == instanceBeingRemovedFromLocalPlayer)
+    {
+        // allow parents to be changed from local player
+        result = Accept;
+    }
+    else if (newParent && (instance->fastDynamicCast<Aya::Tool>() || weld || instance->fastDynamicCast<Aya::Accoutrement>() ||
+                              instance->fastDynamicCast<Aya::Seat>() ||
+                              instance->fastDynamicCast<Aya::SkateboardPlatform>())) // Only accept reparenting of tools, hats, and seats
+    {
+        if (Player* player = replicator->findTargetPlayer())
+        {
+            if (ModelInstance* playerModel = player->getCharacter())
+            {
+                if (isOrUnderAncestor(instance, newParent, playerModel))
+                {
+                    result = Accept;
+
+                    if (weld)
+                    {
+                        if (weld->getName() != "RightGrip" && (weld->getPart0() && weld->getPart0()->isDescendantOf(playerModel)))
+                            result = Reject;
+                    }
+                }
+            }
+
+            if (result == Reject)
+            {
+                if (isOrUnderAncestor(instance, newParent, player))
+                    result = Accept;
+            }
+        }
+    }
+
+    // this should be set in onChildRemoved, which should happen right before filterParent, so reset after ever use
+    instanceBeingRemovedFromLocalPlayer = NULL;
+
+    return result;
+}
+
+FilterResult StrictNetworkFilter::filterEvent(Instance* instance, const Reflection::EventDescriptor& desc)
+{
+    if (desc.isScriptable())
+    {
+        // accept only white listed scriptable events
+        if (eventWhiteList.find(desc.name.toString()) != eventWhiteList.end())
+            return Accept;
+
+        return Reject;
+    }
+
+    // accept non-scriptable events
+    return Accept;
+}
+
+FilterResult StrictNetworkFilter::filterNew(const Instance* instance, const Instance* parent)
+{
+    // filter all new instances except local player
+    if (const Player* player = Instance::fastDynamicCast<const Player>(instance))
+    {
+        if (const Players* players = Instance::fastDynamicCast<const Players>(instance->getParent()))
+        {
+            if (players->getConstLocalPlayer() == player) // Client
+            {
+                return Accept;
+            }
+        }
+        else // Server
+        {
+            // This Player is just being inserted into the world, so it is presumably the local player.
+            // Let it go if it is going to the right place:
+            return Instance::fastDynamicCast<Players>(parent) ? Accept : Reject;
+        }
+    }
+
+    // TODO: create StarterGear on server
+    if (Instance::fastDynamicCast<StarterGear>(instance))
+        return Accept;
+    else if (Instance::fastDynamicCast<Weld>(instance)) // allow new tool weld under the player character model
+    {
+        if (Player* player = replicator->findTargetPlayer())
+        {
+            if (ModelInstance* playerModel = player->getCharacter())
+            {
+                if (instance->getParent()) // Client, parent should already be set
+                {
+                    AYAASSERT(Players::frontendProcessing(instance));
+                    return isOrUnderAncestor(instance, parent, playerModel) ? Accept : Reject;
+                }
+                else
+                {
+                    if (parent)
+                    {
+                        // Server, parent haven't been set yet
+                        AYAASSERT(Players::backendProcessing(parent));
+
+                        if (parent->isDescendantOf(playerModel))
+                            return Accept;
+                    }
+                }
+            }
+        }
+    }
+
+    return Reject;
+}
+
+FilterResult StrictNetworkFilter::filterDelete(const Instance* instance)
+{
+    if (Player* player = replicator->findTargetPlayer())
+    {
+        if (ModelInstance* playerModel = player->getCharacter())
+        {
+            if (instance->isDescendantOf(playerModel))
+                return Accept;
+        }
+        else
+        {
+            if (instance->isDescendantOf(player))
+                return Accept;
+        }
+    }
+
+    return Reject;
+}
+
+FilterResult StrictNetworkFilter::filterTerrainCellChange()
+{
+    return Reject;
+}

engine/network/src/NetworkFilter.hpp

@@ -0,0 +1,58 @@
+#pragma once
+
+#include "Reflection/Property.hpp"
+#include "Reflection/Event.hpp"
+#include "API.hpp"
+
+#include <boost/unordered_set.hpp>
+
+namespace Aya
+{
+class Instance;
+namespace Network
+{
+class Replicator;
+
+// Allow only white listed data
+class StrictNetworkFilter
+{
+    Replicator* replicator;
+    Instance* instanceBeingRemovedFromLocalPlayer;
+
+    static boost::unordered_set<std::string> propertyWhiteList;
+    static boost::unordered_set<std::string> eventWhiteList;
+
+public:
+    StrictNetworkFilter(Replicator* replicator);
+    ~StrictNetworkFilter() {}
+
+    FilterResult filterChangedProperty(Instance* instance, const Reflection::PropertyDescriptor& desc);
+    FilterResult filterParent(Instance* instance, Instance* newParent);
+    FilterResult filterEvent(Instance* instance, const Reflection::EventDescriptor& desc);
+    FilterResult filterNew(const Instance* instance, const Instance* parent);
+    FilterResult filterDelete(const Instance* instance);
+    FilterResult filterTerrainCellChange();
+
+    void onChildRemoved(Instance* removed, const Instance* oldParent);
+};
+
+// Algorithms for filtering common incoming data.
+// Whitelists certain physics packets and certain humanoind packets.
+// Blacklists certain data.
+// Used as a first-pass filtering of known safe data.
+class NetworkFilter
+{
+    Replicator* replicator;
+
+public:
+    NetworkFilter(Replicator* replicator);
+    ~NetworkFilter(void);
+
+    bool filterChangedProperty(Instance* instance, const Reflection::PropertyDescriptor& desc, FilterResult& result);
+    bool filterParent(Instance* instance, Instance* parent, FilterResult& result);
+    bool filterNew(Instance* instance, Instance* parent, FilterResult& result);
+    bool filterDelete(Instance* instance, FilterResult& result);
+    bool filterEvent(Instance* instance, const Reflection::EventDescriptor& desc, FilterResult& result);
+};
+} // namespace Network
+} // namespace Aya

engine/network/src/NetworkOwner.hpp

@@ -0,0 +1,82 @@
+
+
+#pragma once
+
+#include "Utility/SystemAddress.hpp"
+
+#include "Utility/Color.hpp"
+
+
+namespace Aya
+{
+namespace Network
+{
+
+class NetworkOwner
+{
+public:
+    static const Aya::SystemAddress Server()
+    {
+        static Aya::SystemAddress s(1, 0);
+        return s;
+    }
+
+    // created on server, have not be properly assigned via network owner job
+    static const Aya::SystemAddress ServerUnassigned()
+    {
+        static Aya::SystemAddress s(1, 1);
+        return s;
+    }
+
+    // default
+    static const Aya::SystemAddress Unassigned()
+    {
+        static Aya::SystemAddress s;
+        AYAASSERT(s == SystemAddress());
+        AYAASSERT(s != NetworkOwner::Server());
+        AYAASSERT(s != NetworkOwner::ServerUnassigned());
+        AYAASSERT(s != NetworkOwner::AssignedOther());
+        return s;
+    }
+
+    // generic value used on client indicating assigned to other clients or server (i.e. not self)
+    static const Aya::SystemAddress AssignedOther()
+    {
+        static Aya::SystemAddress s(0, 1);
+        return s;
+    }
+
+    static bool isClient(const Aya::SystemAddress& address)
+    {
+        return ((address != Server()) && (address != Unassigned()) && (address != ServerUnassigned()));
+    }
+
+    static bool isServer(const Aya::SystemAddress& address)
+    {
+        return address == Server() || address == ServerUnassigned();
+    }
+
+    static Color3 colorFromAddress(const Aya::SystemAddress& systemAddress)
+    {
+        if (systemAddress == Server())
+        {
+            return Color3::white();
+        }
+        else if (systemAddress == Unassigned() || systemAddress == ServerUnassigned())
+        {
+            return Color3::black();
+        }
+        else if (systemAddress == AssignedOther())
+            return Color3::gray();
+        else
+        {
+            unsigned int address = systemAddress.getAddress();
+            unsigned int port = systemAddress.getPort();
+            address += port;
+            return Aya::Color::colorFromInt(address);
+        }
+    }
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/NetworkOwnerJob.cpp

@@ -0,0 +1,323 @@
+
+#include "NetworkOwnerJob.hpp"
+
+#include "NetworkSettings.hpp"
+#include "Players.hpp"
+#include "Server.hpp"
+#include "ServerReplicator.hpp"
+#include "Util.hpp"
+
+#include "DataModel/DataModel.hpp"
+
+#include "DataModel/PartInstance.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+#include "World/Assembly.hpp"
+#include "World/Mechanism.hpp"
+#include "World/Primitive.hpp"
+#include "World/World.hpp"
+#include "World/SpatialFilter.hpp"
+#include "World/DistributedPhysics.hpp"
+#include "NetworkOwner.hpp"
+
+
+using namespace Aya;
+using namespace Aya::Network;
+
+DYNAMIC_FASTFLAG(CyclicExecutiveForServerTweaks)
+
+
+#define DEC_SIM_RADIUS_FPS 15.0f
+#define INC_SIM_RADIUS_FPS 17.5f
+
+NetworkOwnerJob::NetworkOwnerJob(shared_ptr<DataModel> dataModel)
+    : DataModelJob("Distributed Physics Ownership", DataModelJob::Write, false, dataModel, Time::Interval(0))
+    , dataModel(dataModel)
+    , networkOwnerRate(NetworkSettings::singleton().networkOwnerRate)
+{
+    cyclicExecutive = true;
+}
+
+Time::Interval NetworkOwnerJob::sleepTime(const Stats& stats)
+{
+    return computeStandardSleepTime(stats, networkOwnerRate);
+}
+
+TaskScheduler::Job::Error NetworkOwnerJob::error(const Stats& stats)
+{
+    if (DFFlag::CyclicExecutiveForServerTweaks)
+    {
+        return TaskScheduler::Job::computeStandardErrorCyclicExecutiveSleeping(stats, networkOwnerRate);
+    }
+    return TaskScheduler::Job::computeStandardError(stats, networkOwnerRate);
+}
+
+
+TaskScheduler::StepResult NetworkOwnerJob::stepDataModelJob(const Stats& stats)
+{
+    if (shared_ptr<DataModel> safeDataModel = dataModel.lock())
+    {
+        DataModel::scoped_write_request request(safeDataModel.get());
+        AYAASSERT(Server::serverIsPresent(safeDataModel.get()));
+
+        if (Server* server = ServiceProvider::find<Server>(safeDataModel.get()))
+        {
+            updatePlayerLocations(server);
+
+            SpatialFilter* spatialFilter =
+                safeDataModel->getWorkspace()->getWorld()->getSpatialFilter(); // go through all parts/primitives in the spatial filter server side
+            for (size_t phase = 0; phase < Assembly::NUM_PHASES; phase++)
+            {
+                const SpatialFilter::AssemblySet& assemblies = spatialFilter->getAssemblies(static_cast<Assembly::FilterPhase>(phase));
+                SpatialFilter::AssemblySet::const_iterator it;
+                for (it = assemblies.begin(); it != assemblies.end(); ++it)
+                {
+                    Assembly* assembly = *it;
+                    AYAASSERT(Mechanism::isMovingAssemblyRoot(assembly));
+                    PartInstance* part = PartInstance::fromPrimitive(assembly->getAssemblyPrimitive());
+                    updateNetworkOwner(part);
+                }
+            }
+        }
+        return TaskScheduler::Stepped;
+    }
+    return TaskScheduler::Done;
+}
+
+////////////////////////////////////////////////////////////////////////
+
+void NetworkOwnerJob::updatePlayerLocations(Server* server)
+{
+    clientMap.clear();
+
+    for (size_t i = 0; i < server->numChildren(); ++i)
+    {
+        if (ServerReplicator* proxy = Instance::fastDynamicCast<ServerReplicator>(server->getChild(i)))
+        {
+            Region2::WeightedPoint weightedPoint;
+            if (const PartInstance* head = proxy->readPlayerSimulationRegion(weightedPoint))
+            {
+                const Primitive* headPrim = head->getConstPartPrimitive();
+                const Mechanism* headMechanism = headPrim->getConstMechanism();
+                LEGACY_ASSERT(Mechanism::isMovingAssemblyRoot(headPrim->getConstAssembly()));
+
+                Aya::SystemAddress rbxAddress = RakNetToRbxAddress(proxy->remotePlayerId);
+                ClientMapPair pair(rbxAddress, ClientLocation(weightedPoint, proxy, headMechanism));
+                clientMap.insert(pair);
+
+                // reset part count
+                proxy->numPartsOwned = 0;
+            }
+        }
+    }
+}
+
+// TODO - untangle, make more efficient
+
+void NetworkOwnerJob::updateNetworkOwner(PartInstance* part)
+{
+    AYAASSERT(part->getPartPrimitive());
+    AYAASSERT(Assembly::isAssemblyRootPrimitive(part->getPartPrimitive()));
+    AYAASSERT(part->getPartPrimitive()->getAssembly()->getAssemblyPrimitive() == part->getPartPrimitive());
+
+    Aya::SystemAddress owner = part->getNetworkOwner();
+
+    if (part->getNetworkOwnershipRule() == NetworkOwnership_Manual)
+    {
+        ClientMapIt clientLoc = clientMap.find(owner);
+        if ((clientLoc == clientMap.end() && !Network::Players::findPlayerWithAddress(owner, part)) && owner != NetworkOwner::Server())
+        {
+            resetNetworkOwner(part, NetworkOwner::Server());
+            part->setNetworkOwnershipAuto();
+        }
+        else
+        {
+            return;
+        }
+    }
+
+
+    if ((owner == NetworkOwner::Unassigned()) || (owner == NetworkOwner::ServerUnassigned()))
+    {
+
+        Aya::SystemAddress newOwner = NetworkOwner::Server();
+
+        // check to see if this part belong to a player character
+        if (ModelInstance* m = Instance::fastDynamicCast<ModelInstance>(part->getParent()))
+        {
+            if (Players::getPlayerFromCharacter(m))
+            {
+                ClientMapConstIt closestClient = findClosestClientToPart(part);
+
+                if (closestClient != clientMap.end())
+                {
+                    newOwner = closestClient->first;
+
+                    closestClient->second.clientProxy->numPartsOwned++;
+                }
+            }
+        }
+
+        resetNetworkOwner(part, newOwner);
+        return;
+    }
+
+    ClientMapConstIt currentClient = clientMap.find(owner); // will be clientMap.end() for server
+    ClientMapConstIt closestClient;
+
+    bool currentOk = false;
+    bool closestOk = false;
+
+    // projectile special handling to prevent the ownership change from one client to another
+    if (!currentClient->second.overloaded)
+    {
+        if (part->isProjectile() && currentClient != clientMap.end())
+        {
+            currentClient->second.clientProxy->numPartsOwned++;
+            return;
+        }
+    }
+
+    closestClient = findClosestClientToPart(part);
+    currentOk = ((currentClient != clientMap.end()) && clientCanSimulate(part, currentClient));
+    closestOk = ((closestClient != currentClient) && (closestClient != clientMap.end()) && part->networkOwnerTimeUp() &&
+                 clientCanSimulate(part, closestClient));
+
+    if (currentOk)
+    {
+        if (closestOk && switchOwners(part, currentClient, closestClient))
+        {
+            resetNetworkOwner(part, closestClient->first);
+            closestClient->second.clientProxy->numPartsOwned++;
+        }
+        else
+            currentClient->second.clientProxy->numPartsOwned++;
+        return;
+    }
+    // !currentOk
+    if (closestOk)
+    { // no timer needed if current is not ok
+        resetNetworkOwner(part, closestClient->first);
+        closestClient->second.clientProxy->numPartsOwned++;
+        return;
+    }
+
+    resetNetworkOwner(part, NetworkOwner::Server());
+}
+
+void NetworkOwnerJob::resetNetworkOwner(PartInstance* part, const Aya::SystemAddress value)
+{
+    if (part->getNetworkOwner() != value)
+    {
+        part->setNetworkOwnerAndNotify(value);
+
+        part->raknetTime = 0; // clear the last update timestamp
+        // 3 seconds is too slow for iPad clients to push parts to server when stressed
+        // float delayTime = (value == NetworkOwner::Server()) ? 0.1f : 3.0f;		// i.e. - a server part only waits 0.1 second before it can
+        // switch.  A client part waits 3 seconds.
+        float delayTime = 0.1f;
+        part->resetNetworkOwnerTime(delayTime);
+    }
+}
+
+NetworkOwnerJob::ClientMapConstIt NetworkOwnerJob::findClosestClientToPart(PartInstance* part)
+{
+    return findClosestClient(part->getCoordinateFrame().translation.xz());
+}
+
+NetworkOwnerJob::ClientMapConstIt NetworkOwnerJob::findClosestClient(const Vector2& testLocation)
+{
+    ClientMapConstIt answer = clientMap.end();
+    float bestError = 1e20f;
+
+    for (ClientMapConstIt it = clientMap.begin(); it != clientMap.end(); ++it)
+    {
+        float temp = Region2::getRelativeError(testLocation, it->second.clientPoint);
+        if (temp < bestError)
+        {
+            bestError = temp;
+            answer = it;
+
+            // can't get closer then 0
+            if (temp == 0.0f)
+                return answer;
+        }
+    }
+
+    return answer;
+}
+
+/*
+        For a client to be able to simulate a part/assembly, it must be
+
+        Mechanisms that DO NOT include the client's character:		Within the client's simulation region
+
+        Mechanisms that DO include the client's character:
+                                                        Standing on top of something simulated by someone else?		... never simulate
+                                                        NOT standing on something simulated by others:				Always simulate
+*/
+
+
+bool NetworkOwnerJob::clientCanSimulate(PartInstance* part, ClientMapConstIt testLocation)
+{
+    AYAASSERT(testLocation != clientMap.end());
+
+    //	return false;
+
+    if (isClientCharacterMechanism(part, testLocation))
+    {
+        return true;
+    }
+    else
+    {
+        Vector2 partLoc = part->getCoordinateFrame().translation.xz();
+        return Region2::pointInRange(partLoc, testLocation->second.clientPoint, DistributedPhysics::SERVER_SLOP());
+    }
+}
+
+bool NetworkOwnerJob::isClientCharacterMechanism(PartInstance* part, ClientMapConstIt testLocation)
+{
+    const Mechanism* characterMechanism = testLocation->second.characterMechanism;
+
+    const Primitive* dPhysPrim = part->getConstPartPrimitive();
+    const Mechanism* dPhysMechanism = dPhysPrim->getConstMechanism();
+
+    AYAASSERT(Mechanism::isMovingAssemblyRoot(dPhysPrim->getConstAssembly()));
+
+    if (characterMechanism == dPhysMechanism)
+    {
+        return true;
+    }
+    else
+    {
+        return false;
+    }
+}
+
+bool NetworkOwnerJob::switchOwners(PartInstance* part, ClientMapConstIt currentOwner, ClientMapConstIt possibleNewOwner)
+{
+    AYAASSERT(currentOwner != clientMap.end());
+    AYAASSERT(possibleNewOwner != clientMap.end());
+    AYAASSERT(part->getNetworkOwner() == currentOwner->first);
+    AYAASSERT(part->getNetworkOwner() != possibleNewOwner->first);
+
+    if (isClientCharacterMechanism(part, currentOwner))
+    { // no switch allowed if current mechanism is current owner mechanism
+        return false;
+    }
+    else
+    {
+        return Region2::closerToOtherPoint(part->getCoordinateFrame().translation.xz(), currentOwner->second.clientPoint,
+            possibleNewOwner->second.clientPoint, DistributedPhysics::SERVER_SLOP());
+    }
+}
+
+void NetworkOwnerJob::invalidateProjectileOwnership(Aya::SystemAddress addr)
+{
+    ClientMapIt clientLoc = clientMap.find(addr);
+    if (clientLoc != clientMap.end())
+    {
+        clientLoc->second.overloaded = true;
+    }
+}

engine/network/src/NetworkOwnerJob.hpp

@@ -0,0 +1,78 @@
+#pragma once
+
+#include "DataModel/DataModelJob.hpp"
+
+#include "Utility/SystemAddress.hpp"
+
+#include "Utility/G3DCore.hpp"
+
+#include "Utility/Region2.hpp"
+
+#include <map>
+
+namespace Aya
+{
+
+class PartInstance;
+class Mechanism;
+class DataModel;
+
+namespace Network
+{
+
+class Server;
+class ServerReplicator;
+
+class NetworkOwnerJob : public DataModelJob
+{
+private:
+    boost::weak_ptr<DataModel> dataModel;
+    float networkOwnerRate;
+
+    class ClientLocation
+    {
+    public:
+        Region2::WeightedPoint clientPoint;
+        ServerReplicator* clientProxy;
+        const Mechanism* characterMechanism;
+        bool overloaded;
+
+        ClientLocation(const Region2::WeightedPoint& clientPoint, ServerReplicator* clientProxy, const Mechanism* characterMechanism)
+            : clientPoint(clientPoint)
+            , clientProxy(clientProxy)
+            , characterMechanism(characterMechanism)
+            , overloaded(false)
+        {
+        }
+    };
+
+    typedef std::map<Aya::SystemAddress, ClientLocation> ClientMap;
+    typedef std::pair<Aya::SystemAddress, ClientLocation> ClientMapPair;
+    typedef ClientMap::const_iterator ClientMapConstIt;
+    typedef ClientMap::iterator ClientMapIt;
+
+    ClientMap clientMap;
+
+    void updatePlayerLocations(Server* server);
+    void updateNetworkOwner(PartInstance* part);
+    ClientMapConstIt findClosestClientToPart(PartInstance* part);
+    ClientMapConstIt findClosestClient(const Vector2& testLocation);
+
+    bool clientCanSimulate(PartInstance* part, ClientMapConstIt testLocation);
+    bool isClientCharacterMechanism(PartInstance* part, ClientMapConstIt testLocation);
+    bool switchOwners(PartInstance* part, ClientMapConstIt currentOwner, ClientMapConstIt possibleNewOwner);
+    static void resetNetworkOwner(PartInstance* part, const Aya::SystemAddress value);
+
+    // Job overrides
+    /*override*/ virtual Error error(const Stats& stats);
+    /*override*/ Time::Interval sleepTime(const Stats& stats);
+    /*override*/ virtual TaskScheduler::StepResult stepDataModelJob(const Stats& stats);
+
+
+public:
+    NetworkOwnerJob(shared_ptr<DataModel> dataModel);
+    void invalidateProjectileOwnership(Aya::SystemAddress addr);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/NetworkPacketCache.cpp

@@ -0,0 +1,337 @@
+
+#include "NetworkPacketCache.hpp"
+#include "FastLog.hpp"
+#include "DataModel/PhysicsService.hpp"
+
+#include "World/World.hpp"
+#include "World/Assembly.hpp"
+
+
+// raknet
+#include "RakNet/BitStream.hpp"
+#include "NetworkOwner.hpp"
+
+LOGGROUP(NetworkCache)
+DYNAMIC_FASTFLAGVARIABLE(DebugLogStaleInstanceCacheEntry, false)
+
+
+const char* const Aya::Network::sPhysicsPacketCache = "PhysicsPacketCache";
+const char* const Aya::Network::sInstancePacketCache = "InstancePacketCache";
+
+using namespace Aya;
+using namespace Aya::Network;
+
+PhysicsPacketCache::PhysicsPacketCache()
+{
+    setName(sPhysicsPacketCache);
+
+    FASTLOG(FLog::NetworkCache, "PhysicsPacketCache Start");
+}
+
+PhysicsPacketCache::~PhysicsPacketCache() {}
+
+void PhysicsPacketCache::insert(const Assembly* key)
+{
+    boost::unique_lock<boost::shared_mutex> lock(sharedMutex);
+
+    std::pair<StreamCacheMap::iterator, bool> res = streamCache.insert(std::make_pair(key, InnerMap()));
+    AYAASSERT(res.second);
+
+    // create cache entry for child assemblies
+    key->visitConstDescendentAssemblies(boost::bind(&PhysicsPacketCache::insertChildAssembly, this, _1));
+}
+
+void PhysicsPacketCache::insertChildAssembly(const Assembly* assembly)
+{
+    AYAASSERT(assembly);
+
+    std::pair<StreamCacheMap::iterator, bool> res = streamCache.insert(std::make_pair(assembly, InnerMap()));
+    AYAASSERT(res.second);
+}
+
+void PhysicsPacketCache::remove(const Assembly* key)
+{
+    AYAASSERT(key);
+
+    boost::unique_lock<boost::shared_mutex> lock(sharedMutex);
+
+    // remove child assemblies
+    key->visitConstDescendentAssemblies(boost::bind(&PhysicsPacketCache::removeChildAssembly, this, _1));
+
+    streamCache.erase(key);
+}
+
+void PhysicsPacketCache::removeChildAssembly(const Assembly* assembly)
+{
+    AYAASSERT(assembly);
+
+    streamCache.erase(assembly);
+}
+
+bool PhysicsPacketCache::fetchIfUpToDate(const Assembly* key, unsigned char index, RakNet::BitStream& outBitStream)
+{
+    boost::shared_lock<boost::shared_mutex> lock(sharedMutex);
+
+    StreamCacheMap::iterator iter = streamCache.find(key);
+
+    if (iter != streamCache.end())
+    {
+        InnerMap& innerMap = iter->second;
+
+        // entry has not been created
+        InnerMap::iterator innerIter = innerMap.find(index);
+        if (innerIter == innerMap.end())
+            return false;
+
+        CachedBitStream* cachedItem = innerIter->second.get();
+
+        AYAASSERT(key->getConstAssemblyPrimitive()->getWorld());
+
+        int id = cachedItem->lastStepId;
+        if ((id > 0) && (id == key->getConstAssemblyPrimitive()->getWorld()->getWorldStepId()))
+        {
+            AYAASSERT(cachedItem->bitStream);
+
+            outBitStream.WriteBits(cachedItem->bitStream->GetData(), cachedItem->dataBitStartOffset, (RakNet::BitSize_t)cachedItem->totalNumBits);
+
+            return true;
+        }
+    }
+
+    return false;
+}
+
+bool PhysicsPacketCache::update(
+    const Assembly* key, unsigned char index, RakNet::BitStream& bitStream, unsigned int startReadBitPos, unsigned int numBits)
+{
+    boost::upgrade_lock<boost::shared_mutex> lock(sharedMutex);
+
+    StreamCacheMap::iterator iter = streamCache.find(key);
+
+    if (iter != streamCache.end())
+    {
+        boost::upgrade_to_unique_lock<boost::shared_mutex> uniqueLock(lock);
+
+        InnerMap& innerMap = iter->second;
+        std::pair<InnerMap::iterator, bool> i = innerMap.insert(std::make_pair(index, boost::shared_ptr<CachedBitStream>()));
+        if (i.second)
+        {
+            // create new entry if index is not found
+            i.first->second.reset(new CachedBitStream());
+        }
+
+        InnerMap::iterator innerIter = i.first;
+        AYAASSERT(innerIter != innerMap.end());
+
+        CachedBitStream* cachedItem = innerIter->second.get();
+
+        // calculate how many bytes to write
+        int numBytes = ((numBits + 7) >> 3);
+        if ((startReadBitPos & 7) != 0)
+            numBytes++;
+
+        if (!cachedItem->bitStream)
+            cachedItem->bitStream.reset(new RakNet::BitStream(numBytes));
+
+        cachedItem->bitStream->Reset();
+
+        cachedItem->totalNumBits = numBits;
+        int startByte = ((startReadBitPos + 7) >> 3);
+
+        // start at beginning of the byte if read position is between 2 bytes
+        if ((startReadBitPos & 7) != 0)
+            startByte--;
+
+        // write byte aligned for speed and record starting bit so later we know where to start reading
+        cachedItem->bitStream->Write((const char*)bitStream.GetData() + startByte, numBytes);
+        cachedItem->dataBitStartOffset = startReadBitPos - (startByte << 3);
+
+        AYAASSERT(key->getConstAssemblyPrimitive()->getWorld());
+        cachedItem->lastStepId = key->getConstAssemblyPrimitive()->getWorld()->getWorldStepId();
+
+        return true;
+    }
+
+    return false;
+}
+
+void PhysicsPacketCache::onServiceProvider(ServiceProvider* oldProvider, ServiceProvider* newProvider)
+{
+    if (oldProvider)
+    {
+        addingAssemblyConnection.disconnect();
+        removedAssemblyConnection.disconnect();
+
+        streamCache.clear();
+    }
+
+    if (newProvider)
+    {
+        boost::shared_ptr<PhysicsService> physicsService = shared_from(ServiceProvider::find<PhysicsService>(newProvider));
+        AYAASSERT(physicsService);
+
+        // Get all current assemblies
+        std::for_each(physicsService->begin(), physicsService->end(), boost::bind(&PhysicsPacketCache::addPart, this, _1));
+
+        // Listen for assembly changes
+        addingAssemblyConnection = physicsService->assemblyAddingSignal.connect(boost::bind(&PhysicsPacketCache::onAddingAssembly, this, _1));
+        removedAssemblyConnection = physicsService->assemblyRemovedSignal.connect(boost::bind(&PhysicsPacketCache::onRemovedAssembly, this, _1));
+    }
+
+    Super::onServiceProvider(oldProvider, newProvider);
+}
+
+void PhysicsPacketCache::onAddingAssembly(shared_ptr<Instance> assembly)
+{
+    shared_ptr<PartInstance> part = Instance::fastSharedDynamicCast<PartInstance>(assembly);
+    AYAASSERT(part);
+
+    const Assembly* partAssembly = part->getConstPartPrimitive()->getConstAssembly();
+
+    insert(partAssembly);
+}
+
+void PhysicsPacketCache::onRemovedAssembly(shared_ptr<Instance> assembly)
+{
+    shared_ptr<PartInstance> part = Instance::fastSharedDynamicCast<PartInstance>(assembly);
+    AYAASSERT(part);
+
+    const Assembly* partAssembly = part->getConstPartPrimitive()->getConstAssembly();
+
+    remove(partAssembly);
+}
+
+void PhysicsPacketCache::addPart(PartInstance& part)
+{
+    const Assembly* partAssembly = part.getConstPartPrimitive()->getConstAssembly();
+    insert(partAssembly);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+InstancePacketCache::InstancePacketCache()
+{
+    setName(sInstancePacketCache);
+}
+
+InstancePacketCache::~InstancePacketCache() {}
+
+void InstancePacketCache::onServiceProvider(ServiceProvider* oldProvider, ServiceProvider* newProvider)
+{
+    if (oldProvider)
+    {
+        for (StreamCacheMap::iterator i = streamCache.begin(); i != streamCache.end(); i++)
+        {
+            i->second->ancestorChangedConnection.disconnect();
+            i->second->propChangedConnection.disconnect();
+        }
+
+        streamCache.clear();
+    }
+
+    Super::onServiceProvider(oldProvider, newProvider);
+}
+
+void InstancePacketCache::onAncestorChanged(shared_ptr<Instance> instance, shared_ptr<Instance> newParent)
+{
+    if (!newParent)
+    {
+        // remove from cache
+        remove(instance.get());
+
+        // StandardOut::singleton()->printf(MESSAGE_WARNING, "Cache Removing %s", instance->getName().c_str());
+    }
+}
+
+void InstancePacketCache::insert(const Instance* key)
+{
+    boost::shared_ptr<CachedBitStream> newCacheItem(new CachedBitStream(key->getGuid().readableString()));
+
+    std::pair<StreamCacheMap::iterator, bool> result = streamCache.insert(std::make_pair(key, newCacheItem));
+
+    // new item, add listener for prop change
+    if (result.second)
+    {
+        Instance* instance = const_cast<Instance*>(key);
+        result.first->second->propChangedConnection =
+            instance->propertyChangedSignal.connect(boost::bind(&InstancePacketCache::CachedBitStream::onPropertyChanged, result.first->second, _1));
+        result.first->second->ancestorChangedConnection =
+            instance->ancestryChangedSignal.connect(boost::bind(&InstancePacketCache::onAncestorChanged, this, _1, _2));
+    }
+    else
+    {
+        if (!(key->getGuid().readableString() == result.first->second->guidString))
+        {
+            result.first->second->dirty = true;
+        }
+    }
+}
+
+void InstancePacketCache::remove(const Instance* key)
+{
+    StreamCacheMap::iterator iter = streamCache.find(key);
+
+    if (iter != streamCache.end())
+    {
+        iter->second->ancestorChangedConnection.disconnect();
+        iter->second->propChangedConnection.disconnect();
+        streamCache.erase(iter);
+    }
+}
+
+bool InstancePacketCache::fetchIfUpToDate(const Instance* key, RakNet::BitStream& outBitStream, bool isJoinData)
+{
+    boost::shared_lock<boost::shared_mutex> lock(sharedMutex);
+
+    StreamCacheMap::iterator iter = streamCache.find(key);
+
+    if (iter != streamCache.end())
+    {
+        CachedBitStream* cachedItem = iter->second.get();
+
+        if (!(key->getGuid().readableString() == cachedItem->guidString))
+        {
+            cachedItem->dirty = true;
+        }
+
+        if (!cachedItem->bitStream[(int)isJoinData] || cachedItem->dirty)
+            return false;
+
+        outBitStream.WriteBits(
+            cachedItem->bitStream[(int)isJoinData]->GetData(), cachedItem->bitStream[(int)isJoinData]->GetNumberOfBitsUsed(), false);
+        return true;
+    }
+
+    // StandardOut::singleton()->printf(MESSAGE_INFO, "Instance cache miss, part: %s, parent: %s",
+    //								key->getName().c_str(),
+    //								key->getParent() ? key->getParent()->getName().c_str() : ""
+    //								);
+
+    return false;
+}
+
+bool InstancePacketCache::update(const Instance* key, RakNet::BitStream& bitStream, unsigned int numBits, bool isJoinData)
+{
+    boost::upgrade_lock<boost::shared_mutex> lock(sharedMutex);
+
+    StreamCacheMap::iterator iter = streamCache.find(key);
+
+    if (iter != streamCache.end())
+    {
+        boost::upgrade_to_unique_lock<boost::shared_mutex> uniqueLock(lock);
+
+        boost::shared_ptr<CachedBitStream> cachedItem = iter->second;
+
+        if (!cachedItem->bitStream[(int)isJoinData])
+            cachedItem->bitStream[(int)isJoinData].reset(new RakNet::BitStream());
+
+        cachedItem->bitStream[(int)isJoinData]->Reset();
+        cachedItem->bitStream[(int)isJoinData]->Write(bitStream, numBits);
+        cachedItem->dirty = false;
+
+        return true;
+    }
+
+    return false;
+}

engine/network/src/NetworkPacketCache.hpp

@@ -0,0 +1,143 @@
+#pragma once
+
+
+#include "Tree/Service.hpp"
+
+#include "DataModel/PartInstance.hpp"
+
+#include "signal.hpp"
+
+#include <boost/unordered/unordered_map.hpp>
+
+namespace RakNet
+{
+class BitStream;
+}
+
+namespace Aya
+{
+namespace Network
+{
+
+extern const char* const sPhysicsPacketCache;
+class PhysicsPacketCache
+    : public DescribedNonCreatable<PhysicsPacketCache, Instance, sPhysicsPacketCache>
+    , public Service
+{
+    typedef DescribedNonCreatable<PhysicsPacketCache, Instance, sPhysicsPacketCache> Super;
+
+public:
+    class CachedBitStream
+    {
+    public:
+        int lastStepId;
+        int dataBitStartOffset;
+        int totalNumBits;
+        boost::shared_ptr<RakNet::BitStream> bitStream;
+
+        CachedBitStream()
+            : lastStepId(0)
+            , dataBitStartOffset(0)
+            , totalNumBits(0)
+        {
+        }
+        ~CachedBitStream() {}
+    };
+
+private:
+    typedef boost::unordered_map<unsigned char, boost::shared_ptr<CachedBitStream>> InnerMap;
+    typedef boost::unordered_map<const Assembly*, InnerMap> StreamCacheMap;
+    StreamCacheMap streamCache;
+    boost::shared_mutex sharedMutex;
+
+    Aya::signals::scoped_connection addingAssemblyConnection;
+    Aya::signals::scoped_connection removedAssemblyConnection;
+
+public:
+    PhysicsPacketCache();
+    ~PhysicsPacketCache();
+
+    bool fetchIfUpToDate(const Assembly* key, unsigned char index, RakNet::BitStream& outBitStream);
+
+    // copy data from bitStream starting at its read position to numBits
+    bool update(const Assembly* key, unsigned char index, RakNet::BitStream& bitStream, unsigned int startReadBitPos, unsigned int numBits);
+
+protected:
+    virtual void onServiceProvider(ServiceProvider* oldProvider, ServiceProvider* newProvider);
+
+private:
+    void insert(const Assembly* key);
+    void insertChildAssembly(const Assembly* assembly);
+    void remove(const Assembly* key);
+    void removeChildAssembly(const Assembly* assembly);
+    void onAddingAssembly(shared_ptr<Instance> assembly);
+    void onRemovedAssembly(shared_ptr<Instance> assembly);
+    void addPart(PartInstance& part);
+
+    boost::unique_lock<boost::shared_mutex> debugLock(boost::upgrade_lock<boost::shared_mutex>* upgradeLock = NULL);
+};
+
+
+extern const char* const sInstancePacketCache;
+class InstancePacketCache
+    : public DescribedNonCreatable<InstancePacketCache, Instance, sInstancePacketCache>
+    , public Service
+{
+    typedef DescribedNonCreatable<InstancePacketCache, Instance, sInstancePacketCache> Super;
+    typedef std::list<Aya::signals::connection> ConnectionList;
+
+    class CachedBitStream
+    {
+    public:
+        bool dirty;
+
+        // regular new instance data containing non dictionary properties, and join time data containing all properties (except parent)
+        boost::shared_ptr<RakNet::BitStream> bitStream[2];
+
+        // debug
+        const std::string guidString;
+
+        Aya::signals::scoped_connection propChangedConnection;
+        Aya::signals::scoped_connection ancestorChangedConnection;
+
+        CachedBitStream(const std::string& guid)
+            : dirty(true)
+            , guidString(guid)
+        {
+        }
+        ~CachedBitStream() {}
+
+        void onPropertyChanged(const Aya::Reflection::PropertyDescriptor* desc)
+        {
+            dirty = true;
+        }
+    };
+
+    typedef boost::unordered_map<const Instance*, boost::shared_ptr<CachedBitStream>> StreamCacheMap;
+    StreamCacheMap streamCache;
+    boost::shared_mutex sharedMutex;
+
+    ConnectionList connections;
+
+    void onAncestorChanged(shared_ptr<Instance> instance, shared_ptr<Instance> newParent);
+
+public:
+    InstancePacketCache();
+    ~InstancePacketCache();
+
+    // not thread safe
+    void insert(const Instance* key);
+
+    // not thread safe
+    void remove(const Instance* key);
+
+    bool fetchIfUpToDate(const Instance* key, RakNet::BitStream& outBitStream, bool isJoinData);
+
+    // copy data from bitStream starting at its read position to numBits
+    bool update(const Instance* key, RakNet::BitStream& bitStream, unsigned int numBits, bool isJoinData);
+
+protected:
+    virtual void onServiceProvider(ServiceProvider* oldProvider, ServiceProvider* newProvider);
+};
+} // namespace Network
+} // namespace Aya

engine/network/src/NetworkProfiler.cpp

@@ -0,0 +1,250 @@
+#include "NetworkProfiler.hpp"
+
+#ifdef NETWORK_PROFILER
+
+#include "Debug.hpp"
+
+#include "RakNetTypes.hpp"
+
+namespace Aya
+{
+namespace Network
+{
+
+const char* const columnNames = "tag, offset, bitSize, layer1, layer2, layer3, layer4, layer5, layer6, layer7, layer8, layer9";
+
+NetworkProfiler::NetworkProfiler(void)
+    : deepestLayer(0)
+    , networkSettings(&NetworkSettings::singleton())
+    , loggerPlugin(NULL)
+{
+}
+
+NetworkProfiler::~NetworkProfiler(void)
+{
+    Disconnect();
+}
+
+bool NetworkProfiler::CanProfile()
+{
+    if (networkSettings->profiling)
+    {
+        Connect();
+        if (networkSettings->profilerTimedSeconds > 0.f)
+        {
+            if (profilerTimer.delta().seconds() > networkSettings->profilerTimedSeconds)
+            {
+                // timeout, stop profiling
+                networkSettings->profiling = false;
+                Disconnect();
+                return false; // note, early return
+            }
+        }
+        return true;
+    }
+    else
+    {
+        Disconnect();
+        return false;
+    }
+}
+
+void NetworkProfiler::Connect()
+{
+    if (!connected)
+    {
+        loggerPlugin = new RakNet::SQLiteClientLoggerPlugin();
+        packetizedTCP.AttachPlugin(loggerPlugin);
+        packetizedTCP.Start(0, 0);
+        RakNet::SystemAddress serverAddress =
+            packetizedTCP.Connect(networkSettings->profilerServerIp.c_str(), networkSettings->profilerServerPort, true);
+        if (serverAddress == RakNet::UNASSIGNED_SYSTEM_ADDRESS)
+        {
+            StandardOut::singleton()->printf(Aya::MESSAGE_ERROR, "Failed to connect to profiler log server.");
+            packetizedTCP.Stop();
+            packetizedTCP.DetachPlugin(loggerPlugin);
+            delete loggerPlugin;
+            networkSettings->profiling = false;
+        }
+        else
+        {
+            loggerPlugin->SetServerParameters(serverAddress, "roblox.db3");
+            connected = true;
+            if (networkSettings->profilerTimedSeconds > 0.f)
+            {
+                profilerTimer.reset();
+            }
+        }
+    }
+}
+
+void NetworkProfiler::Disconnect()
+{
+    if (connected)
+    {
+        loggerPlugin->ClearResultHandlers();
+        packetizedTCP.Stop();
+        packetizedTCP.DetachPlugin(loggerPlugin);
+        delete loggerPlugin;
+        dataBlobStack.clear();
+        connected = false;
+    }
+}
+
+NetworkProfiler* NetworkProfiler::singleton()
+{
+    static NetworkProfiler networkProfiler;
+    return &networkProfiler;
+}
+
+void NetworkProfiler::logPacket(const std::string& type, const RakNet::Packet* packet)
+{
+    if (CanProfile())
+    {
+        const char* profilerTag = networkSettings->profilerTag.c_str();
+        rakSqlLog("general", "tag, packetType, packetBitSize", (profilerTag, type.c_str(), packet->bitSize));
+    }
+}
+
+void NetworkProfiler::startCpuProfiling(int tag)
+{
+    if (networkSettings->profilecpu)
+    {
+        cpuProfilingStats[tag].newSample();
+    }
+}
+
+void NetworkProfiler::stepCpuProfiling(int tag)
+{
+    if (networkSettings->profilecpu)
+    {
+        cpuProfilingStats[tag].step();
+    }
+}
+
+void NetworkProfiler::outputCpuProfiling()
+{
+    for (int tag = 0; tag < PROFILER_TAG_COUNT; tag++)
+    {
+        if (cpuProfilingStats[tag].getNumSample() > 0)
+        {
+            // output the stats
+            StandardOut::singleton()->printf(Aya::MESSAGE_INFO, "[%d] profiling results (%d samples):", tag, cpuProfilingStats[tag].getNumSample());
+            float lastStepDelta = 0.0f;
+            for (int i = 0; cpuProfilingStats[tag].stepDelta[i] > 0.f; i++)
+            {
+                float deltaBetweenSteps;
+                float stepDelta = cpuProfilingStats[tag].stepDelta[i];
+                deltaBetweenSteps = stepDelta - lastStepDelta;
+                lastStepDelta = stepDelta;
+                StandardOut::singleton()->printf(Aya::MESSAGE_INFO, "Step %d: %f (delta %f, total %f)", i + 1, stepDelta, deltaBetweenSteps,
+                    deltaBetweenSteps * cpuProfilingStats[tag].getNumSample());
+            }
+        }
+    }
+}
+
+void NetworkProfiler::startProfiling(const std::string& dataBlobName, const RakNet::BitStream* bitStream)
+{
+    if (CanProfile())
+    {
+        dataBlobStack.push_back(DataBlobInfo(dataBlobName, bitStream->GetReadOffset()));
+        if (dataBlobStack.size() > deepestLayer)
+        {
+            deepestLayer = dataBlobStack.size();
+        }
+    }
+}
+
+void NetworkProfiler::endProfiling(const std::string& dataBlobName, const RakNet::BitStream* bitStream)
+{
+    if (CanProfile())
+    {
+        DataBlobInfo dataBlobInfo = dataBlobStack.back();
+        AYAASSERT(dataBlobName == dataBlobInfo.name); // Make sure startProfiling and endProfiling are always in pairs
+        AYAASSERT(dataBlobStack.size() > 0);
+        const char* profilerTag = networkSettings->profilerTag.c_str();
+        if (dataBlobStack.size() == deepestLayer) // only log the leaf
+        {
+            if (!NetworkSettings::singleton().profilerOneColumnPerStack)
+            {
+                std::string layers = "";
+                for (std::vector<DataBlobInfo>::iterator iter = dataBlobStack.begin(); iter != dataBlobStack.end(); iter++)
+                {
+                    layers += ((*iter).name + ".");
+                }
+                rakSqlLog("report", "tag, offset, bitsize, layers",
+                    (profilerTag, dataBlobInfo.bitStreamOffset, bitStream->GetReadOffset() - dataBlobInfo.bitStreamOffset, layers.c_str()));
+            }
+            else
+            {
+                switch (dataBlobStack.size())
+                {
+                case 1:
+                    rakSqlLog("details", columnNames,
+                        (profilerTag, dataBlobInfo.bitStreamOffset, bitStream->GetReadOffset() - dataBlobInfo.bitStreamOffset,
+                            dataBlobStack.at(0).name.c_str(), "", "", "", "", "", "", "", ""));
+                    break;
+                case 2:
+                    rakSqlLog("details", columnNames,
+                        (profilerTag, dataBlobInfo.bitStreamOffset, bitStream->GetReadOffset() - dataBlobInfo.bitStreamOffset,
+                            dataBlobStack.at(0).name.c_str(), dataBlobStack.at(1).name.c_str(), "", "", "", "", "", "", ""));
+                    break;
+                case 3:
+                    rakSqlLog("details", columnNames,
+                        (profilerTag, dataBlobInfo.bitStreamOffset, bitStream->GetReadOffset() - dataBlobInfo.bitStreamOffset,
+                            dataBlobStack.at(0).name.c_str(), dataBlobStack.at(1).name.c_str(), dataBlobStack.at(2).name.c_str(), "", "", "", "", "",
+                            ""));
+                    break;
+                case 4:
+                    rakSqlLog("details", columnNames,
+                        (profilerTag, dataBlobInfo.bitStreamOffset, bitStream->GetReadOffset() - dataBlobInfo.bitStreamOffset,
+                            dataBlobStack.at(0).name.c_str(), dataBlobStack.at(1).name.c_str(), dataBlobStack.at(2).name.c_str(),
+                            dataBlobStack.at(3).name.c_str(), "", "", "", "", ""));
+                    break;
+                case 5:
+                    rakSqlLog("details", columnNames,
+                        (profilerTag, dataBlobInfo.bitStreamOffset, bitStream->GetReadOffset() - dataBlobInfo.bitStreamOffset,
+                            dataBlobStack.at(0).name.c_str(), dataBlobStack.at(1).name.c_str(), dataBlobStack.at(2).name.c_str(),
+                            dataBlobStack.at(3).name.c_str(), dataBlobStack.at(4).name.c_str(), "", "", "", ""));
+                    break;
+                case 6:
+                    rakSqlLog("details", columnNames,
+                        (profilerTag, dataBlobInfo.bitStreamOffset, bitStream->GetReadOffset() - dataBlobInfo.bitStreamOffset,
+                            dataBlobStack.at(0).name.c_str(), dataBlobStack.at(1).name.c_str(), dataBlobStack.at(2).name.c_str(),
+                            dataBlobStack.at(3).name.c_str(), dataBlobStack.at(4).name.c_str(), dataBlobStack.at(5).name.c_str(), "", "", ""));
+                    break;
+                case 7:
+                    rakSqlLog("details", columnNames,
+                        (profilerTag, dataBlobInfo.bitStreamOffset, bitStream->GetReadOffset() - dataBlobInfo.bitStreamOffset,
+                            dataBlobStack.at(0).name.c_str(), dataBlobStack.at(1).name.c_str(), dataBlobStack.at(2).name.c_str(),
+                            dataBlobStack.at(3).name.c_str(), dataBlobStack.at(4).name.c_str(), dataBlobStack.at(5).name.c_str(),
+                            dataBlobStack.at(6).name.c_str(), "", ""));
+                    break;
+                case 8:
+                    rakSqlLog("details", columnNames,
+                        (profilerTag, dataBlobInfo.bitStreamOffset, bitStream->GetReadOffset() - dataBlobInfo.bitStreamOffset,
+                            dataBlobStack.at(0).name.c_str(), dataBlobStack.at(1).name.c_str(), dataBlobStack.at(2).name.c_str(),
+                            dataBlobStack.at(3).name.c_str(), dataBlobStack.at(4).name.c_str(), dataBlobStack.at(5).name.c_str(),
+                            dataBlobStack.at(6).name.c_str(), dataBlobStack.at(7).name.c_str(), ""));
+                    break;
+                case 9:
+                    rakSqlLog("details", columnNames,
+                        (profilerTag, dataBlobInfo.bitStreamOffset, bitStream->GetReadOffset() - dataBlobInfo.bitStreamOffset,
+                            dataBlobStack.at(0).name.c_str(), dataBlobStack.at(1).name.c_str(), dataBlobStack.at(2).name.c_str(),
+                            dataBlobStack.at(3).name.c_str(), dataBlobStack.at(4).name.c_str(), dataBlobStack.at(5).name.c_str(),
+                            dataBlobStack.at(6).name.c_str(), dataBlobStack.at(7).name.c_str(), dataBlobStack.at(8).name.c_str()));
+                    break;
+                default:
+                    AYAASSERT(false && "Overflowing the network profiler data layer stack (9). Maybe add more?");
+                }
+            }
+        }
+        deepestLayer = dataBlobStack.size(); // this will prevent the non-leaf node to be printed
+        dataBlobStack.pop_back();
+    }
+}
+} // namespace Network
+} // namespace Aya
+
+#endif

engine/network/src/NetworkProfiler.hpp

@@ -0,0 +1,137 @@
+#pragma once
+
+#include "RakNet/SQLite3Plugin/SQLite3ClientPlugin.hpp"
+#include "RakNet/SQLite3Plugin/SQLiteClientLoggerPlugin.hpp"
+#include "RakNet/PacketizedTCP.hpp"
+#include "NetworkSettings.hpp"
+#include "Util.hpp"
+
+#ifdef NETWORK_PROFILER
+#define NETPROFILE_LOG(typeStr, packetPtr) Aya::Network::NetworkProfiler::singleton()->logPacket(typeStr, packetPtr)
+#define NETPROFILE_START(dataBlobNameStr, bitStreamPtr) Aya::Network::NetworkProfiler::singleton()->startProfiling(dataBlobNameStr, bitStreamPtr)
+#define NETPROFILE_END(dataBlobNameStr, bitStreamPtr) Aya::Network::NetworkProfiler::singleton()->endProfiling(dataBlobNameStr, bitStreamPtr)
+#define CPUPROFILER_START(tag) Aya::Network::NetworkProfiler::singleton()->startCpuProfiling(tag);
+#define CPUPROFILER_STEP(tag) Aya::Network::NetworkProfiler::singleton()->stepCpuProfiling(tag);
+#define CPUPROFILER_OUTPUT() Aya::Network::NetworkProfiler::singleton()->outputCpuProfiling();
+#else
+#define NETPROFILE_LOG(typeStr, packetPtr)
+#define NETPROFILE_START(dataBlobNameStr, bitStreamPtr)
+#define NETPROFILE_END(dataBlobNameStr, bitStreamPtr)
+#define CPUPROFILER_START(tag)
+#define CPUPROFILER_STEP(tag)
+#define CPUPROFILER_OUTPUT()
+#endif
+
+#ifdef NETWORK_PROFILER
+
+namespace Aya
+{
+namespace Network
+{
+
+class NetworkProfiler
+{
+public:
+    class DataBlobInfo
+    {
+    public:
+        DataBlobInfo(const std::string& _name, RakNet::BitSize_t _bitStreamOffset)
+        {
+            name = _name;
+            bitStreamOffset = _bitStreamOffset;
+        }
+        std::string name;
+        RakNet::BitSize_t bitStreamOffset;
+    };
+
+    enum ProfilerTags
+    {
+        PROFILER_streamOutPart,
+        PROFILER_jointRemoval,
+        PROFILER_gcStep,
+        PROFILER_TAG_3,
+        PROFILER_TAG_4,
+        PROFILER_TAG_5,
+        PROFILER_TAG_6,
+        PROFILER_TAG_7,
+        PROFILER_TAG_8,
+        PROFILER_TAG_9,
+        PROFILER_TAG_COUNT
+    };
+
+    class CpuProfilingStat
+    {
+        int currentStep;
+        int numSample;
+        Aya::Timer<Aya::Time::Precise> timer;
+
+    public:
+        double stepDelta[256]; // maximum 256 steps
+        CpuProfilingStat()
+        {
+            reset();
+        }
+        inline void newSample()
+        {
+            timer.reset();
+            numSample++;
+            currentStep = 0;
+        }
+        inline void step()
+        {
+            AYAASSERT(numSample > 0);
+            AYAASSERT(currentStep < 256);
+            stepDelta[currentStep] = (stepDelta[currentStep] * (numSample - 1) + timer.delta().seconds()) / numSample;
+            currentStep++;
+        }
+        int getNumSample()
+        {
+            return numSample;
+        }
+        void reset()
+        {
+            for (int i = 0; i < numSample; i++)
+            {
+                stepDelta[i] = 0.0f;
+            }
+            currentStep = 0;
+            numSample = 0;
+        }
+    };
+
+    CpuProfilingStat cpuProfilingStats[PROFILER_TAG_COUNT];
+
+    static NetworkProfiler* singleton();
+    void logPacket(const std::string& type, const RakNet::Packet* packet);
+
+    void startProfiling(const std::string& dataBlobName, const RakNet::BitStream* bitStream);
+    void endProfiling(const std::string& dataBlobName, const RakNet::BitStream* bitStream);
+
+    void startCpuProfiling(int);
+    void stepCpuProfiling(int);
+    void outputCpuProfiling();
+
+    virtual ~NetworkProfiler(void);
+
+private:
+    // members
+    RakNet::PacketizedTCP packetizedTCP;
+    RakNet::SQLiteClientLoggerPlugin* loggerPlugin;
+    const RakNet::BitStream* currentBitStream;
+    std::vector<DataBlobInfo> dataBlobStack; // use vector to make use of its iterator
+    std::size_t deepestLayer;
+    bool connected;
+    Aya::Timer<Aya::Time::Fast> profilerTimer;
+    NetworkSettings* networkSettings;
+
+    // functions
+    NetworkProfiler(void);
+    void Connect();
+    void Disconnect();
+    bool CanProfile();
+};
+
+} // namespace Network
+} // namespace Aya
+
+#endif

engine/network/src/NetworkSettings.cpp

@@ -0,0 +1,423 @@
+#include "NetworkSettings.hpp"
+#include "RakNet/ReliabilityLayer.hpp"
+#include "Utility/Statistics.hpp"
+
+#include "NetworkProfiler.hpp"
+#include "Utility/MemoryStats.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+
+namespace Aya
+{
+const char* const sNetworkSettings = "NetworkSettings";
+
+namespace Reflection
+{
+template<>
+EnumDesc<NetworkSettings::PhysicsSendMethod>::EnumDesc()
+    : EnumDescriptor("PhysicsSendMethod")
+{
+    addPair(NetworkSettings::ErrorComputation, "ErrorComputation");
+    addPair(NetworkSettings::ErrorComputation2, "ErrorComputation2");
+    addPair(NetworkSettings::RoundRobin, "RoundRobin");
+    addPair(NetworkSettings::TopNErrors, "TopNErrors");
+}
+
+template<>
+EnumDesc<NetworkSettings::PhysicsReceiveMethod>::EnumDesc()
+    : EnumDescriptor("PhysicsReceiveMethod")
+{
+    addPair(NetworkSettings::Direct, "Direct");
+    addPair(NetworkSettings::Interpolation, "Interpolation");
+}
+
+template<>
+EnumDesc<PacketReliability>::EnumDesc()
+    : EnumDescriptor("PacketReliability")
+{
+    addPair(UNRELIABLE, "UNRELIABLE");
+    addPair(UNRELIABLE_SEQUENCED, "UNRELIABLE_SEQUENCED");
+    addPair(RELIABLE, "RELIABLE");
+    addPair(RELIABLE_ORDERED, "RELIABLE_ORDERED");
+    addPair(RELIABLE_SEQUENCED, "RELIABLE_SEQUENCED");
+}
+
+template<>
+EnumDesc<PacketPriority>::EnumDesc()
+    : EnumDescriptor("PacketPriority")
+{
+    addPair(IMMEDIATE_PRIORITY, "IMMEDIATE_PRIORITY");
+    addPair(HIGH_PRIORITY, "HIGH_PRIORITY");
+    addPair(MEDIUM_PRIORITY, "MEDIUM_PRIORITY");
+    addPair(LOW_PRIORITY, "LOW_PRIORITY");
+}
+
+} // namespace Reflection
+
+
+static Reflection::BoundProp<int> prop_PreferredClientPort("PreferredClientPort", "Network", &NetworkSettings::preferredClientPort);
+static Reflection::PropDescriptor<NetworkSettings, float> prop_DataSendRate(
+    "DataSendRate", "Network", &NetworkSettings::getDataSendRate, &NetworkSettings::setDataSendRate);
+static Reflection::PropDescriptor<NetworkSettings, float> prop_DataGCRate(
+    "DataGCRate", "Network", &NetworkSettings::getDataGCRate, &NetworkSettings::setDataGCRate);
+static Reflection::PropDescriptor<NetworkSettings, float> prop_PhysicsSendRate(
+    "PhysicsSendRate", "Network", &NetworkSettings::getPhysicsSendRate, &NetworkSettings::setPhysicsSendRate);
+static Reflection::PropDescriptor<NetworkSettings, float> prop_ClientPhysicsSendRate(
+    "ClientPhysicsSendRate", "Network", &NetworkSettings::getClientPhysicsSendRate, &NetworkSettings::setClientPhysicsSendRate);
+static Reflection::BoundProp<float> prop_SendRate("NetworkOwnerRate", "Network", &NetworkSettings::networkOwnerRate);
+static Reflection::PropDescriptor<NetworkSettings, float> prop_TouchSendRate(
+    "TouchSendRate", "Network", &NetworkSettings::getTouchSendRate, &NetworkSettings::setTouchSendRate);
+static Reflection::BoundProp<bool> prop_isThrottledByOutgoingBandwidthLimit(
+    "IsThrottledByOutgoingBandwidthLimit", "Network", &NetworkSettings::isThrottledByOutgoingBandwidthLimit);
+static Reflection::BoundProp<bool> prop_IsThrottledByCongestionControl(
+    "IsThrottledByCongestionControl", "Network", &NetworkSettings::isThrottledByCongestionControl);
+static Reflection::PropDescriptor<NetworkSettings, double> prop_ReceiveRate(
+    "ReceiveRate", "Network", &NetworkSettings::getReceiveRate, &NetworkSettings::setReceiveRate);
+static Reflection::BoundProp<int> prop_WaitingForCharacterLogRate("WaitingForCharacterLogRate", "Network",
+    &NetworkSettings::deprecatedWaitingForCharacterLogRate,
+    Reflection::PropertyDescriptor::Attributes::deprecated(Reflection::PropertyDescriptor::HIDDEN_SCRIPTING));
+static Reflection::PropDescriptor<NetworkSettings, std::string> prop_ReportStatUrl("ReportStatURL", "Network", &NetworkSettings::getReportStatURL,
+    &NetworkSettings::setReportStatURL, Reflection::PropertyDescriptor::Attributes::deprecated(Reflection::PropertyDescriptor::HIDDEN_SCRIPTING));
+static Reflection::BoundProp<bool> prop_UsePhysicsPacketCache("UsePhysicsPacketCache", "Network", &NetworkSettings::usePhysicsPacketCache);
+static Reflection::BoundProp<bool> prop_UseInstancePacketCache("UseInstancePacketCache", "Network", &NetworkSettings::useInstancePacketCache);
+static Reflection::BoundProp<bool> propIsQueueErrorComputed("IsQueueErrorComputed", "Network", &NetworkSettings::isQueueErrorComputed);
+
+static Reflection::BoundProp<bool> prop_TrackDataTypes("TrackDataTypes", "Diagnostics", &NetworkSettings::trackDataTypes);
+static Reflection::BoundProp<bool> prop_TrackPhysicsDetails("TrackPhysicsDetails", "Diagnostics", &NetworkSettings::trackPhysicsDetails);
+static Reflection::BoundProp<bool> prop_PrintSplitMessages("PrintSplitMessage", "Diagnostics", &NetworkSettings::printSplitMessages);
+static Reflection::BoundProp<bool> prop_PrintTouches("PrintTouches", "Diagnostics", &NetworkSettings::printTouches);
+static Reflection::BoundProp<bool> prop_PrintInstances("PrintInstances", "Diagnostics", &NetworkSettings::printInstances);
+static Reflection::BoundProp<bool> prop_PrintStreamInstanceQuota(
+    "PrintStreamInstanceQuota", "Diagnostics", &NetworkSettings::printStreamInstanceQuota);
+static Reflection::BoundProp<bool> prop_PrintProperties("PrintProperties", "Diagnostics", &NetworkSettings::printProperties);
+static Reflection::BoundProp<bool> prop_PrintPhysicsErrors("PrintPhysicsErrors", "Diagnostics", &NetworkSettings::printPhysicsErrors);
+static Reflection::BoundProp<bool> prop_PrintDataFilters("PrintFilters", "Diagnostics", &NetworkSettings::printDataFilters);
+static Reflection::BoundProp<bool> prop_ArePhysicsRejectionsReported(
+    "ArePhysicsRejectionsReported", "Diagnostics", &NetworkSettings::printPhysicsFilters);
+static Reflection::BoundProp<bool> propPrintEvents("PrintEvents", "Diagnostics", &NetworkSettings::printEvents);
+static Reflection::BoundProp<bool> propPrintBits("PrintBits", "Diagnostics", &NetworkSettings::printBits);
+static Reflection::BoundProp<double> prop_incommingReplicationLag(
+    "IncommingReplicationLag", "Diagnostics", &NetworkSettings::incommingReplicationLag);
+
+static Reflection::EnumPropDescriptor<NetworkSettings, NetworkSettings::PhysicsSendMethod> prop_PhysicsSend(
+    "PhysicsSend", "Physics", &NetworkSettings::getPhysicsSendMethod, &NetworkSettings::setPhysicsSendMethod);
+static Reflection::EnumPropDescriptor<NetworkSettings, NetworkSettings::PhysicsReceiveMethod> prop_PhysicsReceive(
+    "PhysicsReceive", "Physics", &NetworkSettings::dummyGetPhysicsReceiveMethod, &NetworkSettings::dummySetPhysicsReceiveMethod);
+static Reflection::EnumPropDescriptor<NetworkSettings, PacketPriority> prop_PhysicsSendPriority("PhysicsSendPriority", "Physics",
+    &NetworkSettings::getPhysicsSendPriority, &NetworkSettings::setPhysicsSendPriority, Reflection::PropertyDescriptor::HIDDEN_SCRIPTING);
+
+Reflection::BoundProp<bool> NetworkSettings::prop_DistributedPhysics(
+    "ExperimentalPhysicsEnabled", "Physics", &NetworkSettings::distributedPhysicsEnabled);
+
+static Reflection::PropDescriptor<NetworkSettings, int> prop_PhysicsMtuAdjust(
+    "PhysicsMtuAdjust", "Physics", &NetworkSettings::getPhysicsMtuAdjust, &NetworkSettings::setPhysicsMtuAdjust);
+static Reflection::PropDescriptor<NetworkSettings, int> prop_ReplicationMtuAdjust(
+    "DataMtuAdjust", "Data", &NetworkSettings::getReplicationMtuAdjust, &NetworkSettings::setReplicationMtuAdjust);
+static Reflection::BoundProp<int> propCanSendPacketBufferLimit("CanSendPacketBufferLimit", "Data", &NetworkSettings::canSendPacketBufferLimit);
+static Reflection::BoundProp<int> propSendPacketBufferLimit("SendPacketBufferLimit", "Data", &NetworkSettings::sendPacketBufferLimit);
+static Reflection::BoundProp<int> propMaxDataModelSendBuffer("MaxDataModelSendBuffer", "Data", &NetworkSettings::canSendPacketBufferLimit,
+    Reflection::PropertyDescriptor::Attributes::deprecated(propCanSendPacketBufferLimit));
+static Reflection::EnumPropDescriptor<NetworkSettings, PacketPriority> prop_DataSendPriority("DataSendPriority", "Data",
+    &NetworkSettings::getDataSendPriority, &NetworkSettings::setDataSendPriority, Reflection::PropertyDescriptor::HIDDEN_SCRIPTING);
+
+#ifdef NETWORK_PROFILER
+static Reflection::BoundProp<bool> prop_Profiling("Profiling", "Profiler", &NetworkSettings::profiling, Reflection::PropertyDescriptor::UI);
+static Reflection::BoundProp<bool> prop_ProfilingCpu("ProfileCpu", "Profiler", &NetworkSettings::profilecpu);
+static Reflection::BoundProp<bool> prop_ProfilerOneColumnPerStack(
+    "ProfilerOneColumnPerStack", "Profiler", &NetworkSettings::profilerOneColumnPerStack, Reflection::PropertyDescriptor::UI);
+static Reflection::BoundProp<std::string> prop_ProfilerServerIp("ProfilerServerIp", "Profiler", &NetworkSettings::profilerServerIp);
+static Reflection::BoundProp<int> prop_ProfilerServerPort("ProfilerServerPort", "Profiler", &NetworkSettings::profilerServerPort);
+static Reflection::BoundProp<std::string> prop_ProfilerTag(
+    "ProfilerTag", "Profiler", &NetworkSettings::profilerTag, Reflection::PropertyDescriptor::UI);
+static Reflection::BoundProp<double> prop_ProfilerTimedSeconds("ProfilerTimedSeconds", "Profiler", &NetworkSettings::profilerTimedSeconds);
+static Reflection::BoundFuncDesc<NetworkSettings, void()> func_printProfilingResult(
+    &NetworkSettings::printProfilingResult, "PrintProfilingResult", Security::None);
+#endif
+
+#ifdef NETWORK_PROFILER
+static Reflection::BoundProp<bool> prop_EnableHeavyCompression(
+    "EnableHeavyCompression", "Optimization", &NetworkSettings::enableHeavyCompression, Reflection::PropertyDescriptor::UI);
+#else
+static Reflection::BoundProp<bool> prop_EnableHeavyCompression(
+    "EnableHeavyCompression", "Optimization", &NetworkSettings::enableHeavyCompression, Reflection::PropertyDescriptor::HIDDEN_SCRIPTING);
+#endif
+
+static Reflection::PropDescriptor<NetworkSettings, int> prop_extraMemoryUsed("ExtraMemoryUsed", category_Data,
+    &NetworkSettings::getExtraMemoryUsedInMB, &NetworkSettings::setExtraMemoryUsedInMB, Reflection::PropertyDescriptor::HIDDEN_SCRIPTING,
+    Security::Plugin);
+static Reflection::PropDescriptor<NetworkSettings, float> prop_freeMemoryMBytes("FreeMemoryMBytes", category_Data,
+    &NetworkSettings::getFreeMemoryMBytes, NULL, Reflection::PropertyDescriptor::HIDDEN_SCRIPTING, Security::Plugin);
+static Reflection::PropDescriptor<NetworkSettings, float> prop_freeMemoryPoolMBytes("FreeMemoryPoolMBytes", category_Data,
+    &NetworkSettings::getFreeMemoryPoolMBytes, NULL, Reflection::PropertyDescriptor::HIDDEN_SCRIPTING, Security::Plugin);
+static Reflection::PropDescriptor<NetworkSettings, bool> prop_RenderRegions(
+    "RenderStreamedRegions", category_Appearance, &NetworkSettings::getRenderStreamedRegions, &NetworkSettings::setRenderStreamedRegions);
+static Reflection::PropDescriptor<NetworkSettings, bool> prop_ShowPartMovementPath(
+    "ShowPartMovementWayPoint", category_Appearance, &NetworkSettings::getShowPartMovementPath, &NetworkSettings::setShowPartMovementPath);
+static Reflection::PropDescriptor<NetworkSettings, int> prop_TotalNumMovementWayPoint(
+    "TotalNumMovementWayPoint", category_Appearance, &NetworkSettings::getTotalNumMovementWayPoint, &NetworkSettings::setTotalNumMovementWayPoint);
+#ifdef AYA_TEST_BUILD
+static Reflection::PropDescriptor<NetworkSettings, int> prop_TaskSchedulerFindJobFPS(
+    "TaskSchedulerFindJobFPS", category_Appearance, &NetworkSettings::getTaskSchedulerFindJobFPS, &NetworkSettings::setTaskSchedulerFindJobFPS);
+static Reflection::PropDescriptor<NetworkSettings, bool> prop_TaskSchedulerUpdateJobPriorityOnWake("TaskSchedulerUpdateJobPriorityOnWake",
+    category_Appearance, &NetworkSettings::getTaskSchedulerUpdateJobPriorityOnWake, &NetworkSettings::setTaskSchedulerUpdateJobPriorityOnWake);
+#endif
+static Reflection::PropDescriptor<NetworkSettings, bool> prop_ShowActiveAnimationAsset(
+    "ShowActiveAnimationAsset", category_Appearance, &NetworkSettings::getShowActiveAnimationAsset, &NetworkSettings::setShowActiveAnimationAsset);
+REFLECTION_END();
+
+NetworkSettings::NetworkSettings(void)
+    : printInstances(false)
+    , printStreamInstanceQuota(false)
+    , printTouches(false)
+    , distributedPhysicsEnabled(true)
+    , printPhysicsErrors(false)
+    , printPhysicsFilters(false)
+    , printProperties(false)
+    , printEvents(false)
+    , printDataFilters(false)
+    , printBits(false)
+    , incommingReplicationLag(0.0)
+    , isQueueErrorComputed(true)
+    , usePhysicsPacketCache(false)
+    , useInstancePacketCache(false)
+    , canSendPacketBufferLimit(1)
+    , sendPacketBufferLimit(-1) // default is to not check it
+    , physicsMtuAdjust(-200)
+    , preferredClientPort(0)
+    , replicationMtuAdjust(-200)
+    , dataSendRate(30.0f)
+    , physicsSendRate(20.0f)
+    , clientPhysicsSendRate(20.0f)
+    , dataGCRate(20.0f)
+    , networkOwnerRate(10.0f)
+    , touchSendRate(10.0f)
+    , isThrottledByOutgoingBandwidthLimit(false)
+    , isThrottledByCongestionControl(false)
+    , receiveRate(60.0f)
+    , physicsSendMethod(
+          TopNErrors) // default to TopNErrors as unit tests will be using the default value, if we are going to use other types of senders please
+                      // make coordinating changes in PhysicsReceiver as it is expecting packets from either RoundRobin sender or TopNError sender
+    , physicsSendPriority(HIGH_PRIORITY)
+    , dataSendPriority(MEDIUM_PRIORITY)
+    , physicsReceiveMethod(Interpolation)
+    , printSplitMessages(false)
+    , trackDataTypes(false)
+    , trackPhysicsDetails(false)
+    , profiling(false)
+    , profilecpu(false)
+    , profilerOneColumnPerStack(false)
+    , profilerTag("")
+    , profilerServerIp("127.0.0.1")
+    , profilerServerPort(38123)
+    , profilerTimedSeconds(0.f)
+    , enableHeavyCompression(true)
+    , extraMemoryUsed(0)
+    , totalNumMovementWayPoint(1000)
+{
+    setName("Network");
+}
+
+template<typename A>
+static A clamp(const A& min, const A& value, const A& max)
+{
+    return std::max(min, std::min(value, max));
+}
+
+void NetworkSettings::setPhysicsMtuAdjust(int value)
+{
+    value = clamp(-1000, value, 0);
+    if (value != physicsMtuAdjust)
+    {
+        physicsMtuAdjust = value;
+        raisePropertyChanged(prop_PhysicsMtuAdjust);
+    }
+}
+void NetworkSettings::setReplicationMtuAdjust(int value)
+{
+    value = clamp(-1000, value, 0);
+    if (value != replicationMtuAdjust)
+    {
+        replicationMtuAdjust = value;
+        raisePropertyChanged(prop_ReplicationMtuAdjust);
+    }
+}
+void NetworkSettings::setDataSendRate(float value)
+{
+    value = clamp(5.0f, value, 120.0f);
+    if (value != dataSendRate)
+    {
+        dataSendRate = value;
+        raisePropertyChanged(prop_DataSendRate);
+    }
+}
+void NetworkSettings::setDataGCRate(float value)
+{
+    value = clamp(2.0f, value, 60.0f);
+    if (value != dataGCRate)
+    {
+        dataGCRate = value;
+        raisePropertyChanged(prop_DataGCRate);
+    }
+}
+void NetworkSettings::setPhysicsSendRate(float value)
+{
+    value = clamp(0.1f, value, 120.0f);
+    if (value != physicsSendRate)
+    {
+        physicsSendRate = value;
+        raisePropertyChanged(prop_PhysicsSendRate);
+    }
+}
+
+void NetworkSettings::setClientPhysicsSendRate(float value)
+{
+    value = clamp(0.1f, value, 120.0f);
+    if (value != clientPhysicsSendRate)
+    {
+        clientPhysicsSendRate = value;
+        raisePropertyChanged(prop_ClientPhysicsSendRate);
+    }
+}
+
+void NetworkSettings::setTouchSendRate(float value)
+{
+    value = clamp(1.0f, value, 120.0f);
+    if (value != touchSendRate)
+    {
+        touchSendRate = value;
+        raisePropertyChanged(prop_TouchSendRate);
+    }
+}
+
+void NetworkSettings::setReceiveRate(double value)
+{
+    value = clamp(5.0, value, 120.0);
+    if (value != receiveRate)
+    {
+        receiveRate = value;
+        raisePropertyChanged(prop_ReceiveRate);
+    }
+}
+
+void NetworkSettings::setPhysicsSendMethod(const PhysicsSendMethod& value)
+{
+    if (value != physicsSendMethod)
+    {
+        physicsSendMethod = value;
+        raiseChanged(prop_PhysicsSend);
+    }
+}
+
+void NetworkSettings::setPhysicsSendPriority(const PacketPriority& value)
+{
+    if (value != physicsSendPriority)
+    {
+        physicsSendPriority = value;
+        raiseChanged(prop_PhysicsSendPriority);
+    }
+}
+
+void NetworkSettings::setDataSendPriority(const PacketPriority& value)
+{
+    if (value != dataSendPriority)
+    {
+        dataSendPriority = value;
+        raiseChanged(prop_DataSendPriority);
+    }
+}
+
+void NetworkSettings::dummySetPhysicsReceiveMethod(const PhysicsReceiveMethod& value)
+{
+    if (value != physicsReceiveMethod)
+    {
+        physicsReceiveMethod = value;
+        raiseChanged(prop_PhysicsReceive);
+    }
+}
+
+bool NetworkSettings::heavyCompressionEnabled()
+{
+    return enableHeavyCompression;
+}
+
+void NetworkSettings::setExtraMemoryUsedInMB(int value)
+{
+    if (value < 0)
+        value = 0;
+
+    if (value != extraMemoryUsed)
+    {
+        extraMemoryUsed = value;
+        raiseChanged(prop_extraMemoryUsed);
+    }
+}
+
+float NetworkSettings::getFreeMemoryMBytes() const
+{
+    return MemoryStats::freeMemoryBytes() / 1024.0f / 1024.0f;
+}
+
+float NetworkSettings::getFreeMemoryPoolMBytes() const
+{
+    return MemoryStats::slowGetMemoryPoolAvailability() / 1024.0f / 1024.0f;
+}
+
+void NetworkSettings::setRenderStreamedRegions(bool value)
+{
+    if (value != Workspace::showStreamedRegions)
+    {
+        Workspace::showStreamedRegions = value;
+        raiseChanged(prop_RenderRegions);
+    }
+}
+
+bool NetworkSettings::getRenderStreamedRegions() const
+{
+    return Workspace::showStreamedRegions;
+}
+
+void NetworkSettings::setShowPartMovementPath(bool value)
+{
+    if (value != Workspace::showPartMovementPath)
+    {
+        Workspace::showPartMovementPath = value;
+    }
+}
+
+bool NetworkSettings::getShowPartMovementPath() const
+{
+    return Workspace::showPartMovementPath;
+}
+
+void NetworkSettings::setTotalNumMovementWayPoint(int value)
+{
+    if (value < 0)
+        value = 0;
+    if (value > 1000000)
+        value = 1000000;
+
+    if (value != totalNumMovementWayPoint)
+    {
+        totalNumMovementWayPoint = value;
+    }
+}
+
+bool NetworkSettings::getShowActiveAnimationAsset() const
+{
+    return Workspace::showActiveAnimationAsset;
+}
+
+void NetworkSettings::setShowActiveAnimationAsset(bool value)
+{
+    if (value != Workspace::showActiveAnimationAsset)
+    {
+        Workspace::showActiveAnimationAsset = value;
+    }
+}
+
+void NetworkSettings::printProfilingResult()
+{
+    CPUPROFILER_OUTPUT();
+}
+} // namespace Aya

engine/network/src/NetworkSettings.hpp

@@ -0,0 +1,226 @@
+#pragma once
+
+#include "DataModel/GlobalSettings.hpp"
+
+#include "RakNet/PacketPriority.hpp"
+#include "TaskScheduler.hpp"
+
+#include "Utility/ObscureValue.hpp"
+
+
+namespace Aya
+{
+extern const char* const sNetworkSettings;
+class NetworkSettings : public GlobalAdvancedSettingsItem<NetworkSettings, sNetworkSettings>
+{
+public:
+    NetworkSettings();
+
+    bool printSplitMessages;
+    bool printPhysicsErrors;
+    bool printInstances;
+    bool printStreamInstanceQuota;
+    bool printTouches;
+    bool printProperties;
+    bool printEvents;
+    bool printPhysicsFilters;
+    bool printDataFilters;
+    bool printBits;
+    bool trackDataTypes;
+    bool trackPhysicsDetails;
+    double incommingReplicationLag;
+    typedef enum
+    {
+        ErrorComputation,
+        ErrorComputation2,
+        RoundRobin,
+        TopNErrors
+    } PhysicsSendMethod;
+    typedef enum
+    {
+        Direct,
+        Interpolation
+    } PhysicsReceiveMethod;
+
+    bool isQueueErrorComputed;
+
+    bool usePhysicsPacketCache;
+    bool useInstancePacketCache;
+
+    bool profiling;
+    bool profilecpu;
+    bool profilerOneColumnPerStack;
+    std::string profilerServerIp;
+    int profilerServerPort;
+    std::string profilerTag;
+    double profilerTimedSeconds; // 0 or negative for un-timed profiling
+    bool enableHeavyCompression;
+    bool heavyCompressionEnabled();
+
+private:
+    PhysicsSendMethod physicsSendMethod;
+    PhysicsReceiveMethod physicsReceiveMethod;
+
+    PacketPriority physicsSendPriority;
+    PacketPriority dataSendPriority;
+
+public:
+    PhysicsSendMethod getPhysicsSendMethod() const
+    {
+        return physicsSendMethod;
+    }
+    void setPhysicsSendMethod(const PhysicsSendMethod& value);
+
+    PacketPriority getPhysicsSendPriority() const
+    {
+        return physicsSendPriority;
+    }
+    void setPhysicsSendPriority(const PacketPriority& value);
+
+    PacketPriority getDataSendPriority() const
+    {
+        return dataSendPriority;
+    }
+    void setDataSendPriority(const PacketPriority& value);
+
+    PhysicsReceiveMethod dummyGetPhysicsReceiveMethod() const
+    {
+        return physicsReceiveMethod;
+    }
+    void dummySetPhysicsReceiveMethod(const PhysicsReceiveMethod& value);
+
+    const std::string getReportStatURL() const
+    {
+        return "";
+    }
+    void setReportStatURL(const std::string& value) {}
+
+    static Reflection::BoundProp<bool> prop_DistributedPhysics;
+    bool distributedPhysicsEnabled;
+
+    int preferredClientPort;
+
+    int getPhysicsMtuAdjust() const
+    {
+        return physicsMtuAdjust;
+    }
+    void setPhysicsMtuAdjust(int value);
+
+    int getReplicationMtuAdjust() const
+    {
+        return replicationMtuAdjust;
+    }
+    void setReplicationMtuAdjust(int value);
+
+    // Data-out throttle
+    float getDataSendRate() const
+    {
+        return dataSendRate;
+    }
+    void setDataSendRate(float value);
+
+    float getDataGCRate() const
+    {
+        return dataGCRate;
+    }
+    void setDataGCRate(float value);
+
+    float getPhysicsSendRate() const
+    {
+        return physicsSendRate;
+    }
+    void setPhysicsSendRate(float value);
+
+    float getClientPhysicsSendRate() const
+    {
+        return clientPhysicsSendRate;
+    }
+    void setClientPhysicsSendRate(float value);
+
+    // Streaming debug
+    void setExtraMemoryUsedInMB(int value);
+    int getExtraMemoryUsedInMB() const
+    {
+        return extraMemoryUsed;
+    }
+    float getFreeMemoryMBytes() const;
+    float getFreeMemoryPoolMBytes() const;
+
+    bool getRenderStreamedRegions() const;
+    void setRenderStreamedRegions(bool value);
+
+    bool getShowPartMovementPath() const;
+    void setShowPartMovementPath(bool value);
+    int getTotalNumMovementWayPoint() const
+    {
+        return totalNumMovementWayPoint;
+    }
+    void setTotalNumMovementWayPoint(int value);
+#ifdef AYA_TEST_BUILD
+    int getTaskSchedulerFindJobFPS() const
+    {
+        return TaskScheduler::findJobFPS;
+    }
+    void setTaskSchedulerFindJobFPS(int value)
+    {
+        TaskScheduler::findJobFPS = value;
+    }
+    bool getTaskSchedulerUpdateJobPriorityOnWake() const
+    {
+        return TaskScheduler::updateJobPriorityOnWake;
+    }
+    void setTaskSchedulerUpdateJobPriorityOnWake(bool value)
+    {
+        TaskScheduler::updateJobPriorityOnWake = value;
+    }
+#endif
+
+    bool getShowActiveAnimationAsset() const;
+    void setShowActiveAnimationAsset(bool value);
+
+    void printProfilingResult();
+
+    float touchSendRate;
+
+    int canSendPacketBufferLimit;
+    int sendPacketBufferLimit;
+    float networkOwnerRate;
+    bool isThrottledByOutgoingBandwidthLimit;
+    bool isThrottledByCongestionControl;
+
+    // Data-in throttle
+    double getReceiveRate() const
+    {
+        return receiveRate;
+    }
+    void setReceiveRate(double value);
+
+    float getTouchSendRate() const
+    {
+        return touchSendRate;
+    }
+    void setTouchSendRate(float value);
+
+    int deprecatedWaitingForCharacterLogRate;
+
+protected:
+    int physicsMtuAdjust;
+    int replicationMtuAdjust;
+
+    // Data-out throttle
+    ObscureValue<float> dataSendRate;
+    ObscureValue<float> physicsSendRate;
+    ObscureValue<float> clientPhysicsSendRate;
+    float dataGCRate;
+
+    // Data-in throttle
+    ObscureValue<double> receiveRate;
+
+    // streaming debug
+    int extraMemoryUsed;
+
+    // movement debug
+    int totalNumMovementWayPoint;
+};
+
+} // namespace Aya

engine/network/src/PacketIds.hpp

@@ -0,0 +1,81 @@
+
+#pragma once
+
+#include "RakNet/MessageIdentifiers.hpp"
+#include "RakNet/PacketPriority.hpp"
+
+namespace Aya
+{
+namespace Network
+{
+
+enum
+{
+    ID_SET_GLOBALS = ID_USER_PACKET_ENUM, // Sent by Server to Client
+
+    ID_TEACH_DESCRIPTOR_DICTIONARIES,
+
+    ID_DATA,
+    ID_REQUEST_MARKER,
+
+    ID_PHYSICS,
+    ID_PHYSICS_TOUCHES,
+
+    ID_CHAT_ALL,
+    ID_CHAT_TEAM,
+    ID_REPORT_ABUSE,
+
+    ID_SUBMIT_TICKET,
+
+    ID_CHAT_GAME,
+    ID_CHAT_PLAYER,
+    ID_CLUSTER,
+
+    ID_PROTOCAL_MISMATCH,
+
+    ID_SPAWN_NAME,
+
+    ID_PROTOCOL_SYNC,
+    ID_SCHEMA_SYNC,
+
+    ID_PLACEID_VERIFICATION,
+
+    ID_DICTIONARY_FORMAT,
+
+    ID_HASH_MISMATCH,
+    ID_SECURITYKEY_MISMATCH,
+
+    ID_REQUEST_STATS,
+
+#ifdef ENABLE_VOICE_CHAT
+    ID_OPUS_DATA
+#endif
+};
+
+// Order of packets on a connection:
+//  Client --> Server ID_NEW_INCOMING_CONNECTION
+//  If server accepts connection, the following happen:
+//  Client --> Server ID_SUBMIT_TICKET (contains client info like network protocol version)
+//  Server --> Client ID_SET_GLOBALS (contains all the top replication containers, stuff like workspace, lighting, etc.)
+//  Client --> Server ID_INSTANCE_NEW
+//  Client --> Server ID_INSTANCE_NEW
+//  ...
+//  Client --> Server ID_REQUEST_CHARACTER
+
+const PacketPriority PHYSICS_GENERAL_PRIORITY = MEDIUM_PRIORITY;
+const int PHYSICS_CHANNEL = 0;
+
+const PacketPriority DATAMODEL_PRIORITY = MEDIUM_PRIORITY;
+const PacketReliability DATAMODEL_RELIABILITY = RELIABLE_ORDERED;
+const int DATA_CHANNEL = 0;
+
+const PacketPriority CHAT_PRIORITY = HIGH_PRIORITY;
+const PacketReliability CHAT_RELIABILITY = RELIABLE;
+const int CHAT_CHANNEL = 2;
+
+const PacketPriority VOICE_CHAT_PRIORITY = HIGH_PRIORITY;
+const PacketReliability VOICE_CHAT_RELIABILITY = UNRELIABLE_SEQUENCED;
+const int VOICE_CHAT_CHANNEL = 3;
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Peer.cpp

@@ -0,0 +1,260 @@
+#include "Peer.hpp"
+
+#include "Replicator.hpp"
+#include "RakNet/GetTime.hpp"
+#include "RakNet/RakPeer.hpp"
+#include "ConcurrentRakPeer.hpp"
+#include "Log.hpp"
+
+#include "Debug.hpp"
+
+#include "DataModel/Stats.hpp"
+
+#include "DataModel/DataModelJob.hpp"
+
+#include "DataModel/PhysicsService.hpp"
+
+#include "DataModel/DataModel.hpp"
+
+#include "Utility/ObscureValue.hpp"
+
+#include "Utility/StandardOut.hpp"
+
+
+#include "DataBlockEncryptor.hpp"
+
+const char* const Aya::Network::sPeer = "NetworkPeer";
+
+namespace Aya
+{
+namespace Network
+{
+
+
+static Reflection::BoundFuncDesc<Peer, void(int)> func_SetOutgoingKBPSLimit(
+    &Peer::setOutgoingKBPSLimit, "SetOutgoingKBPSLimit", "limit", Security::Plugin);
+REFLECTION_END();
+
+class ProfiledRakPeer : public RakNet::RakPeer
+{
+private:
+    Peer& peer; // time spent per step in the Raknet thread
+    typedef RakNet::RakPeer Super;
+
+public:
+    ProfiledRakPeer(Peer& peer)
+        : peer(peer)
+    {
+    }
+    virtual bool RunUpdateCycle(RakNet::TimeUS timeNS, RakNet::Time timeMS)
+    {
+        Timer<Time::Benchmark> timer;
+        RakNet::BitStream updateBitStream(MAXIMUM_MTU_SIZE);
+        bool result = Super::RunUpdateCycle(timeNS, timeMS, updateBitStream);
+        peer.rakDutyCycle.sample(timer.delta());
+        return result;
+    }
+};
+
+static double lerp = 0.05;
+unsigned char Peer::aesKey[16];
+
+Peer::Peer()
+    : rakDutyCycle(lerp)
+{
+    for (int i = 0; i < 16; ++i)
+        aesKey[i] = 0xFE ^ 7 * i;
+
+    protocolVersion = NETWORK_PROTOCOL_VERSION;
+}
+
+Peer::~Peer() {}
+
+void Peer::onCreateRakPeer()
+{
+    RakNet::RakNetGUID guid = rakPeer->rawPeer()->GetGuidFromSystemAddress(RakNet::UNASSIGNED_SYSTEM_ADDRESS);
+    unsigned int seed = (unsigned int)((guid.g >> 32) ^ guid.g);
+    rnr.SeedMT(seed);
+
+    rakPeer->rawPeer()->SetOccasionalPing(true);
+}
+
+void Peer::encryptDataPart(RakNet::BitStream& bitStream)
+{
+    DataBlockEncryptor encryptor;
+    encryptor.SetKey(Peer::aesKey);
+
+    unsigned int length = (unsigned int)bitStream.GetNumberOfBytesUsed() - 1;
+
+    unsigned int bytesRequired = length + 6;
+    bytesRequired = ((bytesRequired + 15) / 16) * 16;
+
+    int deltaBits = 8 * bytesRequired - (bitStream.GetNumberOfBitsAllocated() - 8);
+    if (deltaBits > 0)
+        bitStream.AddBitsAndReallocate(deltaBits);
+
+    encryptor.Encrypt((unsigned char*)bitStream.GetData() + 1, length, (unsigned char*)bitStream.GetData() + 1, &length, &rnr);
+    bitStream.SetWriteOffset((1 + length) * 8);
+}
+
+
+void Peer::decryptDataPart(RakNet::BitStream& inBitstream)
+{
+    DataBlockEncryptor encryptor;
+    encryptor.SetKey(Peer::aesKey);
+
+    unsigned int length = (unsigned int)inBitstream.GetNumberOfBytesUsed() - 1;
+
+    bool success = encryptor.Decrypt((unsigned char*)inBitstream.GetData() + 1, length, (unsigned char*)inBitstream.GetData() + 1, &length);
+    if (!success)
+        throw std::runtime_error("Data error");
+}
+
+bool Peer::askAddChild(const Instance* instance) const
+{
+    return Instance::fastDynamicCast<Replicator>(instance) != NULL;
+}
+
+class PeerStatsItem : public Stats::Item
+{
+    Peer* peer;
+    Stats::Item* rakRate;
+    Stats::Item* rakActivity;
+    Stats::Item* physicsSenders;
+    Stats::Item* bufferHealth;
+
+public:
+    PeerStatsItem(Peer* peer)
+        : peer(peer)
+    {
+        Stats::Item* item =
+            createChildItem("Packets Thread"); // TODO: Rename this when possible. Unfortunately, Game service requires this to be here
+        rakRate = item->createChildItem("Rate");
+        rakActivity = item->createChildItem("Activity");
+        physicsSenders = item->createChildItem("Physics Senders"); // TODO: More this out of "Packets Thread" when we refactor Game Service
+        bufferHealth = item->createChildItem("Send Buffer Health");
+    }
+
+    /* override */ void update()
+    {
+        rakActivity->formatPercent(peer->rakDutyCycle.dutyCycle());
+        double rate = peer->rakDutyCycle.rate();
+        rakRate->formatValue(rate, "%.2g/s", rate);
+
+        PhysicsService* physicsService = ServiceProvider::find<PhysicsService>(this);
+        physicsSenders->formatValue(physicsService ? physicsService->numSenders() : 0);
+
+        double health = peer->rakPeer->GetBufferHealth();
+        bufferHealth->formatValue(health, "%.4g", health);
+    }
+};
+
+
+class PacketReceiveJob : public DataModelJob
+{
+    weak_ptr<DataModel> dataModel;
+    ObscureValue<double> receiveRate;
+
+public:
+    weak_ptr<ConcurrentRakPeer> rakPeer;
+    PacketReceiveJob(shared_ptr<ConcurrentRakPeer> rakPeer, DataModel* dataModel)
+        : DataModelJob("Net PacketReceive", DataModelJob::DataIn, false, shared_from(dataModel), Time::Interval(0))
+        , rakPeer(rakPeer)
+        , dataModel(shared_from(dataModel))
+        , receiveRate(NetworkSettings::singleton().getReceiveRate())
+    {
+        cyclicExecutive = true;
+        cyclicPriority = CyclicExecutiveJobPriority_Network_ReceiveIncoming;
+    }
+
+private:
+    Time::Interval sleepTime(const Stats& stats)
+    {
+        return computeStandardSleepTime(stats, receiveRate);
+    }
+
+    virtual Job::Error error(const Stats& stats)
+    {
+        return computeStandardError(stats, receiveRate);
+    }
+
+
+    virtual TaskScheduler::StepResult stepDataModelJob(const Stats& stats)
+    {
+        if (shared_ptr<DataModel> safeDataModel = dataModel.lock())
+        {
+            FASTLOG1(FLog::DataModelJobs, "Packet receive start, data model: %p", safeDataModel.get());
+            DataModel::scoped_write_request request(safeDataModel.get());
+
+            if (shared_ptr<ConcurrentRakPeer> safeRakPeer = rakPeer.lock())
+                while (RakNet::Packet* packet = safeRakPeer->rawPeer()->Receive())
+                    safeRakPeer->DeallocatePacket(packet);
+
+            FASTLOG1(FLog::DataModelJobs, "Packet receive finish, data model: %p", safeDataModel.get());
+            return TaskScheduler::Stepped;
+        }
+        return TaskScheduler::Done;
+    }
+};
+
+
+void Peer::setOutgoingKBPSLimit(int limit)
+{
+    if (limit <= 0)
+        rakPeer->rawPeer()->SetPerConnectionOutgoingBandwidthLimit(0);
+    else
+        rakPeer->rawPeer()->SetPerConnectionOutgoingBandwidthLimit(1000 * G3D::iClamp(limit, 10, 10000));
+}
+
+void Peer::onServiceProvider(ServiceProvider* oldProvider, ServiceProvider* newProvider)
+{
+    Aya::Stats::StatsService* stats = ServiceProvider::find<Aya::Stats::StatsService>(oldProvider);
+    if (stats)
+    {
+        shared_ptr<Stats::Item> network = shared_from_polymorphic_downcast<Stats::Item>(stats->findFirstChildByName("Network"));
+        if (network)
+            network->setParent(NULL);
+    }
+
+    if (receiveJob)
+    {
+        receiveJob->rakPeer.reset(); // Make sure it doesn't try to run from now on. (The concurrency is such that it isn't running now)
+        TaskScheduler::singleton().remove(receiveJob);
+        receiveJob.reset();
+    }
+
+    // Ensure that all Replicators are removed, because they
+    // point to rakPeer, which is about to be deleted
+    this->removeAllChildren();
+
+    if (rakPeer)
+    {
+        rakPeer->rawPeer()->DetachPlugin(this);
+        rakPeer.reset();
+    }
+
+    Super::onServiceProvider(oldProvider, newProvider);
+
+    if (newProvider)
+    {
+        rakPeer.reset(new ConcurrentRakPeer(new ProfiledRakPeer(*this), boost::polymorphic_downcast<DataModel*>(newProvider)));
+        rakPeer->rawPeer()->AttachPlugin(this);
+        // rakPeer->rawPeer()->SetMTUSize(1400);
+        onCreateRakPeer();
+
+        receiveJob = shared_ptr<PacketReceiveJob>(new PacketReceiveJob(rakPeer, boost::polymorphic_downcast<DataModel*>(newProvider)));
+        TaskScheduler::singleton().add(receiveJob);
+    }
+
+    stats = ServiceProvider::find<Aya::Stats::StatsService>(newProvider);
+    if (stats)
+    {
+        AYAASSERT(!shared_from_polymorphic_downcast<Stats::Item>(stats->findFirstChildByName("Network")));
+        shared_ptr<Stats::Item> network = Creatable<Instance>::create<PeerStatsItem>(this);
+        network->setName("Network");
+        network->setParent(stats);
+    }
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Peer.hpp

@@ -0,0 +1,68 @@
+#pragma once
+
+#include "PacketIds.hpp"
+#include "Streaming.hpp"
+#include "NetworkSettings.hpp"
+#include "Item.hpp"
+#include "API.hpp"
+
+#include "RakNet/PluginInterface2.hpp"
+#include "Tree/Instance.hpp"
+
+#include "Utility/RunStateOwner.hpp"
+
+#include "Utility/Region2.hpp"
+
+#include "DataModel/DataModelJob.hpp"
+
+#include "queue"
+#include "RakNet/Rand.hpp"
+
+class RakPeerInterface;
+class PacketLogger;
+class RakPeer;
+
+namespace Aya
+{
+
+namespace Network
+{
+
+class ConcurrentRakPeer;
+class Replicator;
+class PacketReceiveJob;
+
+// Client and Server descend from this class
+extern const char* const sPeer;
+class Peer
+    : public Reflection::Described<Peer, sPeer, Instance>
+    , public RakNet::PluginInterface2
+{
+private:
+    typedef Reflection::Described<Peer, sPeer, Instance> Super;
+    shared_ptr<PacketReceiveJob> receiveJob;
+    static unsigned char aesKey[16];
+    RakNet::RakNetRandom rnr;
+
+public:
+    RunningAverageDutyCycle<> rakDutyCycle;
+    boost::shared_ptr<ConcurrentRakPeer> rakPeer;
+    void setOutgoingKBPSLimit(int limit);
+
+    void encryptDataPart(RakNet::BitStream& bitStream);
+    static void decryptDataPart(RakNet::BitStream& bitStream);
+
+protected:
+    int protocolVersion; // default network protocol, shared between client and server
+
+    Peer();
+    ~Peer();
+    // Called after rakPeer is created
+    virtual void onCreateRakPeer();
+    bool askAddChild(const Instance* instance) const;
+    /*override*/ void onServiceProvider(ServiceProvider* oldProvider, ServiceProvider* newProvider);
+};
+
+
+} // namespace Network
+} // namespace Aya

engine/network/src/PersistentDataStore.cpp

@@ -0,0 +1,299 @@
+#include "PersistentDataStore.hpp"
+#include "DataModel/Value.hpp"
+
+#include "Script/script.hpp"
+#include "Xml/XmlSerializer.hpp"
+#include "Xml/WebSerializer.hpp"
+#include "DataModel/Message.hpp"
+
+#include "DataModel/Animation.hpp"
+
+#include "DataModel/TextLabel.hpp"
+
+#include "DataModel/TextButton.hpp"
+
+#include "DataModel/TextBox.hpp"
+
+#include "Players.hpp"
+
+
+
+namespace
+{
+static inline void sendDataPersistenceStats(int placeId)
+{
+    std::ostringstream ss;
+    ss << placeId;
+}
+
+} // namespace
+
+namespace Aya
+{
+namespace Network
+{
+
+static int computeLimit(const Reflection::Variant& value);
+
+
+static void computeInstanceLimit(shared_ptr<Instance> instance, int* result)
+{
+    (*result) += instance->getPersistentDataCost();
+}
+
+static void computeValueMapLimit(const std::pair<std::string, Reflection::Variant>& pair, int* result)
+{
+    (*result) += computeLimit(pair.second);
+}
+static void computeValueCollectionLimit(const Reflection::Variant& value, int* result)
+{
+    (*result) += computeLimit(value);
+}
+static int computeLimit(const Reflection::Variant& value)
+{
+    if (value.isType<std::string>())
+    {
+        return Instance::computeStringCost(value.get<std::string>());
+    }
+    if (value.isType<shared_ptr<Instance>>())
+    {
+        int result = 0;
+        computeInstanceLimit(value.get<shared_ptr<Instance>>(), &result);
+        return result;
+    }
+    if (value.isType<shared_ptr<const Aya::Reflection::ValueMap>>())
+    {
+        int sum = 1;
+        shared_ptr<const Aya::Reflection::ValueMap> valueMap = value.get<shared_ptr<const Aya::Reflection::ValueMap>>();
+        std::for_each(valueMap->begin(), valueMap->end(), boost::bind(&computeValueMapLimit, _1, &sum));
+        return sum;
+    }
+    if (value.isType<shared_ptr<const Reflection::ValueArray>>())
+    {
+        int sum = 1;
+        shared_ptr<const Reflection::ValueArray> valueCollection = value.get<shared_ptr<const Reflection::ValueArray>>();
+        std::for_each(valueCollection->begin(), valueCollection->end(), boost::bind(&computeValueCollectionLimit, _1, &sum));
+        return sum;
+    }
+
+    return 1;
+}
+
+PersistentDataStore::PersistentDataStore(const Reflection::ValueMap* input, const Players* players, int complexityLimit)
+    : complexity(0)
+    , players(players)
+    , complexityLimit(complexityLimit)
+    , leaderboardDirty(false)
+{
+    if (input)
+    {
+        valueMap = (*input);
+        leaderboardDirty = true;
+        std::for_each(valueMap.begin(), valueMap.end(), boost::bind(&computeValueMapLimit, _1, &complexity));
+
+        if (valueMap.size() > 0)
+        {
+            static boost::once_flag flag = BOOST_ONCE_INIT;
+            const DataModel* dm = DataModel::get(players);
+            boost::call_once(boost::bind(&sendDataPersistenceStats, dm ? dm->getPlaceID() : 0), flag);
+        }
+    }
+}
+bool PersistentDataStore::serializeValueMap(std::string& output, const Reflection::ValueMap& valueMap)
+{
+    boost::scoped_ptr<XmlElement> root(WebSerializer::writeTable(valueMap));
+    if (root)
+    {
+        std::ostringstream stream;
+        TextXmlWriter machine(stream);
+        machine.serialize(root.get());
+        stream.flush();
+
+        output = stream.str();
+
+        return true;
+    }
+    return false;
+}
+
+bool PersistentDataStore::saveLeaderboard(std::string& output)
+{
+    leaderboardDirty = false;
+    Reflection::ValueMap leaderboardValueMap;
+    if (players)
+    {
+        for (boost::unordered_set<std::string>::const_iterator iter = players->beginLeaderboardKey(), end = players->endLeaderboardKey(); iter != end;
+             ++iter)
+        {
+            leaderboardValueMap[*iter] = getNumber(*iter);
+        }
+    }
+    return serializeValueMap(output, leaderboardValueMap);
+}
+
+bool PersistentDataStore::save(std::string& output)
+{
+    return serializeValueMap(output, valueMap);
+}
+void PersistentDataStore::setComplexityLimit(int value)
+{
+    complexityLimit = value;
+}
+void PersistentDataStore::removeKey(const std::string& key)
+{
+    Reflection::ValueMap::iterator iter = valueMap.find(key);
+    if (iter != valueMap.end())
+        complexity -= computeLimit(iter->second);
+    valueMap.erase(key);
+}
+bool PersistentDataStore::enforceComplexity(const std::string& key)
+{
+    int addedComplexity = computeLimit(valueMap[key]);
+
+    if (addedComplexity + complexity <= complexityLimit)
+    {
+        complexity += addedComplexity;
+        return true;
+    }
+    else
+    {
+        valueMap.erase(key);
+        return false;
+    }
+}
+
+
+bool PersistentDataStore::isNumber(const std::string& key)
+{
+    Reflection::ValueMap::iterator iter = valueMap.find(key);
+    if (iter != valueMap.end() && !iter->second.isType<double>())
+        return false;
+    return true;
+}
+double PersistentDataStore::getNumber(const std::string& key)
+{
+    Reflection::ValueMap::iterator iter = valueMap.find(key);
+    if (iter == valueMap.end() || !iter->second.isType<double>())
+        return 0.0f;
+    return iter->second.get<double>();
+}
+bool PersistentDataStore::setNumber(const std::string& key, double value)
+{
+    if (isNumber(key) && getNumber(key) == value)
+    {
+        return true;
+    }
+
+    if (players->hasLeaderboardKey(key))
+        leaderboardDirty = true;
+
+    removeKey(key);
+
+    if (value == 0)
+        return true;
+
+    valueMap[key] = value;
+
+    return enforceComplexity(key);
+}
+std::string PersistentDataStore::getString(const std::string& key)
+{
+    Reflection::ValueMap::iterator iter = valueMap.find(key);
+    if (iter == valueMap.end() || !iter->second.isType<std::string>())
+        return "";
+    return iter->second.get<std::string>();
+}
+bool PersistentDataStore::setString(const std::string& key, const std::string& value)
+{
+    removeKey(key);
+
+    if (value == "")
+        return true;
+
+    valueMap[key] = value;
+
+    return enforceComplexity(key);
+}
+bool PersistentDataStore::getBoolean(const std::string& key)
+{
+    Reflection::ValueMap::iterator iter = valueMap.find(key);
+    if (iter == valueMap.end() || !iter->second.isType<bool>())
+        return false;
+    return iter->second.get<bool>();
+}
+bool PersistentDataStore::setBoolean(const std::string& key, bool value)
+{
+    removeKey(key);
+
+    if (value == false)
+        return true;
+
+    valueMap[key] = value;
+
+    return enforceComplexity(key);
+}
+shared_ptr<Instance> PersistentDataStore::getInstance(const std::string& key)
+{
+    Reflection::ValueMap::iterator iter = valueMap.find(key);
+    if (iter == valueMap.end() || !iter->second.isType<shared_ptr<Instance>>())
+        return shared_ptr<Instance>();
+
+    return iter->second.get<shared_ptr<Instance>>()->clone(SerializationCreator);
+}
+
+bool PersistentDataStore::setInstance(const std::string& key, shared_ptr<Instance> value)
+{
+    removeKey(key);
+
+    if (!value)
+        return true;
+
+    shared_ptr<Instance> clonedInstance = value->clone(SerializationCreator);
+    if (clonedInstance)
+        valueMap[key] = clonedInstance;
+
+    return enforceComplexity(key);
+}
+
+shared_ptr<const Reflection::ValueArray> PersistentDataStore::getList(const std::string& key)
+{
+    Reflection::ValueMap::iterator iter = valueMap.find(key);
+    if (iter == valueMap.end() || !iter->second.isType<shared_ptr<const Reflection::ValueArray>>())
+        return shared_ptr<const Reflection::ValueArray>();
+    return iter->second.get<shared_ptr<const Reflection::ValueArray>>();
+}
+
+bool PersistentDataStore::setList(const std::string& key, shared_ptr<const Reflection::ValueArray> value)
+{
+    removeKey(key);
+
+    if (!value || value->size() == 0)
+        return true;
+
+    valueMap[key] = value;
+
+    return enforceComplexity(key);
+}
+shared_ptr<const Aya::Reflection::ValueMap> PersistentDataStore::getTable(const std::string& key)
+{
+    Reflection::ValueMap::iterator iter = valueMap.find(key);
+    if (iter == valueMap.end() || !iter->second.isType<shared_ptr<const Aya::Reflection::ValueMap>>())
+        return shared_ptr<const Aya::Reflection::ValueMap>();
+    return iter->second.get<shared_ptr<const Aya::Reflection::ValueMap>>();
+}
+
+
+bool PersistentDataStore::setTable(const std::string& key, shared_ptr<const Aya::Reflection::ValueMap> value)
+{
+    removeKey(key);
+
+    if (!value || value->empty())
+        return true;
+
+    valueMap[key] = value;
+
+    return enforceComplexity(key);
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/PersistentDataStore.hpp

@@ -0,0 +1,72 @@
+#pragma once
+
+#include "Tree/Instance.hpp"
+
+#include <boost/noncopyable.hpp>
+namespace Aya
+{
+namespace Network
+{
+class Players;
+
+class PersistentDataStore : boost::noncopyable
+{
+private:
+    Reflection::ValueMap valueMap;
+    const Players* players;
+
+    bool leaderboardDirty;
+    int complexity;
+    int complexityLimit;
+
+    void removeKey(const std::string& key);
+    bool enforceComplexity(const std::string& key);
+
+    static bool serializeValueMap(std::string& output, const Reflection::ValueMap& valueMap);
+
+    bool isNumber(const std::string& key);
+
+public:
+    PersistentDataStore(const Reflection::ValueMap* input, const Players* players, int complexityLimit);
+
+    bool empty() const
+    {
+        return valueMap.empty();
+    }
+    bool isLeaderboardDirty() const
+    {
+        return leaderboardDirty;
+    }
+
+    bool save(std::string& output);
+    bool saveLeaderboard(std::string& output);
+
+    int getComplexity() const
+    {
+        return complexity;
+    }
+    int getComplexityLimit() const
+    {
+        return complexityLimit;
+    }
+    void setComplexityLimit(int value);
+
+    float getLeaderboard(const std::string& key);
+    bool setLeaderboard(const std::string& key, float value);
+
+    double getNumber(const std::string& key);
+    bool setNumber(const std::string& key, double value);
+    std::string getString(const std::string& key);
+    bool setString(const std::string& key, const std::string& value);
+    bool getBoolean(const std::string& key);
+    bool setBoolean(const std::string& key, bool value);
+    shared_ptr<Instance> getInstance(const std::string& key);
+    bool setInstance(const std::string&, shared_ptr<Instance> value);
+
+    shared_ptr<const Reflection::ValueArray> getList(const std::string& key);
+    bool setList(const std::string&, shared_ptr<const Reflection::ValueArray> value);
+    shared_ptr<const Aya::Reflection::ValueMap> getTable(const std::string& key);
+    bool setTable(const std::string&, shared_ptr<const Aya::Reflection::ValueMap> value);
+};
+} // namespace Network
+} // namespace Aya

engine/network/src/PhysicsReceiver.cpp

@@ -0,0 +1,890 @@
+
+
+#include "PhysicsReceiver.hpp"
+
+#include "Compressor.hpp"
+#include "RakNet/RakPeerInterface.hpp"
+#include "NetworkSettings.hpp"
+
+#include "Streaming.hpp"
+#include "Replicator.hpp"
+#include "Util.hpp"
+
+#include "DataModel/PartInstance.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+#include "DataModel/PhysicsService.hpp"
+
+
+#include "World/World.hpp"
+#include "World/Primitive.hpp"
+#include "World/Assembly.hpp"
+#include "World/Joint.hpp"
+#include "World/Mechanism.hpp"
+
+#include "Utility/StandardOut.hpp"
+
+
+#include "PhysicsSender.hpp"
+#include "NetworkProfiler.hpp"
+
+#include "DrawAdorn.hpp"
+#include "Base/Adorn.hpp"
+#include <stack>
+#include "Utility/MovementHistory.hpp"
+
+#include "Utility/VarInt.hpp"
+
+
+using namespace Aya;
+using namespace Aya::Network;
+
+DYNAMIC_FASTFLAG(HumanoidFloorPVUpdateSignal)
+DYNAMIC_FASTINTVARIABLE(DebugMovementPathNumTotalWayPoint, 1000)
+DYNAMIC_FASTFLAG(SimpleHermiteSplineInterpolate)
+
+namespace Aya
+{
+namespace Network
+{
+namespace PathBasedMovementDebug
+{
+NodeDebugInfo FirstNode = {Color3::blue(), 13, false};
+NodeDebugInfo CompressedBaselineCFrame = {Color3::orange(), 12, true};
+NodeDebugInfo PathNode = {Color3::yellow(), 6, true};
+NodeDebugInfo XPacketCompressedPathNode = {Color3::green(), 6, true};
+NodeDebugInfo InterpolateCFrameFromNode = {Color3::white(), 10, true};
+NodeDebugInfo InterpolateCFrame = {Color3::white(), 11, true};
+NodeDebugInfo VelocityNode = {Color3::red(), 5, false};
+NodeDebugInfo OldNode = {Color3::red(), 6, true};
+NodeDebugInfo CFrame = {Color3::white(), 5, true}; // regular cframe not from path based movement
+} // namespace PathBasedMovementDebug
+} // namespace Network
+} // namespace Aya
+
+void DeserializedTouchItem::process(Replicator& replicator)
+{
+    if (replicator.physicsReceiver)
+    {
+        for (auto& tp : touchPairs)
+        {
+            replicator.physicsReceiver->processTouchPair(tp);
+        }
+    }
+}
+
+PhysicsReceiver::PhysicsReceiver(Replicator* replicator, bool isServer)
+    : replicator(replicator)
+    , iAmServer(isServer)
+    , stats(NULL)
+    , movementWaypointList(DFInt::DebugMovementPathNumTotalWayPoint)
+{
+    setTime(Time::nowFast());
+}
+
+void PhysicsReceiver::setTime(Time now_)
+{
+    now = now_;
+}
+
+/*
+        Format for a mechanism:
+
+        >> MechanismAttributes
+        >> PrimaryAssembly
+        >> done
+        while (!done)
+        {
+                >> ChildAssembly
+                >> done
+        }
+*/
+
+/*
+void logItem(MechanismItem& item, RakNet::Time timeStamp)
+{
+        float y = item.getAssemblyItem(0).pv.position.translation.y;
+
+        G3D::Log::common()->printf("%u\t", timeStamp);
+        G3D::Log::common()->printf("%f", y);
+        G3D::Log::common()->println("");
+}
+*/
+
+void PhysicsReceiver::receiveMechanismCFrames(RakNet::BitStream& bitStream, RakNet::Time timeStamp, const Aya::RemoteTime& sendTime)
+{
+    NETPROFILE_START("receiveMechanismCFrames", &bitStream);
+    int bitStart = bitStream.GetReadOffset();
+    shared_ptr<PartInstance> part;
+    while (receivePart(part, bitStream))
+    {
+        if (part)
+        {
+            if (part->raknetTime > timeStamp)
+            {
+                if (replicator->settings().printPhysicsErrors)
+                {
+                    Aya::StandardOut::singleton()->print(Aya::MESSAGE_INFO, "Physics-in old packet");
+                }
+                part.reset();
+            }
+        }
+        CoordinateFrame cf;
+        Velocity vel;
+        readCoordinateFrame(bitStream, cf);
+        if (DFFlag::SimpleHermiteSplineInterpolate)
+        {
+            readVelocity(bitStream, vel);
+        }
+
+        if (part)
+        {
+            part->setPhysics(cf);
+            part->addInterpolationSample(cf, vel, sendTime, now, 0);
+        }
+    }
+    if (stats)
+    {
+        stats->details.mechanismCFrame.increment();
+        stats->details.mechanismCFrameSize.sample((bitStream.GetReadOffset() - bitStart) / 8);
+    }
+    NETPROFILE_END("receiveMechanismCFrames", &bitStream);
+}
+
+void PhysicsReceiver::receiveMechanism(
+    RakNet::BitStream& bitStream, PartInstance* rootPart, MechanismItem& item, RemoteTime remoteSendTime, int& numNodesInHistory)
+{
+    if (NetworkSettings::singleton().getTotalNumMovementWayPoint() == 123456) // magic number to clear all after each packet
+    {
+        movementWaypointList.clear();
+    }
+
+    NETPROFILE_START("receiveMechanism", &bitStream);
+    int bitStart = bitStream.GetReadOffset();
+    item.reset();
+
+    readMechanismAttributes(bitStream, item);
+    readMovementHistory(bitStream, remoteSendTime, rootPart, item, numNodesInHistory);
+    if (iAmServer) // client always sends assembly only
+    {
+        readAssembly(bitStream, rootPart, item, false);
+    }
+    bool done;
+
+    bitStream >> done;
+
+    while (!done)
+    {
+        shared_ptr<PartInstance> part;
+        bool ok = receivePart(part, bitStream);
+        AYAASSERT(ok);
+
+        readAssembly(bitStream, part.get(), item, false);
+
+        bitStream >> done;
+    }
+
+    if (stats)
+    {
+        stats->details.mechanism.increment();
+        stats->details.mechanismSize.sample((bitStream.GetReadOffset() - bitStart) / 8);
+    }
+    NETPROFILE_END("receiveMechanism", &bitStream);
+}
+
+void PhysicsReceiver::readMovementHistory(
+    RakNet::BitStream& bitStream, RemoteTime remoteSendTime, PartInstance* rootPart, MechanismItem& mechanismItem, int& numNodesInHistory)
+{
+    numNodesInHistory = 0;
+
+    if (iAmServer)
+    {
+        // client always sends assembly only
+        return;
+    }
+    NETPROFILE_START("readMovementHistory", &bitStream);
+    bool hasMovement;
+    unsigned int numNode;
+    uint8_t precisionLevel, timeInterval2Ms;
+    int8_t x, y, z;
+    float timeInterval, timeToEnd;
+    bitStream >> hasMovement;
+    if (hasMovement)
+    {
+        CoordinateFrame cf;
+        bool crossPacketCompression;
+        bitStream >> crossPacketCompression;
+        readAssembly(bitStream, rootPart, mechanismItem, crossPacketCompression);
+        if (rootPart)
+        {
+            if (crossPacketCompression)
+            {
+                cf = rootPart->getLastCFrame(rootPart->getCoordinateFrame());
+                cf.rotation = mechanismItem.getAssemblyItem(mechanismItem.numAssemblies() - 1).pv.position.rotation;
+            }
+            else
+            {
+                cf = mechanismItem.getAssemblyItem(mechanismItem.numAssemblies() - 1).pv.position;
+            }
+            rootPart->setLastCFrame(cf);
+        }
+
+        // bitStream >> numNode;
+        VarInt<>::decode<RakNet::BitStream>(bitStream, &numNode);
+
+        if (numNode > 0)
+        {
+            nodeStack.clear(); // Moved to header in order to avoid reallocation overhead
+
+            Vector3 nodeTrans = cf.translation;
+            AssemblyItem& assemblyItem = mechanismItem.getAssemblyItem(mechanismItem.numAssemblies() - 1);
+            timeToEnd = 0.f;
+            for (uint8_t i = 0; i < (uint8_t)numNode; i++)
+            {
+                bitStream >> precisionLevel;
+                bitStream >> x;
+                bitStream >> y;
+                bitStream >> z;
+                bitStream >> timeInterval2Ms;
+
+                // reconstruct the time
+                timeInterval = MovementHistory::getSecFrom2Ms(timeInterval2Ms);
+                // reconstruct the cframe
+                Vector3 delta;
+                delta.x = MovementHistory::decompress(x, precisionLevel);
+                delta.y = MovementHistory::decompress(y, precisionLevel);
+                delta.z = MovementHistory::decompress(z, precisionLevel);
+                nodeTrans = nodeTrans - delta;
+                CoordinateFrame nodeCFrame(cf.rotation, nodeTrans);
+
+                bool isBaselineNode = false;
+                if (crossPacketCompression)
+                {
+                    if (i == 0)
+                    {
+                        // this node is the current CFrame
+                        assemblyItem.pv.position.translation = nodeCFrame.translation;
+                        if (rootPart)
+                        {
+                            AYAASSERT(numNode > 1); // we should have at least two nodes: a baseline node and a path node
+                            rootPart->setLastCFrame(nodeCFrame);
+                            isBaselineNode = true;
+                            // addVectorAdorn(cf.translation, nodeCFrame.translation, Color3::purple());
+                        }
+                    }
+                    else if (i == 1)
+                    {
+                        // Calculate the linear velocity based on last delta
+                        assemblyItem.pv.velocity.linear = delta / timeInterval;
+                        if (NetworkSettings::singleton().getTotalNumMovementWayPoint() % 7 == 0) // magic number to print velocity
+                        {
+                            addWayPointAdorn(nodeCFrame.translation + assemblyItem.pv.velocity.linear, PathBasedMovementDebug::VelocityNode);
+                        }
+                    }
+                }
+
+                if (!isBaselineNode)
+                {
+                    TimedCF timedCf;
+                    timedCf.cf = nodeCFrame;
+                    timeToEnd += timeInterval;
+                    timedCf.timeToEnd = timeToEnd;
+                    nodeStack.push_back(timedCf);
+                }
+                else
+                {
+                    addWayPointAdorn(nodeCFrame.translation, PathBasedMovementDebug::CompressedBaselineCFrame);
+                }
+            }
+            if (rootPart)
+            {
+                // StandardOut::singleton()->printf(MESSAGE_INFO, "-------------- %d", crossPacketCompression);
+                int numNodesAhead = nodeStack.size();
+                numNodesInHistory = numNodesAhead;
+
+                while (nodeStack.size() > 0)
+                {
+                    TimedCF timedCf = nodeStack.back();
+                    RemoteTime nodeTime = remoteSendTime - Time::Interval(timedCf.timeToEnd);
+                    if (replicator->remoteRaknetTimeToLocalRbxTime(nodeTime) >= rootPart->getLastUpdateTime()) // only process the new nodes
+                    {
+                        if (DFFlag::SimpleHermiteSplineInterpolate)
+                            rootPart->addInterpolationSample(timedCf.cf, mechanismItem.getAssemblyItem(mechanismItem.numAssemblies() - 1).pv.velocity,
+                                nodeTime, replicator->remoteRaknetTimeToLocalRbxTime(remoteSendTime), timedCf.timeToEnd, numNodesAhead);
+                        else
+                            rootPart->addInterpolationSample(timedCf.cf, Velocity(), nodeTime,
+                                replicator->remoteRaknetTimeToLocalRbxTime(remoteSendTime), timedCf.timeToEnd, numNodesAhead);
+
+                        addWayPointAdorn(timedCf.cf.translation,
+                            crossPacketCompression ? PathBasedMovementDebug::XPacketCompressedPathNode : PathBasedMovementDebug::PathNode);
+                    }
+                    else
+                    {
+                        if (replicator->settings().printPhysicsErrors)
+                        {
+                            Aya::StandardOut::singleton()->print(Aya::MESSAGE_INFO, "Discard old node *****");
+
+                            addWayPointAdorn(timedCf.cf.translation, PathBasedMovementDebug::OldNode);
+                        }
+                    }
+
+                    nodeStack.pop_back();
+                }
+            }
+        }
+    }
+    else
+    {
+        AssemblyItem& assemblyItem = mechanismItem.appendAssembly();
+        assemblyItem.rootPart = shared_from<PartInstance>(rootPart);
+        if (rootPart)
+        {
+            assemblyItem.pv.position = rootPart->getCoordinateFrame();
+            assemblyItem.pv.velocity = rootPart->getVelocity();
+        }
+        readMotorAngles(bitStream, assemblyItem);
+    }
+    NETPROFILE_END("readMovementHistory", &bitStream);
+}
+
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void PhysicsReceiver::readMechanismAttributes(RakNet::BitStream& bitStream, MechanismItem& historyItem)
+{
+    NETPROFILE_START("readMechanismAttributes", &bitStream);
+    historyItem.hasVelocity = replicator->settings().distributedPhysicsEnabled;
+
+    bool hasState;
+    bitStream >> hasState;
+    if (hasState)
+    {
+        bitStream >> historyItem.networkHumanoidState;
+    }
+    else
+    {
+        historyItem.networkHumanoidState = 0;
+    }
+    NETPROFILE_END("readMechanismAttributes", &bitStream);
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void PhysicsReceiver::readAssembly(RakNet::BitStream& bitstream, PartInstance* rootPart, MechanismItem& mechanismItem, bool crossPacketCompression)
+{
+    NETPROFILE_START("readAssembly", &bitstream);
+    AssemblyItem& assemblyItem = mechanismItem.appendAssembly();
+    assemblyItem.rootPart = shared_from<PartInstance>(rootPart);
+
+    readPV(bitstream, assemblyItem, crossPacketCompression);
+    if (crossPacketCompression && rootPart)
+    {
+        // reuse the last translation (for now)
+        assemblyItem.pv.position.translation = rootPart->getCoordinateFrame().translation;
+    }
+
+    readMotorAngles(bitstream, assemblyItem);
+
+    if (rootPart)
+    {
+        addWayPointAdorn(assemblyItem.pv.position.translation, PathBasedMovementDebug::FirstNode);
+    }
+
+    if (rootPart)
+    {
+        AYAASSERT(rootPart->getPartPrimitive()->getAssembly());
+
+        Primitive* primitive = rootPart->getPartPrimitive();
+        Joint* toParent = aya_static_cast<Joint*>(primitive->getEdgeToParent());
+
+        // This joint could be null if the primitive still exists, but is no longer a child joint to parent
+
+        if (toParent)
+        {
+            toParent->setPhysics(); // read on joint angles, etc.
+        }
+    }
+    NETPROFILE_END("readAssembly", &bitstream);
+}
+
+void PhysicsReceiver::deserializeTouches(RakNet::BitStream& bitstream, const RakNet::SystemAddress& from, std::vector<TouchPair>& touchPairs)
+{
+    while (true)
+    {
+        TouchPair tp;
+        if (!deserializeTouch(bitstream, from, tp))
+            return;
+
+        if (tp.p1 && tp.p2)
+            touchPairs.push_back(tp);
+    }
+}
+
+bool PhysicsReceiver::deserializeTouch(RakNet::BitStream& bitstream, const RakNet::SystemAddress& from, TouchPair& touchPair)
+{
+    shared_ptr<PartInstance> part1;
+    if (!receivePart(part1, bitstream))
+        return false; // this marks the end of the packet
+
+    shared_ptr<PartInstance> part2;
+    bool ok = receivePart(part2, bitstream);
+    AYAASSERT(ok);
+
+    bool touched;
+    bitstream >> touched;
+
+    if (!part1)
+        return true;
+    if (!part2)
+        return true;
+
+    if (replicator->settings().printTouches)
+    {
+        if (touched)
+        {
+            Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE, "Replication: Touch:%s->%s << %s", part1->getName().c_str(),
+                part2->getName().c_str(), RakNetAddressToString(replicator->remotePlayerId).c_str());
+        }
+        else
+        {
+            Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE, "Replication: Untouch:%s->%s << %s", part1->getName().c_str(),
+                part2->getName().c_str(), RakNetAddressToString(replicator->remotePlayerId).c_str());
+        }
+    }
+
+    touchPair = TouchPair(part1, part2, touched ? TouchPair::Touch : TouchPair::Untouch, RakNetToRbxAddress(from));
+
+    return true;
+}
+
+void PhysicsReceiver::readTouches(RakNet::BitStream& bitstream, const RakNet::SystemAddress& from)
+{
+    while (true)
+    {
+        TouchPair tp;
+        if (!deserializeTouch(bitstream, from, tp))
+            return;
+
+        if (tp.p1 && tp.p2)
+            processTouchPair(tp);
+    }
+}
+
+void PhysicsReceiver::processTouchPair(const TouchPair& tp)
+{
+    if (tp.type == TouchPair::Touch)
+        tp.p1->reportTouch(tp.p2);
+    else
+        tp.p1->reportUntouch(tp.p2);
+
+    // Send pair to other Replicators
+    if (!physicsService)
+        physicsService = shared_from(ServiceProvider::find<PhysicsService>(replicator));
+
+    physicsService->onTouchStep(tp);
+}
+
+void PhysicsReceiver::readPV(RakNet::BitStream& bitStream, AssemblyItem& item, bool crossPacketCompression)
+{
+    NETPROFILE_START("readPV", &bitStream);
+    if (crossPacketCompression)
+    {
+        // readRotation only
+        Compressor::readRotation(bitStream, item.pv.position.rotation);
+    }
+    else
+    {
+        readCoordinateFrame(bitStream, item.pv.position);
+        readVelocity(bitStream, item.pv.velocity);
+
+        if (NetworkSettings::singleton().getTotalNumMovementWayPoint() % 7 == 0) // magic number to print velocity
+        {
+            addWayPointAdorn(item.pv.position.translation + item.pv.velocity.linear, PathBasedMovementDebug::VelocityNode);
+        }
+    }
+    NETPROFILE_END("readPV", &bitStream);
+}
+
+void PhysicsReceiver::readCoordinateFrame(RakNet::BitStream& bitStream, CoordinateFrame& cFrame)
+{
+    NETPROFILE_START("readCoordinateFrame", &bitStream);
+
+    if (stats == NULL)
+    {
+        Compressor::readTranslation(bitStream, cFrame.translation);
+        Compressor::readRotation(bitStream, cFrame.rotation);
+        NETPROFILE_END("readCoordinateFrame", &bitStream);
+    }
+    else
+    {
+        NETPROFILE_START("readTranslation", &bitStream);
+        int bitStart = bitStream.GetReadOffset();
+        Compressor::readTranslation(bitStream, cFrame.translation);
+        if (stats)
+        {
+            stats->details.translation.increment();
+            stats->details.translationSize.sample((bitStream.GetReadOffset() - bitStart) / 8);
+        }
+        NETPROFILE_END("readTranslation", &bitStream);
+        NETPROFILE_START("readRotation", &bitStream);
+        bitStart = bitStream.GetReadOffset();
+        Compressor::readRotation(bitStream, cFrame.rotation);
+        if (stats)
+        {
+            stats->details.rotation.increment();
+            stats->details.rotationSize.sample((bitStream.GetReadOffset() - bitStart) / 8);
+        }
+        NETPROFILE_END("readRotation", &bitStream);
+        NETPROFILE_END("readCoordinateFrame", &bitStream);
+    }
+}
+
+void PhysicsReceiver::readVelocity(RakNet::BitStream& bitStream, Velocity& velocity)
+{
+    NETPROFILE_START("readVelocity", &bitStream);
+    int bitStart = bitStream.GetReadOffset();
+    if (replicator->settings().distributedPhysicsEnabled)
+    {
+        readVectorFast(bitStream, velocity.linear.x, velocity.linear.y, velocity.linear.z);
+        readVectorFast(bitStream, velocity.rotational.x, velocity.rotational.y, velocity.rotational.z);
+    }
+    else
+    {
+        velocity = Velocity::zero();
+    }
+    if (stats)
+    {
+        stats->details.velocity.increment();
+        stats->details.velocitySize.sample((bitStream.GetReadOffset() - bitStart) / 8);
+    }
+    NETPROFILE_END("readVelocity", &bitStream);
+}
+
+void PhysicsReceiver::readMotorAngles(RakNet::BitStream& bitStream, AssemblyItem& item)
+{
+    NETPROFILE_START("readMotorAngles", &bitStream);
+    unsigned char compactNum;
+    bitStream >> compactNum;
+
+    if (compactNum > 50 && replicator->settings().printPhysicsErrors)
+        Aya::StandardOut::singleton()->printf(Aya::MESSAGE_WARNING, "Physics-in has %d motors", compactNum);
+
+    const int numMotors = compactNum;
+    item.motorAngles.resize(numMotors); // i.e. - fast clear, no allocation
+
+    for (int i = 0; i < numMotors; ++i)
+    {
+        readCompactCFrame(bitStream, item.motorAngles[i]);
+    }
+    NETPROFILE_END("readMotorAngles", &bitStream);
+}
+
+void PhysicsReceiver::readCompactCFrame(RakNet::BitStream& bitStream, CompactCFrame& cFrame)
+{
+    NETPROFILE_START("readCompactCFrame", &bitStream);
+    bool isSimpleZAngle = bitStream.ReadBit();
+
+    if (isSimpleZAngle)
+    {
+        unsigned char byteAngle;
+        bitStream >> byteAngle;
+        cFrame = CompactCFrame(Vector3::zero(), Vector3::unitZ(), Math::rotationFromByte(byteAngle));
+        AYAASSERT(!Math::isNanInfVector3(cFrame.getAxis()));
+        AYAASSERT(!Math::isNanInf(cFrame.getAngle()));
+        AYAASSERT(!Math::isNanInfVector3(cFrame.translation));
+    }
+    else
+    {
+        bool hasTranslation = bitStream.ReadBit();
+        bool hasRotation = bitStream.ReadBit();
+        if (hasTranslation)
+        {
+            Compressor::readTranslation(bitStream, cFrame.translation);
+        }
+        else
+        {
+            cFrame.translation = Vector3::zero();
+        }
+
+        if (hasRotation)
+        {
+            Vector3 axis;
+            readVectorFast(bitStream, axis.x, axis.y, axis.z);
+            unsigned char byteAngle;
+            bitStream >> byteAngle;
+            float angle = Math::rotationFromByte(byteAngle);
+            cFrame.setAxisAngle(axis, angle);
+        }
+        else
+        {
+            cFrame.setAxisAngle(Aya::Vector3::unitX(), 0);
+        }
+
+        AYAASSERT(!Math::isNanInfVector3(cFrame.getAxis()));
+        AYAASSERT(!Math::isNanInf(cFrame.getAngle()));
+        AYAASSERT(!Math::isNanInfVector3(cFrame.translation));
+    }
+    NETPROFILE_END("readCompactCFrame", &bitStream);
+}
+
+void PhysicsReceiver::setPhysics(
+    const MechanismItem& item, const Aya::RemoteTime& remoteSendTime, const RakNet::TimeMS lagInMs, int numNodesInHistory)
+{
+    bool hasVelocity = item.hasVelocity;
+    Time::Interval lag(static_cast<double>(lagInMs) / 1000.0f);
+
+    for (int i = 0; i < item.numAssemblies(); ++i)
+    {
+        AssemblyItem& assemblyItem = item.getAssemblyItem(i);
+
+        // assemblyItem.primitive will be NULL if the Part was destroyed while the physics packet was outstanding
+
+        if (PartInstance* part = assemblyItem.rootPart.get())
+        {
+            if (this->replicator->filterPhysics(part) == Reject)
+            {
+                if (replicator->settings().printPhysicsFilters)
+                    Aya::StandardOut::singleton()->printf(Aya::MESSAGE_INFO, "filterPhysics %s", part->getName().c_str());
+                continue;
+            }
+
+            Primitive* primitive = part->getPartPrimitive();
+
+            // Show david this assert and then destroy - validating the case where we received this history item in the past, and the primitive is no
+            // longer in world
+            AYAASSERT(primitive->getWorld());
+
+            if (!Assembly::isAssemblyRootPrimitive(primitive))
+            {
+                if (replicator->settings().printPhysicsFilters)
+                    Aya::StandardOut::singleton()->printf(Aya::MESSAGE_INFO, "!isAssemblyRootPrimitive %s", part->getName().c_str());
+                continue;
+            }
+
+            Assembly* a = primitive->getAssembly();
+            if (a->computeIsGrounded())
+            {
+                if (replicator->settings().printPhysicsFilters)
+                    Aya::StandardOut::singleton()->printf(Aya::MESSAGE_INFO, "computeIsGrounded %s", part->getName().c_str());
+                continue;
+            }
+
+            a->setPhysics(assemblyItem.motorAngles, assemblyItem.pv); // motor angles
+            a->setNetworkHumanoidState((i == 0) ? item.networkHumanoidState : 0);
+
+            if (hasVelocity)
+            {
+                if (iAmServer)
+                {
+                    part->addInterpolationSample(assemblyItem.pv.position, assemblyItem.pv.velocity, remoteSendTime, now, 0.f, numNodesInHistory);
+                    part->setPhysics(assemblyItem.pv);
+
+                    Time localTime = replicator->remoteRaknetTimeToLocalRbxTime(remoteSendTime);
+                    part->addMovementNode(assemblyItem.pv.position, assemblyItem.pv.velocity, localTime); // force update the movement node
+                }
+                else
+                {
+                    bool isLagCompenstated = false;
+
+                    if (isLagCompenstated)
+                    {
+                        // Extrapolate with velocity
+                        if (assemblyItem.pv.velocity.rotational.squaredLength() >= 0.01)
+                        {
+                            Quaternion qOrientation(assemblyItem.pv.position.rotation);
+                            qOrientation.normalize();
+                            Quaternion qDot(Quaternion(assemblyItem.pv.velocity.rotational) * qOrientation * 0.5f);
+                            qOrientation += qDot * lag.seconds();
+                            qOrientation.normalize();
+                            qOrientation.toRotationMatrix(assemblyItem.pv.position.rotation);
+                        }
+                        if (assemblyItem.pv.velocity.linear.squaredLength() >= 1)
+                            assemblyItem.pv.position.translation += assemblyItem.pv.velocity.linear * lag.seconds();
+
+                        part->addInterpolationSample(
+                            assemblyItem.pv.position, assemblyItem.pv.velocity, remoteSendTime + lag, now, 0.f, numNodesInHistory);
+                    }
+                    else
+                    {
+                        part->addInterpolationSample(assemblyItem.pv.position, assemblyItem.pv.velocity, remoteSendTime, now, 0.f, numNodesInHistory);
+                    }
+
+                    CoordinateFrame oldPosition;
+                    Aya::Velocity previousLinearVelocity;
+                    Aya::Time lastUpdateTime;
+
+                    if (DFFlag::HumanoidFloorPVUpdateSignal)
+                    {
+                        oldPosition = part->getCoordinateFrame();
+                        previousLinearVelocity = part->getVelocity();
+                        lastUpdateTime = part->getLastUpdateTime();
+                    }
+                    part->setPhysics(assemblyItem.pv);
+
+                    if (DFFlag::HumanoidFloorPVUpdateSignal)
+                    {
+                        float deltaTime = (replicator->remoteRaknetTimeToLocalRbxTime(remoteSendTime) - lastUpdateTime).seconds();
+                        // shared_ptr<PartInstance> partPointer = shared_from(part);
+                        if (part->onDemandRead())
+                            part->onDemandWrite()->onPositionUpdatedByNetworkSignal(
+                                boost::ref(part), boost::ref(oldPosition), boost::ref(previousLinearVelocity), boost::ref(deltaTime));
+                    }
+                }
+            }
+            else
+            {
+                part->addInterpolationSample(assemblyItem.pv.position, assemblyItem.pv.velocity, remoteSendTime, now, 0.f, numNodesInHistory);
+                CoordinateFrame oldPosition;
+                Aya::Velocity previousLinearVelocity;
+                Aya::Time lastUpdateTime;
+                if (DFFlag::HumanoidFloorPVUpdateSignal)
+                {
+                    oldPosition = part->getCoordinateFrame();
+                    previousLinearVelocity = part->getVelocity();
+                    lastUpdateTime = part->getLastUpdateTime();
+                }
+                part->setPhysics(assemblyItem.pv.position);
+
+                if (DFFlag::HumanoidFloorPVUpdateSignal)
+                {
+                    float deltaTime = (replicator->remoteRaknetTimeToLocalRbxTime(remoteSendTime) - lastUpdateTime).seconds();
+                    // shared_ptr<PartInstance> partPointer = shared_from(part);
+                    if (part->onDemandRead())
+                        part->onDemandWrite()->onPositionUpdatedByNetworkSignal(
+                            boost::ref(part), boost::ref(oldPosition), boost::ref(previousLinearVelocity), boost::ref(deltaTime));
+                }
+            }
+        }
+    }
+}
+
+
+
+
+////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////
+
+void PhysicsReceiver::addWayPointAdorn(const Vector3& p, const PathBasedMovementDebug::NodeDebugInfo& info, const std::string& debugText)
+{
+    if (Workspace::showPartMovementPath && info.show)
+    {
+        if (movementWaypointList.capacity() != NetworkSettings::singleton().getTotalNumMovementWayPoint())
+        {
+            movementWaypointList.rset_capacity(NetworkSettings::singleton().getTotalNumMovementWayPoint());
+        }
+        if (movementWaypointList.capacity() > 0)
+        {
+            movementWaypointList.push_back(MovementWaypointAdorn(p, info.color, info.size, debugText));
+        }
+    }
+}
+
+void PhysicsReceiver::addVectorAdorn(const Vector3& start, const Vector3& end, const Aya::Color4& c)
+{
+    if (Workspace::showPartMovementPath)
+    {
+        if (movementVectorList.capacity() != NetworkSettings::singleton().getTotalNumMovementWayPoint())
+        {
+            movementVectorList.rset_capacity(NetworkSettings::singleton().getTotalNumMovementWayPoint());
+        }
+        if (movementVectorList.capacity() > 0)
+        {
+            movementVectorList.push_back(MovementVectorAdorn(start, end, c));
+        }
+    }
+}
+
+
+bool PhysicsReceiver::okDistributedReceivePart(const shared_ptr<PartInstance>& part)
+{
+    return (!replicator->settings().distributedPhysicsEnabled || replicator->checkDistributedReceive(part.get()));
+}
+
+
+bool PhysicsReceiver::receiveRootPart(shared_ptr<PartInstance>& part, RakNet::BitStream& inBitstream)
+{
+    NETPROFILE_START("receiveRootPart", &inBitstream);
+    bool answer = receivePart(part, inBitstream);
+
+    if (part)
+    {
+        if (part->getPartPrimitive()->getAssembly()->computeIsGrounded())
+        {
+            part.reset();
+        }
+        else if (!okDistributedReceivePart(part))
+        {
+            part.reset();
+        }
+    }
+    NETPROFILE_END("receiveRootPart", &inBitstream);
+    return answer;
+}
+
+
+bool PhysicsReceiver::receivePart(shared_ptr<PartInstance>& part, RakNet::BitStream& inBitstream)
+{
+    shared_ptr<Instance> instance;
+    Aya::Guid::Data id;
+    if (replicator->deserializeInstanceRef(inBitstream, instance, id))
+    {
+        if (instance == NULL)
+        {
+            return false; // packet end tag
+        }
+        part = Instance::fastSharedDynamicCast<PartInstance>(instance);
+    }
+    else
+    {
+        if (replicator->settings().printPhysicsErrors)
+        {
+            Aya::StandardOut::singleton()->printf(Aya::MESSAGE_WARNING, "Physics-in of unidentified %s", id.readableString().c_str());
+        }
+    }
+
+    if (part)
+    {
+        if (!PartInstance::nonNullInWorkspace(part))
+        {
+            if (replicator->settings().printPhysicsErrors)
+            {
+                Aya::StandardOut::singleton()->printf(Aya::MESSAGE_INFO, "Physics-in of part not in workspace %s", id.readableString().c_str());
+            }
+            part.reset();
+        }
+    }
+    return true;
+}
+
+
+void PhysicsReceiver::renderPartMovementPath(Adorn* adorn)
+{
+
+    for (boost::circular_buffer<MovementWaypointAdorn>::iterator iter = movementWaypointList.begin(); iter != movementWaypointList.end(); iter++)
+    {
+        MovementWaypointAdorn wp = *iter;
+        DrawAdorn::star(adorn, wp.position, wp.size, wp.color, wp.color, wp.color);
+
+        if (wp.text.length() > 0)
+        {
+            Vector3 aboveStar = wp.position;
+            aboveStar.y += 5.0f;
+
+            const Camera& camera = *adorn->getCamera();
+            Vector3 screenLoc = camera.project(aboveStar);
+            if (screenLoc.z == std::numeric_limits<float>::infinity())
+                continue;
+
+            adorn->drawFont2D(wp.text, screenLoc.xy(), 10.0f, false, Color4(Color3::white(), 1.0f), Color4(Color3::black(), 1.0f),
+                Text::FONT_ARIALBOLD, Text::XALIGN_CENTER, Text::YALIGN_BOTTOM);
+        }
+    }
+
+    for (boost::circular_buffer<MovementVectorAdorn>::iterator iter = movementVectorList.begin(); iter != movementVectorList.end(); iter++)
+    {
+        MovementVectorAdorn wp = *iter;
+        adorn->line3d(wp.startPos, wp.endPos, wp.color);
+    }
+}

engine/network/src/PhysicsReceiver.hpp

@@ -0,0 +1,144 @@
+#pragma once
+
+#include "DataModel/Workspace.hpp"
+
+#include "MechanismItem.hpp"
+#include "ReplicatorStats.hpp"
+#include "Declarations.hpp"
+
+#include "boost.hpp"
+#include <vector>
+
+#include "RakNet/RakNetTypes.hpp"
+#include "RakNet/GetTime.hpp"
+
+#include "Base/IAdornable.hpp"
+
+namespace RakNet
+{
+class BitStream;
+}
+
+namespace Aya
+{
+class PartInstance;
+class CompactCFrame;
+
+namespace Network
+{
+
+namespace PathBasedMovementDebug
+{
+struct NodeDebugInfo
+{
+    Color3 color;
+    float size;
+    bool show;
+};
+} // namespace PathBasedMovementDebug
+
+class DeserializedTouchItem : public DeserializedItem
+{
+public:
+    std::vector<TouchPair> touchPairs;
+
+    DeserializedTouchItem() {}
+    ~DeserializedTouchItem() {}
+
+    /*implement*/ void process(Replicator& replicator);
+};
+
+class Replicator;
+
+class AyaBaseClass PhysicsReceiver : boost::noncopyable
+{
+    shared_ptr<PhysicsService> physicsService;
+
+    void readMovementHistory(
+        RakNet::BitStream& bitStream, RemoteTime remoteSendTime, PartInstance* rootPart, MechanismItem& mechanismItem, int& numNodesInHistory);
+    void readMechanismAttributes(RakNet::BitStream& bitStream, MechanismItem& item);
+
+    void readAssembly(RakNet::BitStream& bitstream, PartInstance* rootPart, MechanismItem& mechanismItem, bool crossPacketCompression);
+    void readPV(RakNet::BitStream& bitStream, AssemblyItem& item, bool crossPacketCompression);
+    void readCoordinateFrame(RakNet::BitStream& bitStream, CoordinateFrame& cFrame);
+    void readVelocity(RakNet::BitStream& bitStream, Velocity& velocity);
+    void readMotorAngles(RakNet::BitStream& bitStream, AssemblyItem& item);
+    void readCompactCFrame(RakNet::BitStream& bitStream, CompactCFrame& cFrame);
+
+    bool receivePart(shared_ptr<PartInstance>& part, RakNet::BitStream& inBitstream);
+
+protected:
+    struct MovementWaypointAdorn
+    {
+        Vector3 position;
+        Aya::Color4 color;
+        float size;
+        std::string text;
+        MovementWaypointAdorn(const Vector3& p, const Aya::Color4& c, float s, const std::string& debugText)
+        {
+            position = p;
+            color = c;
+            size = s;
+            text = debugText;
+        }
+    };
+
+    struct MovementVectorAdorn
+    {
+        Vector3 startPos;
+        Vector3 endPos;
+        Aya::Color4 color;
+        MovementVectorAdorn(const Vector3& start, const Vector3& end, const Aya::Color4& c)
+        {
+            startPos = start;
+            endPos = end;
+            color = c;
+        }
+    };
+
+    const bool iAmServer;
+
+    Replicator* const replicator;
+    ReplicatorStats::PhysicsReceiverStats* stats;
+    Time now; // or reasonably close to it.
+
+    struct TimedCF
+    {
+        CoordinateFrame cf;
+        float timeToEnd;
+    };
+    std::vector<TimedCF> nodeStack;
+
+    boost::circular_buffer<MovementWaypointAdorn> movementWaypointList;
+    boost::circular_buffer<MovementVectorAdorn> movementVectorList;
+    void addWayPointAdorn(const Vector3& p, const PathBasedMovementDebug::NodeDebugInfo& info, const std::string& debugText = "");
+    void addVectorAdorn(const Vector3& start, const Vector3& end, const Aya::Color4& c);
+
+    bool okDistributedReceivePart(const shared_ptr<PartInstance>& part);
+    bool receiveRootPart(shared_ptr<PartInstance>& part, RakNet::BitStream& inBitstream);
+
+public:
+    PhysicsReceiver(Replicator* replicator, bool isServer);
+
+    virtual void start(shared_ptr<PhysicsReceiver> physicsReceiver) {}
+
+    virtual ~PhysicsReceiver() {}
+
+    void setTime(Time now_);
+    void receiveMechanism(
+        RakNet::BitStream& bitStream, PartInstance* rootPart, MechanismItem& item, RemoteTime remoteSendTime, int& numNodesInHistory);
+    void receiveMechanismCFrames(RakNet::BitStream& bitStream, RakNet::Time timeStamp, const Aya::RemoteTime& remoteSendTime);
+    void setPhysics(const MechanismItem& item, const RemoteTime& remoteSendTime = RemoteTime(), const RakNet::TimeMS = 0, int numNodesInHistory = 0);
+
+    virtual void receivePacket(RakNet::BitStream& bitsream, RakNet::Time timeStamp, ReplicatorStats::PhysicsReceiverStats* stats) = 0;
+
+    void deserializeTouches(RakNet::BitStream& bitstream, const RakNet::SystemAddress& from, std::vector<TouchPair>& touchPairs);
+    bool deserializeTouch(RakNet::BitStream& bitstream, const RakNet::SystemAddress& from, TouchPair& touchPair);
+    void processTouchPair(const TouchPair& tp);
+    void readTouches(RakNet::BitStream& bitstream, const RakNet::SystemAddress& from);
+
+    void renderPartMovementPath(Adorn* adorn);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/PhysicsSender.cpp

@@ -0,0 +1,635 @@
+
+
+#include "PhysicsSender.hpp"
+#include "Replicator.hpp"
+#include "Compressor.hpp"
+#include "RakNet/RakNetStatistics.hpp"
+#include "ConcurrentRakPeer.hpp"
+#include "Util.hpp"
+
+#include "Utility/Profiling.hpp"
+
+#include "Utility/ObscureValue.hpp"
+
+
+#include "Kernel/Constants.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+#include "DataModel/PartInstance.hpp"
+
+
+#include "World/World.hpp"
+#include "World/Primitive.hpp"
+#include "World/Assembly.hpp"
+#include "World/Mechanism.hpp"
+#include "DataModel/DataModel.hpp"
+
+#include "RoundRobinPhysicsSender.hpp"
+#include "DataModel/PhysicsService.hpp"
+
+
+using namespace Aya;
+using namespace Aya::Network;
+
+FASTINTVARIABLE(NumPhysicsTouchPacketsPerStep, 1)
+DYNAMIC_FASTINTVARIABLE(NumPhysicsPacketsPerStep, 1)
+DYNAMIC_FASTINTVARIABLE(PhysicsSenderRate, 15)
+DYNAMIC_FASTFLAG(CleanUpInterpolationTimestamps)
+
+class PhysicsSender::Job : public ReplicatorJob
+{
+    weak_ptr<PhysicsSender> physicsSender;
+    ObscureValue<float> physicsSendRate;
+    PacketPriority packetPriority;
+    unsigned int skipSteps;
+
+public:
+    Job(shared_ptr<PhysicsSender> physicsSender)
+        : ReplicatorJob("Replicator SendPhysics", physicsSender->replicator, DataModelJob::PhysicsOut)
+        , physicsSender(physicsSender)
+        , physicsSendRate(replicator->settings().getPhysicsSendRate())
+        , packetPriority(replicator->settings().getPhysicsSendPriority())
+        , skipSteps(0)
+    {
+        if (dynamic_cast<RoundRobinPhysicsSender*>(physicsSender.get()))
+        {
+            physicsSendRate = replicator->settings().getClientPhysicsSendRate();
+        }
+
+        // TODO: set as default in NetworkSettings
+        physicsSendRate = (float)DFInt::PhysicsSenderRate;
+
+        cyclicExecutive = true;
+        cyclicPriority = CyclicExecutiveJobPriority_Network_ProcessOutgoing;
+    }
+
+private:
+    Time::Interval sleepTime(const Stats& stats)
+    {
+        return computeStandardSleepTime(stats, physicsSendRate);
+    }
+
+    virtual Error error(const Stats& stats)
+    {
+        if (TaskScheduler::singleton().isCyclicExecutive() && DFFlag::CleanUpInterpolationTimestamps)
+        {
+            return 1.0f; // ALWAYS RUN
+        }
+        return computeStandardErrorCyclicExecutiveSleeping(stats, physicsSendRate);
+    }
+
+    virtual TaskScheduler::StepResult stepDataModelJob(const Stats& stats)
+    {
+        if (DFFlag::CleanUpInterpolationTimestamps)
+        {
+            int stepsTillRun = std::max(Constants::uiStepsPerSec() / DFInt::PhysicsSenderRate, 1);
+            if (skipSteps != (stepsTillRun - 1))
+            {
+                skipSteps++;
+                return TaskScheduler::Stepped;
+            }
+            else
+            {
+                skipSteps = 0;
+            }
+        }
+
+        if (shared_ptr<PhysicsSender> safePhysicsSender = physicsSender.lock())
+        {
+            safePhysicsSender->step();
+            ReplicatorStats::PhysicsSenderStats& physStats = safePhysicsSender->replicator.physicsSenderStats();
+
+            const int limit = replicator->settings().sendPacketBufferLimit;
+            if (limit == -1)
+            {
+                safePhysicsSender->sendPacket(safePhysicsSender->sendPacketsPerStep, packetPriority, &physStats);
+                physStats.physicsSentThrottle.sample(0.0); // not throttled
+            }
+            else if (replicator)
+            {
+                int bufferCount = replicator->getBufferCountAvailable(limit, packetPriority);
+                safePhysicsSender->sendPacket(bufferCount, packetPriority, &physStats);
+                physStats.physicsSentThrottle.sample(1.0 - bufferCount / limit);
+            }
+
+            return TaskScheduler::Stepped;
+        }
+        return TaskScheduler::Done;
+    }
+};
+
+
+class PhysicsSender::TouchJob : public ReplicatorJob
+{
+    weak_ptr<PhysicsSender> physicsSender;
+    float sendRate;
+    PacketPriority packetPriority;
+
+public:
+    TouchJob(shared_ptr<PhysicsSender> physicsSender)
+        : ReplicatorJob("Replicator SendTouches", physicsSender->replicator, DataModelJob::PhysicsOut)
+        , physicsSender(physicsSender)
+        , sendRate(replicator->settings().touchSendRate)
+        , packetPriority(replicator->settings().getPhysicsSendPriority())
+    {
+        cyclicExecutive = true;
+        cyclicPriority = CyclicExecutiveJobPriority_Network_ProcessOutgoing;
+    }
+
+private:
+    Time::Interval sleepTime(const Stats& stats)
+    {
+        return computeStandardSleepTime(stats, sendRate);
+    }
+
+    virtual Error error(const Stats& stats)
+    {
+        if (canSendPacket(replicator, packetPriority))
+            if (shared_ptr<PhysicsSender> safePhysicsSender = physicsSender.lock())
+                if (safePhysicsSender->pendingTouchCount() > 0)
+                    return computeStandardErrorCyclicExecutiveSleeping(stats, sendRate);
+        Error error;
+        return error;
+    }
+
+    virtual TaskScheduler::StepResult stepDataModelJob(const Stats& stats)
+    {
+        if (shared_ptr<PhysicsSender> safePhysicsSender = physicsSender.lock())
+        {
+            safePhysicsSender->sendTouches(packetPriority);
+            return TaskScheduler::Stepped;
+        }
+        return TaskScheduler::Done;
+    }
+};
+
+size_t PhysicsSender::pendingTouchCount()
+{
+    return physicsService->pendingTouchCount();
+}
+
+template<typename T>
+void PhysicsSender::writeTouches(RakNet::BitStream& bitStream, unsigned int maxBitStreamSize, T& touchPairList)
+{
+    Aya::SystemAddress addr = RakNetToRbxAddress(replicator.remotePlayerId);
+    typename T::iterator iter;
+    for (iter = touchPairList.begin(); iter != touchPairList.end(); ++iter)
+    {
+        if (iter->originator == addr)
+            continue;
+
+        const bool touched = iter->type == TouchPair::Touch;
+
+        int byteStart = bitStream.GetNumberOfBytesUsed();
+
+        replicator.serializeId(bitStream, iter->p1.get());
+        replicator.serializeId(bitStream, iter->p2.get());
+        bitStream << touched;
+
+        if (replicator.settings().printTouches)
+        {
+            if (touched)
+            {
+                Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE, "Replication: Touch:%s->%s >> %s, bytes: %d",
+                    iter->p1->getName().c_str(), iter->p2->getName().c_str(), RakNetAddressToString(replicator.remotePlayerId).c_str(),
+                    bitStream.GetNumberOfBytesUsed() - byteStart);
+            }
+            else
+            {
+                Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE, "Replication: Untouch:%s->%s >> %s, bytes: %d",
+                    iter->p1->getName().c_str(), iter->p2->getName().c_str(), RakNetAddressToString(replicator.remotePlayerId).c_str(),
+                    bitStream.GetNumberOfBytesUsed() - byteStart);
+            }
+        }
+
+        if (bitStream.GetNumberOfBytesUsed() > maxBitStreamSize)
+            break;
+    }
+
+    if (iter == touchPairList.end())
+        touchPairList.clear();
+    else
+        touchPairList.erase(touchPairList.begin(), iter);
+}
+
+
+void PhysicsSender::sendTouches(PacketPriority packetPriority)
+{
+    const unsigned int maxStreamSize(replicator.getPhysicsMtuSize());
+    int numPackets = FInt::NumPhysicsTouchPacketsPerStep;
+
+    // append new touches to send list
+    int newTouchId = physicsService->getTouchesId();
+    if (touchPairsId < newTouchId)
+    {
+        physicsService->getTouches(touchPairs);
+        touchPairsId = newTouchId;
+    }
+
+    ReplicatorStats::PhysicsSenderStats& physStats = replicator.physicsSenderStats();
+    physStats.waitingTouches.sample(touchPairs.size());
+
+    if (touchPairs.empty())
+        return;
+
+    while (!touchPairs.empty() && (numPackets > 0))
+    {
+        boost::shared_ptr<RakNet::BitStream> bitStream(new RakNet::BitStream(maxStreamSize));
+
+        *bitStream << (unsigned char)ID_PHYSICS_TOUCHES;
+
+        writeTouches(*bitStream, maxStreamSize, touchPairs);
+
+        // Packet "end" tag
+        replicator.serializeId(*bitStream, NULL);
+
+        // Send ID_PHYSICS_TOUCHES
+        replicator.rakPeer->Send(bitStream, packetPriority, RELIABLE_ORDERED, PHYSICS_CHANNEL, replicator.remotePlayerId, false);
+
+        physStats.touchPacketsSent.sample();
+        physStats.touchPacketsSentSize.sample(bitStream->GetNumberOfBytesUsed());
+        numPackets--;
+
+        if (touchPairs.empty())
+            physicsService->onTouchesSent();
+    }
+};
+
+PhysicsSender::PhysicsSender(Replicator& replicator)
+    : replicator(replicator)
+    , sendPacketsPerStep(DFInt::NumPhysicsPacketsPerStep)
+    , senderStats(NULL)
+    , touchPairsId(0)
+{
+
+    physicsService = shared_from(ServiceProvider::find<PhysicsService>(&replicator));
+};
+
+void PhysicsSender::start(shared_ptr<PhysicsSender> physicsSender)
+{
+    // Someone else is holding a shared_ptr to us at the moment
+    physicsSender->job = shared_ptr<Job>(new Job(physicsSender));
+    TaskScheduler::singleton().add(physicsSender->job);
+
+    physicsSender->touchJob = shared_ptr<TouchJob>(new TouchJob(physicsSender));
+    TaskScheduler::singleton().add(physicsSender->touchJob);
+}
+
+PhysicsSender::~PhysicsSender()
+{
+    TaskScheduler::singleton().remove(job);
+    job.reset();
+    TaskScheduler::singleton().remove(touchJob);
+    touchJob.reset();
+}
+
+/*
+        Format for a mechanism:
+
+        >> MechanismAttributes
+        >> PrimaryAssembly
+        >> done
+        while (!done)
+        {
+                >> ChildPart
+                >> ChildAssembly
+                >> done
+        }
+*/
+
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+Compressor::CompressionType PhysicsSender::getCompressionType(const Assembly* assembly, bool detailed)
+{
+    if (replicator.settings().distributedPhysicsEnabled && Mechanism::isComplexMovingMechanism(assembly))
+    {
+        return Compressor::UNCOMPRESSED;
+    }
+    else if (detailed)
+    {
+        return Compressor::RAKNET_COMPRESSED;
+    }
+    else
+    {
+        return Compressor::HEAVILY_COMPRESSED;
+    }
+}
+
+void PhysicsSender::sendChildPrimitiveCoordinateFrame(
+    Primitive* prim, RakNet::BitStream* bitStream, Replicator* replicator, Compressor::CompressionType compressionType)
+{
+    AYAASSERT(replicator->isStreamingEnabled());
+
+    if (PartInstance* part = PartInstance::fromPrimitive(prim))
+    {
+        if (replicator->isReplicationContainer(part))
+        {
+            // only update the client if the part is replicated
+            RakNet::BitStream& bitStreamRef = *bitStream;
+            if (replicator->trySerializeId(bitStreamRef, part))
+                writeCoordinateFrame(bitStreamRef, prim->getCoordinateFrame(), compressionType);
+        }
+    }
+}
+
+void PhysicsSender::sendMechanismCFrames(RakNet::BitStream& bitStream, const PartInstance* part, bool detailed)
+{
+    AYAASSERT(replicator.isStreamingEnabled());
+
+    int byteStart = bitStream.GetNumberOfBytesUsed();
+
+    const Primitive* primitive = part->getConstPartPrimitive();
+    const Mechanism* mechanism = Mechanism::getConstPrimitiveMechanism(primitive);
+    const Assembly* assembly = primitive->getConstAssembly();
+
+    AYAASSERT(mechanism);
+
+    Compressor::CompressionType compressionType = getCompressionType(assembly, detailed);
+
+    if (replicator.isReplicationContainer(part))
+    {
+        if (replicator.trySerializeId(bitStream, part))
+            writeCoordinateFrame(bitStream, part->getCoordinateFrame(), compressionType);
+    }
+
+    // send each part's cframe if they are in streamed region
+    const_cast<Mechanism*>(mechanism)->visitPrimitives(
+        boost::bind(&PhysicsSender::sendChildPrimitiveCoordinateFrame, this, _1, &bitStream, &replicator, compressionType));
+
+    replicator.serializeId(bitStream, NULL); // token: done
+
+    if (senderStats)
+    {
+        senderStats->details.mechanismCFrame.increment();
+        senderStats->details.mechanismCFrameSize.sample(bitStream.GetNumberOfBytesUsed() - byteStart);
+    }
+}
+
+void PhysicsSender::sendMechanism(RakNet::BitStream& bitStream, const PartInstance* part, bool detailed, const Time& lastSendTime,
+    CoordinateFrame& lastSendCFrame, float& accumulatedError, const PartInstance* playerHead)
+{
+    int byteStart = bitStream.GetNumberOfBytesUsed();
+
+    const Assembly* assembly = part->getConstPartPrimitive()->getConstAssembly();
+
+    AYAASSERT(assembly);
+    // AYAASSERT(Mechanism::isMovingAssemblyRoot(assembly));
+
+    Compressor::CompressionType compressionType = getCompressionType(assembly, detailed);
+
+    writeMechanismAttributes(bitStream, assembly);
+
+    bool hasMovement = true;
+    if (!writeMovementHistory(bitStream, part->getMovementHistory(), assembly, lastSendTime, lastSendCFrame, hasMovement, compressionType,
+            accumulatedError, playerHead))
+    {
+        writeAssembly(bitStream, assembly, compressionType);
+        lastSendCFrame = part->getCoordinateFrame();
+    }
+
+    if (hasMovement)
+    {
+        assembly->visitConstDescendentAssemblies(boost::bind(&PhysicsSender::sendChildAssembly, this, &bitStream, _1, compressionType));
+    }
+    else if (compressionType == Compressor::UNCOMPRESSED)
+    {
+        assembly->visitConstDescendentAssemblies(boost::bind(&PhysicsSender::addChildRotationError, this, _1, boost::ref(accumulatedError)));
+    }
+
+    bitStream << true; // token - done with child assemblies
+
+    if (senderStats)
+    {
+        senderStats->details.mechanism.increment();
+        senderStats->details.mechanismSize.sample(bitStream.GetNumberOfBytesUsed() - byteStart);
+    }
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+bool PhysicsSender::writeMovementHistory(RakNet::BitStream& bitStream, const MovementHistory& history, const Assembly* assembly,
+    const Time& lastSendTime, CoordinateFrame& lastSendCFrame, bool& hasMovement, Compressor::CompressionType compressionType,
+    float& accumulatedError, const PartInstance* playerHead)
+{
+    hasMovement = true;
+    return false; // client will not use path based part movement
+}
+
+void PhysicsSender::writeMechanismAttributes(RakNet::BitStream& bitStream, const Assembly* assembly)
+{
+    bool hasState = (assembly->getNetworkHumanoidState() != 0);
+    bitStream << hasState;
+    if (hasState)
+    {
+        bitStream << assembly->getNetworkHumanoidState();
+    }
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void PhysicsSender::sendChildAssembly(RakNet::BitStream* bitStream, const Assembly* assembly, Compressor::CompressionType compressionType)
+{
+    RakNet::BitStream& bitStreamRef = *bitStream;
+    const PartInstance* part = PartInstance::fromConstAssembly(assembly);
+    AYAASSERT(part);
+
+    // send only if we can serialize the part id
+    if (replicator.canSerializeId(part))
+    {
+        bitStreamRef << false; // token - not done with child assemblies
+        replicator.serializeId(bitStreamRef, part);
+        writeAssembly(bitStreamRef, assembly, compressionType);
+    }
+}
+
+void PhysicsSender::addChildRotationError(const Assembly* assembly, float& accumulatedError)
+{
+    const PartInstance* part = PartInstance::fromConstAssembly(assembly);
+    AYAASSERT(part);
+    accumulatedError += part->getRotationalVelocity().magnitude();
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void PhysicsSender::writeAssembly(
+    RakNet::BitStream& bitStream, const Assembly* assembly, Compressor::CompressionType compressionType, bool crossPacketCompression)
+{
+    writePV(bitStream, assembly, compressionType, crossPacketCompression);
+    writeMotorAngles(bitStream, assembly, compressionType);
+}
+
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////
+void PhysicsSender::writePV(RakNet::BitStream& bitStream, const Assembly* a, Compressor::CompressionType compressionType, bool crossPacketCompression)
+{
+    const PV& pv = a->getConstAssemblyPrimitive()->getPV();
+    writeCoordinateFrame(bitStream, pv.position, compressionType);
+    writeVelocity(bitStream, pv.velocity);
+}
+
+void PhysicsSender::writeCoordinateFrame(RakNet::BitStream& bitStream, const CoordinateFrame& cFrame, Compressor::CompressionType compressionType)
+{
+    int byteStart = bitStream.GetNumberOfBytesUsed();
+    Compressor::writeTranslation(bitStream, cFrame.translation, compressionType);
+    if (senderStats)
+    {
+        senderStats->details.translation.increment();
+        senderStats->details.translationSize.sample(bitStream.GetNumberOfBytesUsed() - byteStart);
+    }
+    byteStart = bitStream.GetNumberOfBytesUsed();
+    Compressor::writeRotation(bitStream, cFrame.rotation, compressionType);
+    if (senderStats)
+    {
+        senderStats->details.rotation.increment();
+        senderStats->details.rotationSize.sample(bitStream.GetNumberOfBytesUsed() - byteStart);
+    }
+}
+
+void PhysicsSender::writeVelocity(RakNet::BitStream& bitStream, const Velocity& velocity)
+{
+    if (replicator.settings().distributedPhysicsEnabled)
+    {
+        int byteStart = bitStream.GetNumberOfBytesUsed();
+
+        bitStream.WriteVector(velocity.linear.x, velocity.linear.y, velocity.linear.z);
+        bitStream.WriteVector(velocity.rotational.x, velocity.rotational.y, velocity.rotational.z);
+
+        if (senderStats)
+        {
+            senderStats->details.velocity.increment();
+            senderStats->details.velocitySize.sample(bitStream.GetNumberOfBytesUsed() - byteStart);
+        }
+    }
+}
+///////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void PhysicsSender::writeMotorAngles(RakNet::BitStream& bitStream, const Assembly* assembly, Compressor::CompressionType compressionType)
+{
+    tempMotorAngles.resize(0);
+    assembly->getPhysics(tempMotorAngles);
+
+    AYAASSERT(tempMotorAngles.size() < 255);
+
+    unsigned char compactNum = static_cast<unsigned char>(tempMotorAngles.size());
+    bitStream << compactNum;
+
+    for (int i = 0; i < tempMotorAngles.size(); ++i)
+    {
+        writeCompactCFrame(bitStream, tempMotorAngles[i], compressionType);
+    }
+}
+
+void PhysicsSender::writeCompactCFrame(RakNet::BitStream& bitStream, const CompactCFrame& cFrame, Compressor::CompressionType compressionType)
+{
+    bool hasTranslation = !cFrame.translation.isZero();
+    bool isSimpleZAngle = G3D::fuzzyEq(fabs(cFrame.getAxis().z), 1.0f);
+    bool hasRotation = !G3D::fuzzyEq(cFrame.getAngle(), 0.0f);
+
+    // common simple surface normal motor
+    if (!hasTranslation && hasRotation && isSimpleZAngle)
+    {
+        bitStream.Write1();
+        unsigned char byteAngle = Math::rotationToByte(cFrame.getAxis().z * cFrame.getAngle());
+        bitStream << byteAngle;
+    }
+    else
+    {
+        bitStream.Write0();
+        // two bits, has trans, and hasrot
+        bitStream << hasTranslation;
+        bitStream << hasRotation;
+
+        if (hasTranslation)
+        {
+            Compressor::writeTranslation(bitStream, cFrame.translation, compressionType);
+        }
+        if (hasRotation)
+        {
+            Vector3 axis = cFrame.getAxis();
+            bitStream.WriteVector(axis.x, axis.y,
+                axis.z); // would be nice to use WriteNormVector, but it's broken. (gives out NANs in certain situations, it needs fixing)
+
+            unsigned char byteAngle = Math::rotationToByte(cFrame.getAngle()); // todo: losing one bit here, angle is always positive.
+            bitStream << byteAngle;
+        }
+    }
+}
+
+
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+bool PhysicsSender::canSend(const PartInstance* part, const Assembly* assembly, RakNet::BitStream& bitStream)
+{
+    AYAASSERT(!assembly || !part || (assembly->getConstAssemblyPrimitive() == part->getConstPartPrimitive()));
+
+    if (assembly && part)
+    {
+        if (!replicator.settings().distributedPhysicsEnabled || replicator.checkDistributedSend(part))
+        {
+            if (replicator.isStreamingEnabled())
+                return true;
+            else if (!replicator.isSerializePending(part))
+                return true;
+        }
+    }
+    return false;
+}
+
+bool PhysicsSender::sendPhysicsData(RakNet::BitStream& bitStream, const PartInstance* part, bool detailed, const Time& lastSendTime,
+    CoordinateFrame& lastSendCFrame, float& accumulatedError, const PartInstance* playerHead)
+{
+    AYAASSERT(part);
+    if (part && Assembly::isAssemblyRootPrimitive(part->getConstPartPrimitive()))
+    {
+        senderStats = NULL;
+        if (replicator.settings().trackPhysicsDetails)
+        {
+            senderStats = &(replicator.replicatorStats.physicsSenderStats);
+        }
+
+        const Assembly* assembly = part->getConstPartPrimitive()->getConstAssembly();
+        if (canSend(part, assembly, bitStream))
+        {
+            if (replicator.isStreamingEnabled())
+            {
+                bitStream << false; // token: not done with packet
+
+                if (!replicator.isInStreamedRegions(part->getConstPartPrimitive()->getExtentsWorld()))
+                {
+                    bitStream << true; // token: CFrame only
+                    sendMechanismCFrames(bitStream, part, detailed);
+                    lastSendCFrame = part->getCoordinateFrame();
+                    return true;
+                }
+                else
+                {
+                    bitStream << false; // token: (not) CFrame only
+                    if (replicator.trySerializeId(bitStream, part))
+                    {
+                        sendMechanism(bitStream, part, detailed, lastSendTime, lastSendCFrame, accumulatedError, playerHead);
+                        return true;
+                    }
+                    else
+                    {
+                        replicator.serializeId(bitStream, NULL); // token: done with this mechanism
+                        return false;
+                    }
+                }
+            }
+            else
+            {
+                if (replicator.trySerializeId(bitStream, part))
+                {
+                    sendMechanism(bitStream, part, detailed, lastSendTime, lastSendCFrame, accumulatedError, playerHead);
+                    return true;
+                }
+            }
+        }
+    }
+
+    return false;
+}

engine/network/src/PhysicsSender.hpp

@@ -0,0 +1,117 @@
+#pragma once
+
+#include "Compressor.hpp"
+#include "Declarations.hpp"
+
+#include "Utility/G3DCore.hpp"
+
+#include "Utility/PV.hpp"
+
+#include "boost.hpp"
+#include "signal.hpp"
+
+#include "time.hpp"
+
+#include <vector>
+#include "CompactCFrame.hpp"
+#include <boost/unordered_set.hpp>
+#include "RakNet/PacketPriority.hpp"
+#include "RunningAverage.hpp"
+
+#include "ReplicatorStats.hpp"
+
+namespace RakNet
+{
+class BitStream;
+}
+
+namespace Aya
+{
+
+class Assembly;
+class Primitive;
+struct TouchPair;
+class PartInstance;
+class MovementHistory;
+class PhysicsService;
+
+namespace Network
+{
+
+class Replicator;
+
+class AyaBaseClass PhysicsSender : boost::noncopyable
+{
+private:
+    class Job; // The Job that sends physics
+    shared_ptr<Job> job;
+    class TouchJob; // The Job that sends Touches
+    shared_ptr<TouchJob> touchJob;
+
+    std::list<TouchPair> touchPairs;
+    int touchPairsId;
+
+private:
+    G3D::Array<CompactCFrame> tempMotorAngles;
+
+    void writeMechanismAttributes(RakNet::BitStream& bitStream, const Assembly* assembly);
+
+    void sendChildAssembly(RakNet::BitStream* bitStream, const Assembly* assembly, Compressor::CompressionType compressionType);
+    void sendChildPrimitiveCoordinateFrame(
+        Primitive* prim, RakNet::BitStream* bitStream, Replicator* replicator, Compressor::CompressionType compressionType);
+
+    void writeCompactCFrame(RakNet::BitStream& bitStream, const CompactCFrame& cFrame, Compressor::CompressionType compressionType);
+    void writeCoordinateFrame(RakNet::BitStream& bitStream, const CoordinateFrame& cFrame, Compressor::CompressionType compressionType);
+
+    Compressor::CompressionType getCompressionType(const Assembly* assembly, bool detailed);
+
+    void addChildRotationError(const Assembly* assembly, float& accumulatedError);
+
+protected:
+    Replicator& replicator;
+    ReplicatorStats::PhysicsSenderStats* senderStats;
+
+    shared_ptr<PhysicsService> physicsService;
+
+    RunningAverage<int> itemsPerPacket;
+    int sendPacketsPerStep;
+
+    Time currentStepTimestamp;
+
+    bool canSend(const PartInstance* part, const Assembly* assembly, RakNet::BitStream& bitStream);
+    bool sendPhysicsData(RakNet::BitStream& bitStream, const PartInstance* part, bool detailed, const Time& lastSendTime,
+        CoordinateFrame& lastSendCFrame, float& accumulatedError, const PartInstance* playerHead);
+
+    void sendMechanism(RakNet::BitStream& bitStream, const PartInstance* part, bool detailed, const Time& lastSendTime,
+        CoordinateFrame& lastSendCFrame, float& accumulatedError, const PartInstance* playerHead);
+    void sendMechanismCFrames(RakNet::BitStream& bitStream, const PartInstance* part, bool detailed);
+    void writeVelocity(RakNet::BitStream& bitStream, const Velocity& velocity);
+    void writeMotorAngles(RakNet::BitStream& bitStream, const Assembly* assembly, Compressor::CompressionType compressionType);
+
+    virtual void writePV(
+        RakNet::BitStream& bitStream, const Assembly* assembly, Compressor::CompressionType compressionType, bool crossPacketCompression);
+
+    virtual void writeAssembly(
+        RakNet::BitStream& bitStream, const Assembly* assembly, Compressor::CompressionType compressionType, bool crossPacketCompression = false);
+    virtual bool writeMovementHistory(RakNet::BitStream& bitStream, const MovementHistory& history, const Assembly* assembly,
+        const Time& lastSendTime, CoordinateFrame& lastSendCFrame, bool& hasMovement, Compressor::CompressionType compressionType,
+        float& accumulatedError, const PartInstance* playerHead);
+
+public:
+    PhysicsSender(Replicator& replicator);
+    static void start(shared_ptr<PhysicsSender> physicsSender);
+    virtual ~PhysicsSender();
+
+    virtual void step() = 0;
+    void sendTouches(PacketPriority packetPriority);
+
+    template<typename T>
+    void writeTouches(RakNet::BitStream& bitstream, unsigned int maxBitStreamSize, T& touchPairList);
+
+    virtual int sendPacket(int maxPackets, PacketPriority packetPriority, ReplicatorStats::PhysicsSenderStats* stats) = 0;
+
+    size_t pendingTouchCount();
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Player.hpp

@@ -0,0 +1,671 @@
+
+
+#pragma once
+
+#include "time.hpp"
+
+#include "Tree/Instance.hpp"
+
+#include "DataModel/Team.hpp"
+
+#include "DataModel/FriendService.hpp"
+
+#include "DataModel/Camera.hpp"
+
+#include "DataModel/StarterPlayerService.hpp"
+
+#include "Utility/BrickColor.hpp"
+
+#include "Utility/G3DCore.hpp"
+
+#include "Utility/HeapValue.hpp"
+
+#include "Utility/Rect.hpp"
+
+#include "Utility/RunningAverageState.hpp"
+
+
+// for player mouse
+#include "DataModel/Mouse.hpp"
+
+
+#include <boost/scoped_ptr.hpp>
+#include "DataModel/TeleportService.hpp"
+
+
+LOGGROUP(Network)
+LOGGROUP(Player)
+
+namespace RakNet
+{
+struct SystemAddress;
+}
+
+namespace Aya
+{
+class Visit;
+class ModelInstance;
+class Backpack;
+class TimerService;
+class Region2;
+class DataModelMesh;
+class Adorn;
+class Primitive;
+class PartInstance;
+class Humanoid;
+class DataModel;
+class SpawnLocation;
+
+namespace Network
+{
+class PersistentDataStore;
+
+extern const char* const sPlayer;
+
+class Player
+    : public DescribedCreatable<Player, Instance, sPlayer, Reflection::ClassDescriptor::RUNTIME>
+    , public Diagnostics::Countable<Player>
+{
+private:
+    typedef DescribedCreatable<Player, Instance, sPlayer, Reflection::ClassDescriptor::RUNTIME> Super;
+    shared_ptr<ModelInstance> character; // The ModelInstance that this player is represented by
+    BrickColor teamColor;
+    bool neutral;
+    int loadAppearanceCounter;
+    int dataComplexityLimit;
+    bool dataReady;
+    weak_ptr<SpawnLocation> respawnLocation;
+
+    bool loadedStarterGear;
+
+    // User Profile data
+    std::string displayName;
+    bool superSafeChat;
+    bool under13;
+    int userId;
+
+    // For Group Building
+    bool hasGroupBuildTools;
+    HeapValue<int> personalServerRank;
+
+    Aya::signals::scoped_connection characterDiedConnection;
+    Aya::signals::scoped_connection backendDiedSignalConnection;
+    Aya::signals::scoped_connection spawnLocationChangedConnection;
+    Aya::signals::scoped_connection simulationRadiusChangedConnection;
+    Aya::signals::scoped_connection setShutdownMessageConnection;
+    boost::function<void()> teamStatusChangedCallback;
+
+    // CharacterAppearance is the url of a handler that returns a delimited list of asset ids
+    std::string characterAppearance;
+    bool canLoadCharacterAppearance;
+
+    // Distributed Simulation
+    float simulationRadius;
+    float maxSimulationRadius;
+
+    std::string osPlatform;
+
+    int loadingInstances;
+    bool appearanceDidLoad;
+    bool characterAppearanceLoaded;
+
+    // Only valid on the server!
+    shared_ptr<RakNet::SystemAddress> remoteAddress;
+
+    bool forceEarlySpawnLocationCalculation;
+    bool hasSpawnedAtLeastOnce;
+    std::string teleportSpawnName;
+    bool teleported;   // describes whether player is currently in the middle of teleporting out
+    bool teleportedIn; // describes whether player joined current place by teleporting
+
+    std::string gameSessionID;
+
+    struct SpawnData
+    {
+        Vector3 position;
+        int forceFieldDuration;
+        CoordinateFrame cf;
+
+        SpawnData()
+            : position(Vector3::zero())
+            , forceFieldDuration(0)
+        {
+        }
+        SpawnData(Vector3 position, int forceFieldDuration, CoordinateFrame frame)
+            : position(position)
+            , forceFieldDuration(forceFieldDuration)
+            , cf(frame)
+        {
+        }
+    };
+
+    boost::scoped_ptr<SpawnData> nextSpawnLocation;
+
+    int followUserId;
+
+    // Instance
+    /*override*/ bool askAddChild(const Instance* instance) const
+    {
+        return true;
+    }
+    /*override*/ void verifySetParent(const Instance* instance) const;
+    /*override*/ void onServiceProvider(ServiceProvider* oldProvider, ServiceProvider* newProvider);
+
+    void onCharacterChangedFrontend();
+
+    void registerLocalPlayerNotIdle();
+
+    void loadInsertPanel(DataModel* dataModel);
+
+    shared_ptr<PersistentDataStore> persistentData;
+    std::list<boost::function<void(bool)>> waitForDataReadyWaiters;
+    void fireWaitForDataReady();
+    static void LoadDataResultHelper(boost::weak_ptr<Player> weakPlayer, shared_ptr<const Reflection::ValueMap> result);
+    void loadDataResult(shared_ptr<const Reflection::ValueMap> result);
+
+    bool autoJumpEnabled;
+
+    Team* myTeam;
+
+public:
+    enum MembershipType
+    {
+        MEMBERSHIP_NONE = 0,
+        MEMBERSHIP_BUILDERS_CLUB = 1,
+        MEMBERSHIP_TURBO_BUILDERS_CLUB = 2,
+        MEMBERSHIP_OUTRAGEOUS_BUILDERS_CLUB = 3
+    };
+    enum ChatMode
+    {
+        CHAT_MODE_MENU = 0,
+        CHAT_MODE_TEXT_AND_MENU = 1
+    };
+    enum ChatFilterType
+    {
+        CHAT_FILTER_WHITELIST,
+        CHAT_FILTER_BLACKLIST
+    };
+
+    // debug timer
+    Aya::Timer<Aya::Time::Fast> appearanceFetchTimer;
+
+    static Reflection::PropDescriptor<Player, std::string> prop_displayName;
+    static Reflection::PropDescriptor<Player, int> prop_userId;
+    static Reflection::PropDescriptor<Player, int> prop_userIdDeprecated;
+    static Reflection::PropDescriptor<Player, bool> prop_SuperSafeChat;
+    static Reflection::BoundProp<float> prop_SimulationRadius;
+    static Reflection::BoundProp<float> prop_MaxSimulationRadius;
+    static Reflection::PropDescriptor<Player, float> prop_DeprecatedMaxSimulationRadius;
+    static Reflection::BoundProp<std::string> prop_OsPlatform;
+
+    Aya::remote_signal<void(std::string, std::string, std::string)> scriptSecurityErrorSignal;
+    Aya::remote_signal<void(std::string, Vector3)> remoteInsertSignal;
+
+    Aya::remote_signal<void()> connectDiedSignalBackend;
+
+    Aya::remote_signal<void(bool, int)> remoteFriendServiceSignal;
+    Aya::remote_signal<void()> killPlayerSignal;
+    Aya::remote_signal<void(float)> simulationRadiusChangedSignal;
+
+    Aya::remote_signal<void(std::string)> statsSignal;
+    Aya::signals::scoped_connection charChildAddedConnection;
+
+    Aya::signal<void(std::string, shared_ptr<Instance>)> chattedSignal;
+    Aya::signal<void(shared_ptr<Instance>)> characterAddedSignal;
+    Aya::remote_signal<void(shared_ptr<Instance>)> CharacterAppearanceLoadedSignal;
+    Aya::signal<void(shared_ptr<Instance>)> characterRemovingSignal;
+    Aya::signal<void(Vector3)> nextSpawnLocationChangedSignal;
+    Aya::signal<void(double)> idledSignal;
+
+    Aya::signal<void(shared_ptr<Instance>, FriendService::FriendStatus)> friendStatusChangedSignal;
+
+    Aya::remote_signal<void(TeleportService::TeleportState, int, std::string)> onTeleportSignal;
+    Aya::remote_signal<void(TeleportService::TeleportState, shared_ptr<const Reflection::ValueTable>, shared_ptr<Instance>)> onTeleportInternalSignal;
+
+    Aya::remote_signal<void(std::string)> setShutdownMessageSignal;
+
+    Player();
+    ~Player();
+
+    void doFirstSpawnLocationCalculation(const ServiceProvider* serviceProvider, const std::string& preferedSpawnName);
+    void reportScriptSecurityError(const std::string& hash, const std::string& error, const std::string& stack);
+    void killPlayer();
+
+    void requestFriendship(shared_ptr<Instance> player);
+    void revokeFriendship(shared_ptr<Instance> player);
+
+    // Keyboard/Mouse input API
+    shared_ptr<Mouse> getMouse()
+    {
+        return mouse;
+    }
+    shared_ptr<Instance> getMouseInstance();
+
+    // Persistent Data API
+    void loadData();
+    void saveData();
+    void saveLeaderboardData();
+    int getDataComplexityLimit() const
+    {
+        return dataComplexityLimit;
+    }
+    void setDataComplexityLimit(int value);
+    int getDataComplexity() const;
+
+    bool getDataReady() const
+    {
+        return dataReady;
+    }
+
+    void waitForDataReady(boost::function<void(bool)> resumeFunction, boost::function<void(std::string)> errorFunction);
+    double loadNumber(std::string key);
+    void saveNumber(std::string key, double value);
+    std::string loadString(std::string key);
+    void saveString(std::string key, std::string value);
+    bool loadBoolean(std::string key);
+    void saveBoolean(std::string key, bool value);
+    shared_ptr<Instance> loadInstance(std::string key);
+    void saveInstance(std::string key, shared_ptr<Instance> value);
+    shared_ptr<const Reflection::ValueArray> loadList(std::string key);
+    void saveList(std::string key, shared_ptr<const Reflection::ValueArray> value);
+    shared_ptr<const Aya::Reflection::ValueMap> loadTable(std::string key);
+    void saveTable(std::string key, shared_ptr<const Aya::Reflection::ValueMap> value);
+
+    /*override*/ void setName(const std::string& value);
+    /*override*/ bool canClientCreate()
+    {
+        return true;
+    }
+
+    ModelInstance* getDangerousCharacter() const
+    {
+        return character.get();
+    }
+
+    boost::shared_ptr<ModelInstance> getSharedCharacter()
+    {
+        return character;
+    }
+    ModelInstance* getCharacter()
+    {
+        return character.get();
+    }
+    const ModelInstance* getConstCharacter() const
+    {
+        return character.get();
+    }
+    void setCharacter(ModelInstance* value);
+
+    const SpawnLocation* getConstRespawnLocation() const
+    {
+        if (shared_ptr<SpawnLocation> shared_respawnLocation = respawnLocation.lock())
+            return shared_respawnLocation.get();
+        else
+            return NULL;
+    }
+    SpawnLocation* getDangerousRespawnLocation() const
+    {
+        if (shared_ptr<SpawnLocation> shared_respawnLocation = respawnLocation.lock())
+            return shared_respawnLocation.get();
+        else
+            return NULL;
+    }
+    void setRespawnLocation(SpawnLocation* value);
+
+    bool getCanLoadCharacterAppearance() const
+    {
+        return canLoadCharacterAppearance;
+    }
+    void setCanLoadCharacterAppearance(bool value);
+
+    bool getAutoJumpEnabled() const
+    {
+        return autoJumpEnabled;
+    }
+    void setAutoJumpEnabled(bool value);
+
+    const PartInstance* hasCharacterHead(CoordinateFrame& headPos) const;
+
+    bool getHasGroupBuildTools() const
+    {
+        return hasGroupBuildTools;
+    }
+    void setHasGroupBuildTools(bool value);
+
+    void giveBuildTools();
+    void removeBuildTools();
+
+    void setAppearanceLoaded();
+
+    // Personal Server API
+    void getWebPersonalServerRank(boost::function<void(std::string)> resumeFunction, boost::function<void(std::string)> errorFunction);
+    void setWebPersonalServerRank(int newRank, boost::function<void(bool)> resumeFunction, boost::function<void(std::string)> errorFunction);
+
+    int getPersonalServerRank() const
+    {
+        return (personalServerRank > 255) ? 0 : personalServerRank;
+    } // 255 is max rank allowed by setter.
+    void setPersonalServerRank(int value);
+
+    const Primitive* getConstCharacterRoot() const;
+
+    Team* getTeam();
+    Team* getTeam2() const
+    {
+        return myTeam;
+    }; // used by reflection
+    BrickColor getTeamColor() const
+    {
+        return teamColor;
+    }
+    void setTeamColor(BrickColor value);
+    void setTeam(Team* team);
+
+    bool getNeutral() const
+    {
+        return neutral;
+    }
+    void setNeutral(bool value);
+
+    std::string getCharacterAppearance() const
+    {
+        return characterAppearance;
+    }
+    void setCharacterAppearance(const std::string& value);
+
+    static void setAppearanceParent(weak_ptr<Player> player, weak_ptr<Instance> instance, bool equipped);
+
+    static void setGearParent(weak_ptr<Player> player, weak_ptr<Instance> instance, bool equipped);
+
+    bool getSuperSafeChat() const;
+    void setSuperSafeChat(bool value);
+
+    float getDeprecatedMaxSimulationRadius() const
+    {
+        return 0.f;
+    }
+    void setDeprecatedMaxSimulationRadius(float val) {}
+
+    ChatMode getChatMode() const;
+
+    void setDisplayName(std::string value);
+    std::string getDisplayName() const
+    {
+        return displayName;
+    }
+
+    void setUnder13(bool value);
+    bool getUnder13()
+    {
+        return under13;
+    };
+
+    void setUserId(int value);
+    int getUserID() const
+    {
+        return userId;
+    }
+
+    void setGameSessionID(std::string value);
+    std::string getGameSessionID()
+    {
+        return gameSessionID;
+    }
+
+    bool isGuest() const
+    {
+        return userId < 0;
+    }
+    void setMembershipType(MembershipType value);
+    MembershipType getMembershipType() const
+    {
+        return membershipType;
+    }
+
+    void setCameraMode(Aya::Camera::CameraMode value);
+    Aya::Camera::CameraMode getCameraMode() const
+    {
+        return cameraMode;
+    }
+
+
+    void setNameDisplayDistance(float value);
+    float getNameDisplayDistance() const
+    {
+        return nameDisplayDistance;
+    }
+
+    void setHealthDisplayDistance(float value);
+    float getHealthDisplayDistance() const
+    {
+        return healthDisplayDistance;
+    }
+
+    void setCameraMaxZoomDistance(float _cameraMaxZoomDistance);
+    float getCameraMaxZoomDistance() const
+    {
+        return cameraMaxZoomDistance;
+    }
+
+    void setCameraMinZoomDistance(float _cameraMinZoomDistance);
+    float getCameraMinZoomDistance() const
+    {
+        return cameraMinZoomDistance;
+    }
+
+    void setDevEnableMouseLockOption(bool setting);
+    bool getDevEnableMouseLockOption() const
+    {
+        return enableMouseLockOption;
+    }
+    void setDevTouchCameraMode(StarterPlayerService::DeveloperTouchCameraMovementMode setting);
+    StarterPlayerService::DeveloperTouchCameraMovementMode getDevTouchCameraMode() const
+    {
+        return touchCameraMovementMode;
+    }
+    void setDevComputerCameraMode(StarterPlayerService::DeveloperComputerCameraMovementMode setting);
+    StarterPlayerService::DeveloperComputerCameraMovementMode getDevComputerCameraMode() const
+    {
+        return computerCameraMovementMode;
+    }
+    void setDevCameraOcclusionMode(StarterPlayerService::DeveloperCameraOcclusionMode setting);
+    StarterPlayerService::DeveloperCameraOcclusionMode getDevCameraOcclusionMode() const
+    {
+        return cameraOcclusionMode;
+    }
+
+    void setDevTouchMovementMode(StarterPlayerService::DeveloperTouchMovementMode setting);
+    StarterPlayerService::DeveloperTouchMovementMode getDevTouchMovementMode() const
+    {
+        return touchMovementMode;
+    }
+    void setDevComputerMovementMode(StarterPlayerService::DeveloperComputerMovementMode setting);
+    StarterPlayerService::DeveloperComputerMovementMode getDevComputerMovementMode() const
+    {
+        return computerMovementMode;
+    }
+
+    void setAccountAge(int value);
+    int getAccountAge() const
+    {
+        return accountAge;
+    }
+
+    void updateSimulationRadius(float value);
+    float getSimulationRadius() const
+    {
+        return simulationRadius;
+    }
+    void setSimulationRadius(float value);
+
+    float getMaxSimulationRadius() const
+    {
+        return maxSimulationRadius;
+    }
+    void setMaxSimulationRadius(float value);
+
+    std::string getOsPlatform() const
+    {
+        return osPlatform;
+    }
+
+    void rebuildBackpack();      // Copy the backpack from the StartPack service
+    void rebuildGui();           // Copy the gui from the StarterGui service
+    void rebuildPlayerScripts(); // Copy the LocalScripts from the StarterPlayer/PlayerScripts service
+
+    void createPlayerGui();
+
+    Backpack* getPlayerBackpack();
+    const Backpack* getConstPlayerBackpack() const;
+
+    void luaJumpCharacter();
+    void luaMoveCharacter(Vector2 walkDirection, float maxWalkDelta);
+
+    void move(Vector3 walkVector, bool relativeToCamera);
+
+    float distanceFromCharacter(Vector3 point);
+
+    void luaLoadCharacter(bool inGame);
+    void loadCharacter(bool inGame, std::string preferedSpawnName);
+
+    void removeCharacter();
+    void removeCharacterAppearance();
+    void removeCharacterAppearanceScript();
+
+    void loadCharacterAppearance(bool blockingCall);
+    void loadCharacterAppearanceScript(shared_ptr<Instance> asset);
+
+    static void onLocalPlayerNotIdle(Aya::ServiceProvider* serviceProvider);
+
+    void renderDPhysicsRegion(Adorn* adorn);
+    void renderStreamedRegion(Adorn* adorn);
+    void renderPartMovementPath(Adorn* adorn);
+
+    static bool physicsOutBandwidthExceeded(const Aya::Instance* context);
+    static double getNetworkBufferHealth(const Aya::Instance* context);
+    void reportStat(std::string stat);
+
+    void addToLoadingInstances(int newInstances)
+    {
+        loadingInstances += newInstances;
+    }
+    void removeFromLoadingInstances(int oldInstances)
+    {
+        loadingInstances -= oldInstances;
+    }
+    int getLoadingInstances()
+    {
+        return loadingInstances;
+    }
+
+    bool getAppearanceDidLoad() const
+    {
+        return appearanceDidLoad;
+    }
+    bool getAppearanceDidLoadNonConst()
+    {
+        return appearanceDidLoad;
+    }
+    void setAppearanceDidLoad(bool value);
+    bool getCharacterAppearanceLoaded() const
+    {
+        return characterAppearanceLoaded;
+    }
+    void setCharacterAppearanceLoaded(bool value);
+
+    void onFriendStatusChanged(shared_ptr<Instance> player, FriendService::FriendStatus friendStatus);
+    FriendService::FriendStatus getFriendStatus(shared_ptr<Instance> player);
+    void isFriendsWith(int otherUserId, boost::function<void(bool)> resumeFunction, boost::function<void(std::string)> errorFunction);
+    void isBestFriendsWith(int otherUserId, boost::function<void(bool)> resumeFunction, boost::function<void(std::string)> errorFunction);
+    void isInGroup(int groupId, boost::function<void(bool)> resumeFunction, boost::function<void(std::string)> errorFunction);
+    void getRankInGroup(int groupId, boost::function<void(int)> resumeFunction, boost::function<void(std::string)> errorFunction);
+    void getRoleInGroup(int groupId, boost::function<void(std::string)> resumeFunction, boost::function<void(std::string)> errorFunction);
+    void getFriendsOnline(int maxFriends, boost::function<void(shared_ptr<const Reflection::ValueArray>)> resumeFunction,
+        boost::function<void(std::string)> errorFunction);
+
+    void loadChatInfo();
+
+    // should be called with at least data model read lock
+    bool isChatInfoValid() const;
+    ChatFilterType getChatFilterType() const;
+
+    // for testing only
+    void setChatInfo(ChatFilterType filterType);
+
+    void kick(std::string msg);
+    void handleTeleportInternalSignal(
+        TeleportService::TeleportState teleportState, shared_ptr<const Reflection::ValueTable> teleportInfo, shared_ptr<Instance> customLoadingGUI);
+    void handleTeleportSignalBackend(TeleportService::TeleportState teleportState);
+    void setForceEarlySpawnLocationCalculation();
+    bool calculatesSpawnLocationEarly() const;
+
+    const RakNet::SystemAddress& getRemoteAddress() const;
+    const SystemAddress getRemoteAddressAsRbxAddress() const;
+    void setRemoteAddress(const RakNet::SystemAddress& address);
+
+    void onTeleport(TeleportService::TeleportState teleportState, int placeId, std::string instanceIdOrSpawnName);
+    void onTeleportInternal(TeleportService::TeleportState teleportState, shared_ptr<const Reflection::ValueTable> teleportInfo,
+        shared_ptr<Instance> customLoadingGUI = shared_ptr<Instance>());
+    bool getTeleported() const
+    {
+        return teleported;
+    }
+    bool getTeleportedIn() const
+    {
+        return teleportedIn;
+    }
+    void setTeleportedIn(bool value);
+
+    int getFollowUserId() const
+    {
+        return followUserId;
+    }
+    void setFollowUserId(int followUserId)
+    {
+        this->followUserId = followUserId;
+    }
+
+    void loadStarterGear();
+    void onCharacterDied();
+
+    /*override*/ void destroy();
+
+private:
+    void setupHumanoid(shared_ptr<Humanoid> humanoid);
+    void characterChildAdded(shared_ptr<Instance> child);
+
+    void doPeriodicIdleCheck();
+    void checkContextReadyToSpawnCharacter();
+    static void calculateNextSpawnLocationHelper(weak_ptr<Player>& weakPlayer, const ServiceProvider* serviceProvider);
+    void calculateNextSpawnLocation(const ServiceProvider* serviceProvider);
+    SpawnData calculateSpawnLocation(const std::string& preferedSpawnName);
+
+    static void loadChatInfoInternal(weak_ptr<Player> weakPlayer);
+
+    Time lastActivityTime;
+
+    MembershipType membershipType;
+    int accountAge;
+
+    bool chatInfoHasBeenLoaded;
+    ChatFilterType chatFilterType;
+
+    shared_ptr<Aya::Mouse> mouse;
+    Aya::Camera::CameraMode cameraMode;
+
+    float nameDisplayDistance;
+    float healthDisplayDistance;
+    float cameraMaxZoomDistance;
+    float cameraMinZoomDistance;
+    bool enableMouseLockOption;
+    Aya::StarterPlayerService::DeveloperTouchCameraMovementMode touchCameraMovementMode;
+    Aya::StarterPlayerService::DeveloperComputerCameraMovementMode computerCameraMovementMode;
+    Aya::StarterPlayerService::DeveloperCameraOcclusionMode cameraOcclusionMode;
+    Aya::StarterPlayerService::DeveloperTouchMovementMode touchMovementMode;
+    Aya::StarterPlayerService::DeveloperComputerMovementMode computerMovementMode;
+
+    bool copiedGuiOnce;
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Players.hpp

@@ -0,0 +1,527 @@
+
+
+#pragma once
+
+#include "Tree/Instance.hpp"
+
+#include "Tree/Service.hpp"
+
+#include "Player.hpp"
+
+#include "ChatFilter.hpp"
+
+#include "Utility/SystemAddress.hpp"
+
+#include "Utility/GameMode.hpp"
+
+#include "DataModel/FriendService.hpp"
+
+#include "DataModel/ContentProvider.hpp"
+
+#include "boost/thread/thread.hpp"
+#include <boost/thread/condition.hpp>
+#include <boost/unordered_set.hpp>
+#include <queue>
+
+
+namespace RakNet
+{
+class RakPeerInterface;
+struct Packet;
+class BitStream;
+struct SystemAddress;
+} // namespace RakNet
+
+namespace Aya
+{
+class PartInstance;
+class ModelInstance;
+class Region2;
+class Adorn;
+class ScriptInformationProvider;
+class DataModel;
+
+namespace Network
+{
+
+struct MemHash
+{
+    size_t checkIdx;
+    unsigned int value;
+    unsigned int failMask;
+};
+
+typedef std::vector<MemHash> MemHashVector;
+typedef std::vector<MemHashVector> MemHashConfigs;
+
+class ConcurrentRakPeer;
+
+class ChatMessage
+{
+public:
+    enum ChatType
+    {
+        CHAT_TYPE_ALL,
+        CHAT_TYPE_TEAM,
+        CHAT_TYPE_WHISPER,
+        CHAT_TYPE_GAME,
+        // CHAT_TYPE_PARTY
+    };
+
+    std::string guid;
+    std::string message;
+    ChatType chatType;
+    shared_ptr<Player> const source;
+    shared_ptr<Player> const destination;
+    ChatMessage(const ChatMessage& other, const std::string& message);
+    ChatMessage(const char* message, ChatType chatType, shared_ptr<Player> source);
+    ChatMessage(const char* message, ChatType chatType, shared_ptr<Player> source, shared_ptr<Player> destination);
+
+    bool isVisibleToPlayer(shared_ptr<Player> player) const;
+    static bool isVisibleToPlayer(
+        shared_ptr<Player> const player, shared_ptr<Player> const source, shared_ptr<Player> const destination, const ChatType chatType);
+    std::string getReportAbuseMessage() const;
+};
+
+struct AbuseReport
+{
+    struct Message
+    {
+        int userID;
+        std::string text;
+        std::string guid;
+    };
+
+    int placeID;
+    std::string gameJobID;
+
+    int submitterID;
+    int allegedAbuserID;
+    std::string comment;
+    std::list<Message> messages;
+    void addMessage(shared_ptr<Player> reportingPlayer, const ChatMessage& cm);
+};
+
+class AbuseReporter
+{
+    struct data
+    {
+        std::queue<AbuseReport> queue;
+        boost::mutex requestSync; // synchronizes the request queue
+    };
+    shared_ptr<data> _data;
+    scoped_ptr<worker_thread> requestProcessor;
+
+public:
+    AbuseReporter(std::string abuseUrl);
+    void add(AbuseReport& r, shared_ptr<Player> reportingPlayer, const std::list<ChatMessage>& chatHistory);
+
+private:
+    static worker_thread::work_result processRequests(shared_ptr<data> _data, std::string abuseUrl);
+};
+
+extern const char* const sPlayers;
+
+class Players
+    : public DescribedNonCreatable<Players, Instance, sPlayers>
+    , public Service
+{
+private:
+    typedef DescribedNonCreatable<Players, Instance, sPlayers> Super;
+
+public:
+    // Unfortunately the RakNet enums are not accessible via the Players class, so this identical enum is created
+    // and mapped in PluginInterfaceAdapter::OnReceive.  - Tim
+    enum ReceiveResult
+    {
+        // The plugin used this message and it shouldn't be given to the user.
+        PLAYERS_STOP_PROCESSING_AND_DEALLOCATE = 0,
+
+        // The plugin is going to hold on to this message.  Do not deallocate it but do not pass it to other plugins either.
+        PLAYERS_STOP_PROCESSING,
+    };
+
+    enum ChatOption
+    {
+        CLASSIC_CHAT = 0,
+        BUBBLE_CHAT = 1,
+        CLASSIC_AND_BUBBLE_CHAT = 2
+    };
+
+    enum PlayerChatType
+    {
+        PLAYER_CHAT_TYPE_ALL = 0,
+        PLAYER_CHAT_TYPE_TEAM = 1,
+        PLAYER_CHAT_TYPE_WHISPER = 2
+    };
+
+
+    static bool isNetworkClient(Instance* instance);
+    void friendEventFired(int userId, int otherUserId, FriendService::FriendEventType friendEvent);
+    void friendStatusChanged(int userId, int otherUserId, FriendService::FriendStatus friendStatus);
+
+private:
+    std::string saveDataUrl;
+    std::string loadDataUrl;
+    std::string saveLeaderboardDataUrl;
+    boost::unordered_set<std::string> leaderboardKeys;
+
+    std::string chatFilterUrl;
+    std::string buildUserPermissionsUrl;
+    std::string sysStatsUrl;
+
+    static bool canKickBecauseRunningInRealGameServer;
+
+    bool characterAutoSpawn;
+
+    scoped_ptr<AbuseReporter> abuseReporter;
+    boost::intrusive_ptr<GuidItem<Instance>::Registry> guidRegistry;
+    std::list<ChatMessage> chatHistory;
+
+    const boost::intrusive_ptr<GuidItem<Instance>::Registry>& getGuidRegistry();
+
+    copy_on_write_ptr<Instances> players;
+    ConcurrentRakPeer* rakPeer;
+
+    int maxPlayers;
+    int preferredPlayers;
+    int testPlayerNameId;
+    int testPlayerUserId;
+    shared_ptr<Player> localPlayer;
+
+    ChatOption chatOption;
+
+    bool nonSuperSafeChatForAllPlayersEnabled;
+
+    Aya::signals::connection blockUserClientSignalConnection;
+    Aya::signals::connection blockUserFinishedFromServerConnection;
+    Aya::signals::connection loadLocalPlayerGuisConnection;
+
+    boost::unordered_map<std::pair<int, int>, std::pair<boost::function<void(std::string)>, boost::function<void(std::string)>>> clientBlockUserMap;
+
+    void raiseChatMessageSignal(const ChatMessage& message);
+    void raisePlayerChattedSignal(const ChatMessage& message);
+
+    void loadLocalPlayerGuis();
+    void gotBlockUserSuccess(std::string response, bool blockUser, int blockerUserId, int blockeeUserId,
+        boost::function<void(std::string)> resumeFunction, boost::function<void(std::string)> errorFunction);
+    void gotBlockUserError(std::string error, bool blockUser, int blockerUserId, int blockeeUserId, boost::function<void(std::string)> errorFunction);
+
+    static void onReceivedRawGetUserIdSuccess(weak_ptr<DataModel> weakDataModel, std::string response, boost::function<void(int)> resumeFunction,
+        boost::function<void(std::string)> errorFunction);
+    static void onReceivedRawGetUserIdError(weak_ptr<DataModel> weakDataModel, std::string error, boost::function<void(std::string)> errorFunction);
+
+    static void onReceivedRawGetUserNameSuccess(weak_ptr<DataModel> weakDataModel, std::string response,
+        boost::function<void(std::string)> resumeFunction, boost::function<void(std::string)> errorFunction);
+    static void onReceivedRawGetUserNameError(weak_ptr<DataModel> weakDataModel, std::string error, boost::function<void(std::string)> errorFunction);
+
+    void serverMakeBlockUserRequest(bool blockUser, int blockerUserId, int blockeeUserId, boost::function<void(std::string)> resumeFunction,
+        boost::function<void(std::string)> errorFunction);
+    void clientReceiveBlockUserFinished(int blockerUserId, int blockeeUserId, std::string errorString);
+
+    void internalBlockUser(int blockerUserId, int blockeeUserId, bool isBlocking, boost::function<void(std::string)> resumeFunction,
+        boost::function<void(std::string)> errorFunction);
+
+public:
+    ReceiveResult OnReceiveChat(Player* sourceValidation, RakNet::RakPeerInterface* peer, RakNet::Packet* packet, unsigned char chatType);
+    ReceiveResult OnReceiveReportAbuse(Player* source, RakNet::RakPeerInterface* peer, RakNet::Packet* packet);
+
+#ifdef ENABLE_VOICE_CHAT
+    ReceiveResult OnReceiveOpusData(Player* source, RakNet::RakPeerInterface* peer, RakNet::Packet* packet, unsigned char chatType);
+
+    void SendOpusData();
+#endif
+
+    Aya::signal<void(shared_ptr<Instance>, shared_ptr<Instance>, FriendService::FriendEventType)> friendRequestEvent;
+    Aya::signal<void(shared_ptr<Instance>)> playerAddedEarlySignal;
+    Aya::signal<void(shared_ptr<Instance>)> playerAddedSignal;
+    Aya::signal<void(shared_ptr<Instance>)> playerRemovingSignal;
+    Aya::signal<void(shared_ptr<Instance>)> playerRemovingLateSignal;
+    Aya::signal<void(const ChatMessage&)> chatMessageSignal;
+    Aya::signal<void(AbuseReport report)> abuseReportedReceived;
+    Aya::signal<void(
+        const RakNet::SystemAddress&, const shared_ptr<RakNet::BitStream>&, const shared_ptr<Instance>, const std::string&, const std::string&)>
+        sendFilteredChatMessageSignal;
+
+    Aya::signal<void(PlayerChatType, shared_ptr<Instance>, std::string, shared_ptr<Instance>)> playerChattedSignal;
+    Aya::signal<void(std::string)> gameAnnounceSignal;
+
+    Aya::remote_signal<void(int, int, bool)> blockUserRequestFromClientSignal;
+    Aya::remote_signal<void(int, int, std::string)> blockUserFinishedFromServerSignal;
+
+    static Reflection::RefPropDescriptor<Players, Instance> propLocalPlayer;
+    static Reflection::PropDescriptor<Players, bool> propCharacterAutoSpawn;
+    Players();
+    ~Players();
+
+    bool superSafeOn() const;
+
+    shared_ptr<Instance> createLocalPlayer(int userId, bool teleportedIn = false);
+    void resetLocalPlayer();
+
+    Player* getLocalPlayer()
+    {
+        return localPlayer.get();
+    }
+    const Player* getConstLocalPlayer() const
+    {
+        return localPlayer.get();
+    }
+    Instance* getLocalPlayerDangerous() const; // only for reflection
+
+    int getNumPlayers() const
+    {
+        return (int)players->size();
+    }
+
+    void getUserIdFromName(std::string userName, boost::function<void(int)> resumeFunction, boost::function<void(std::string)> errorFunction);
+    void getNameFromUserId(int userId, boost::function<void(std::string)> resumeFunction, boost::function<void(std::string)> errorFunction);
+    void getFriends(int userId, boost::function<void(shared_ptr<Instance>)> resumeFunction, boost::function<void(std::string)> errorFunction);
+
+    static void setAppearanceParent(shared_ptr<Instance> model, weak_ptr<Instance> instance);
+    static void doLoadAppearance(AsyncHttpQueue::RequestResult result, shared_ptr<Instances> instances, std::string contentDescription,
+        shared_ptr<ModelInstance> model, shared_ptr<int> amountToLoad, boost::function<void(shared_ptr<Instance>)> resumeFunction,
+        boost::function<void(std::string)> errorFunction);
+    static void doMakeAccoutrementRequests(std::string response, weak_ptr<DataModel> dataModel, shared_ptr<ModelInstance> model,
+        boost::function<void(shared_ptr<Instance>)> resumeFunction, boost::function<void(std::string)> errorFunction);
+    static void makeAccoutrementRequests(std::string* response, std::exception* err, weak_ptr<DataModel> dataModel, shared_ptr<ModelInstance> model,
+        boost::function<void(shared_ptr<Instance>)> resumeFunction, boost::function<void(std::string)> errorFunction);
+    void getCharacterAppearance(
+        int userId, boost::function<void(shared_ptr<Instance>)> resumeFunction, boost::function<void(std::string)> errorFunction);
+
+    int getMaxPlayers() const;
+    int getPreferredPlayers() const;
+
+    void setMaxPlayers(int value);
+    void setPreferredPlayers(int value);
+    void setSysStatsUrl(std::string url);
+
+    std::string getSaveDataUrl(int userId) const;
+    void setSaveDataUrl(std::string saveDataUrl);
+
+    std::string getLoadDataUrl(int userId) const;
+    void setLoadDataUrl(std::string loadDataUrl);
+
+    std::string getSaveLeaderboardDataUrl(int userId) const;
+    void setSaveLeaderboardDataUrl(std::string saveLeaderboardDataUrl);
+    void addLeaderboardKey(std::string);
+
+    bool hasLeaderboardKey(const std::string& key) const;
+    boost::unordered_set<std::string>::const_iterator beginLeaderboardKey() const;
+    boost::unordered_set<std::string>::const_iterator endLeaderboardKey() const;
+
+    void setChatOption(ChatOption value);
+    void setNonSuperSafeChatForAllPlayersEnabled(bool enabled);
+    bool getNonSuperSafeChatForAllPlayersEnabled() const;
+    bool getClassicChat() const
+    {
+        return chatOption == CLASSIC_CHAT || chatOption == CLASSIC_AND_BUBBLE_CHAT;
+    }
+    bool getBubbleChat() const
+    {
+        return chatOption == BUBBLE_CHAT || chatOption == CLASSIC_AND_BUBBLE_CHAT;
+    }
+
+    shared_ptr<const Instances> getPlayers()
+    {
+        return players.read();
+    }
+
+    // Chat-related functions
+    void gamechat(const std::string& message);
+
+    void chat(std::string message);
+    void teamChat(std::string);
+    void whisperChat(std::string message, shared_ptr<Instance> player);
+
+    void reportAbuse(Player* player, const std::string& comment);
+    void reportAbuseLua(shared_ptr<Instance> instance, std::string reason, std::string comment);
+    std::list<ChatMessage>::const_iterator chatHistory_begin()
+    {
+        return chatHistory.begin();
+    }
+    std::list<ChatMessage>::const_iterator chatHistory_end()
+    {
+        return chatHistory.end();
+    }
+    bool canReportAbuse() const;
+    void setAbuseReportUrl(std::string value);
+    void setChatFilterUrl(std::string value);
+    void setBuildUserPermissionsUrl(std::string value);
+    bool hasBuildUserPermissionsUrl() const;
+    std::string getBuildUserPermissionsUrl(int playerId) const;
+
+    void friendServiceRequest(bool makeFriends, weak_ptr<Player> sourcePlayer, int otherUserId);
+
+    bool getCharacterAutoSpawnProperty() const
+    {
+        return characterAutoSpawn;
+    }
+    void setCharacterAutoSpawnProperty(bool value);
+    bool getShouldAutoSpawnCharacter() const;
+
+    void blockUser(int blockerUserId, int blockeeUserId, boost::function<void(std::string)> resumeFunction = boost::function<void(bool)>(),
+        boost::function<void(std::string)> errorFunction = boost::function<void(std::string)>());
+    void unblockUser(int blockerUserId, int blockeeUserId, boost::function<void(std::string)> resumeFunction = boost::function<void(bool)>(),
+        boost::function<void(std::string)> errorFunction = boost::function<void(std::string)>());
+
+    void setConnection(ConcurrentRakPeer* rakPeer);
+
+    shared_ptr<Instance> playerFromCharacter(shared_ptr<Instance> character);
+    shared_ptr<Instance> getPlayerInstanceByID(int userID);
+    shared_ptr<Player> getPlayerByID(int userID);
+
+    static Player* getPlayerFromCharacter(Aya::Instance* character);
+
+    void buildClientRegion(Region2& clientRegion);
+
+    void renderDPhysicsRegions(Adorn* adorn);
+
+    ///////////////////////////////////////////////////////////////////////////
+    //
+    // STATICS
+    //
+    // If Client == Client Network Address;  If Server == NetworkOwner::Server()
+    static Aya::SystemAddress findLocalSimulatorAddress(const Aya::Instance* context);
+
+    static ModelInstance* findLocalCharacter(Aya::Instance* context);
+    static const ModelInstance* findConstLocalCharacter(const Aya::Instance* context);
+
+    static Player* findLocalPlayer(Aya::Instance* context);
+    static const Player* findConstLocalPlayer(const Aya::Instance* context);
+
+    static shared_ptr<Player> findAncestorPlayer(const Aya::Instance* context);
+
+    static shared_ptr<Player> findPlayerWithAddress(const Aya::SystemAddress& playerAddres, const Aya::Instance* context);
+
+    static bool clientIsPresent(const Aya::Instance* context, bool testInDatamodel = true);
+
+    static bool serverIsPresent(const Aya::Instance* context, bool testInDatamodel = true);
+
+    static bool frontendProcessing(const Aya::Instance* context, bool testInDatamodel = true);
+
+    static bool backendProcessing(const Aya::Instance* context, bool testInDatamodel = true);
+
+    static int getPlayerCount(const Aya::Instance* context);
+
+    // TODO: Remove this some day!
+    static bool getDistributedPhysicsEnabled();
+
+    ///////////////////////////////////////////////////////////////////////////
+    //
+    // These are here as "official" designation of Big-Picture states
+    //
+    //						Frontend	Backend
+    // GameServer:						x		serverIsPresent (assumes !findLocalPlayer)
+    // Visit Online:		x					clientIsPresent && findLocalPlayer
+    // Watch Online:		x					clientIsPresent && !findLocalPlayer	(i.e. - visit, no character)
+    // Visit Solo:			x			x		!clientIsPresent && !serverIsPresent && findLocalPlayer
+    // Local Play:			x					clientIsPresent && findLocalPlayer && userID == 0
+    // Edit Mode:			x			x		!clientIsPresent && !serverIsPresent && !findLocalPlayer
+    //
+    // typedef enum {GAME_SERVER, DPHYS_GAME_SERVER, CLIENT, DPHYS_CLIENT, WATCH_ONLINE, VISIT_SOLO, EDIT} GameMode;
+
+    static Aya::Network::GameMode getGameMode(const Aya::Instance* context)
+    {
+        bool client = clientIsPresent(context);
+        bool server = serverIsPresent(context);
+        bool localPlayer = (findConstLocalPlayer(context) != NULL);
+        bool dPhysics = getDistributedPhysicsEnabled();
+
+        AYAASSERT(!(server && (client || localPlayer)));
+
+        if (server)
+        {
+            return dPhysics ? DPHYS_GAME_SERVER : GAME_SERVER;
+        }
+        if (client && localPlayer)
+        {
+            return dPhysics ? DPHYS_CLIENT : CLIENT;
+        }
+        if (client && !localPlayer)
+        {
+            return WATCH_ONLINE;
+        }
+        if (!client && localPlayer)
+        {
+            return VISIT_SOLO;
+        }
+        else
+        {
+            return EDIT;
+        }
+    }
+
+    static Aya::Network::GameMode getGameMode(const Aya::Instance* context, const int placeID)
+    {
+        bool client = clientIsPresent(context);
+        bool server = serverIsPresent(context);
+        bool localPlayer = (findConstLocalPlayer(context) != NULL);
+        bool dPhysics = getDistributedPhysicsEnabled();
+
+        AYAASSERT(!(server && (client || localPlayer)));
+        ;
+
+        if (placeID <= 0)
+        {
+            return LOCAL_PLAY;
+        }
+        if (server)
+        {
+            return dPhysics ? DPHYS_GAME_SERVER : GAME_SERVER;
+        }
+        if (client && localPlayer)
+        {
+            return dPhysics ? DPHYS_CLIENT : CLIENT;
+        }
+        if (client && !localPlayer)
+        {
+            return WATCH_ONLINE;
+        }
+        if (!client && localPlayer)
+        {
+            return VISIT_SOLO;
+        }
+        else
+        {
+            return EDIT;
+        }
+    }
+
+    void onRemoteSysStats(int userId, const std::string& stat, const std::string& message, bool desireKick = true);
+    bool hashMatches(const std::string& hash);
+
+    void disconnectPlayer(int userId, int reason);
+    void disconnectPlayerLocal(int userId, int reason);
+
+    bool getUseCoreScriptHealthBar();
+
+protected:
+    void disconnectPlayer(Instance& instance, int userId, int reason);
+
+    std::map<int, std::set<std::string>> cheatingPlayers;
+    bool askAddChild(const Instance* instance) const;
+    /*override*/ void onChildAdded(Instance* child);
+    /*override*/ void onChildRemoving(Instance* child);
+    /*override*/ void onDescendantRemoving(const shared_ptr<Instance>& instance);
+    /*override*/ void processRemoteEvent(
+        const Reflection::EventDescriptor& descriptor, const Reflection::EventArguments& args, const Aya::SystemAddress& source);
+
+private:
+    void reportScriptSecurityError(int userId, std::string hash, std::string error, std::string stack);
+
+    void killPlayer(int userId);
+
+    void addChatMessage(const ChatMessage& message);
+
+    void checkChat(const std::string& message);
+
+    void sendFilteredChatMessageSignalHelper(const RakNet::SystemAddress& systemAddress, const shared_ptr<RakNet::BitStream>& baseData,
+        const shared_ptr<Instance> sourceInstance, shared_ptr<ChatMessage> chatEvent, const ChatFilter::Result& response);
+
+    // Instance
+    /*override*/ void onServiceProvider(ServiceProvider* oldProvider, ServiceProvider* newProvider);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/PropertySynchronization.hpp

@@ -0,0 +1,349 @@
+#pragma once
+
+/*
+This unit solves the "ships passing in the night" replication problem.
+If two peers change a property simultaneously, then there is a risk that
+they will swap values and never settle on a common shared value. For
+example, if peer A sets a color to Red and peer B sets a color to Blue,
+then after replication peer A will get Blue and peer B will get Red.
+
+The solution involves appointing one peer as the master and the other peer
+as the slave. When the master sets a property, the slave must acknowledge
+the change before it can successfully change the same property.
+
+The method involves associating a version number with each property.
+When the master changes the property, it increments the version number.
+The slave must send an acknowledgment to the master that it has received
+the property update (and its current version number). Until then, the
+master will reject any property changes coming from the slave. Since the
+slave will eventually receive the master<65>s property value, we are
+guaranteed eventual consistency.
+
+Note that a side-effect of this approach is that a slave might change a
+value temporarily. The master will never experience this temporary state.
+
+The implementation is designed to avoid massive memory costs and it
+minimizes network bandwidth. Instead of storing a version number for every
+single property, we rely on the fact that network latency is finite.
+When the master changes a property, we associate a version number with the
+property for a "short" period of time - on the order of a network
+round-trip. Likewise, the slave keeps the association temporarily. If the
+slave doesn<73>t try to change a property during the lifetime of the version
+number, then it never needs to send an acknowledgement, so we save
+bandwidth. The end result is a modest memory footprint and nearly
+no networking overhead.
+*/
+
+namespace Aya
+{
+namespace Network
+{
+namespace PropSync
+{
+
+namespace detail
+{
+class PropertyKey
+{
+    const Reflection::PropertyDescriptor* descriptor;
+    Guid::Data guidData;
+
+public:
+    PropertyKey()
+        : descriptor(NULL)
+    {
+        guidData.scope.setNull();
+        guidData.index = 0;
+    }
+
+    PropertyKey(const Reflection::ConstProperty& prop)
+        : descriptor(&prop.getDescriptor())
+    {
+        guidData.scope.setNull();
+        guidData.index = 0;
+
+        if (prop.getInstance())
+        {
+            if (const Instance* instance = prop.getInstance()->fastDynamicCast<const Instance>())
+                instance->getGuid().extract(guidData);
+        }
+    }
+
+    bool operator==(const PropertyKey& other) const
+    {
+        return this->descriptor == other.descriptor && this->guidData.scope == other.guidData.scope && this->guidData.index == other.guidData.index;
+    }
+
+    friend std::size_t hash_value(const PropertyKey& key)
+    {
+        std::size_t seed = boost::hash<const Reflection::PropertyDescriptor*>()(key.descriptor);
+        boost::hash_combine(seed, key.guidData.scope.getName());
+        boost::hash_combine(seed, key.guidData.index);
+        return seed;
+    }
+};
+
+// This class contains common code between the master and the slave
+template<class Item>
+class Base
+{
+protected:
+    Aya::Time::Interval expirationTime;
+    typedef boost::unordered_map<PropertyKey, Item> Map;
+    Map map;
+    Aya::timestamped_safe_queue<PropertyKey> expirationQueue;
+
+    Base(Aya::Time::Interval expirationTime)
+        : expirationTime(expirationTime)
+    {
+    }
+
+public:
+    size_t itemCount() const
+    {
+        return map.size();
+    }
+    void expireItems()
+    {
+        PropertyKey prop;
+        while (expirationQueue.pop_if_waited(expirationTime, prop))
+        {
+            typename Map::iterator iter = map.find(prop);
+            AYAASSERT(iter != map.end());
+            if (Aya::Time::now<Aya::Time::Fast>() > iter->second.expiration)
+                // expire the item
+                map.erase(iter);
+            else
+                // Put it back in the queue for another time
+                expirationQueue.push(prop);
+        }
+    }
+};
+
+class Item
+{
+public:
+    int version;
+    Aya::Time expiration;
+    Item()
+        : version(0)
+    {
+    }
+};
+class MasterItem : public Item
+{
+public:
+    bool isVersionSent; // == the version number has been sent to the slave
+    int slaveVersion;   // the version that the slave has acknowledged
+    MasterItem()
+        : isVersionSent(false)
+        , slaveVersion(-1)
+    {
+    }
+};
+class SlaveItem : public Item
+{
+public:
+    bool isAckSent; // == an acknowledgement has been sent from slave to master
+};
+} // namespace detail
+
+class Master : protected detail::Base<detail::MasterItem>
+{
+    typedef detail::Base<detail::MasterItem> Super;
+
+public:
+    unsigned int propertyRejectionCount;
+    Master()
+        : Super(Aya::Time::Interval(2))
+        , propertyRejectionCount(0)
+    {
+    }
+
+    // Call this periodically to cull the database
+    void expireItems()
+    {
+        Super::expireItems();
+    }
+
+    // Call this function when the master changes a property:
+    void onPropertyChanged(Reflection::ConstProperty prop)
+    {
+        // get-or-create the Item
+        // TODO: use emplace when it becomes available
+        std::pair<Map::iterator, bool> pair = map.insert(Map::value_type(detail::PropertyKey(prop), detail::MasterItem()));
+        detail::MasterItem& item(pair.first->second);
+        bool createdItem = pair.second;
+
+        if (createdItem)
+        {
+            AYAASSERT(item.version == 0);
+            AYAASSERT(!item.isVersionSent);
+            // Set the expiration, queue for expiration
+            item.expiration = Aya::Time::now<Aya::Time::Fast>() + expirationTime;
+            expirationQueue.push(detail::PropertyKey(prop));
+        }
+        else if (item.isVersionSent)
+        {
+            item.version++;
+            item.isVersionSent = false;
+            // touch the expiration time
+            item.expiration = Aya::Time::now<Aya::Time::Fast>() + expirationTime;
+        }
+    }
+
+    // Call this function when the master is about to send a property.
+    // The returned value specifies if a version reset message should
+    // be sent to the slave
+    typedef enum
+    {
+        SendVersionReset,
+        DontSendVersionReset
+    } PropertySendResult;
+    PropertySendResult onPropertySend(Reflection::ConstProperty prop)
+    {
+        // get-or-create the Item
+        // TODO: use emplace when it becomes available
+        std::pair<Map::iterator, bool> pair = map.insert(Map::value_type(detail::PropertyKey(prop), detail::MasterItem()));
+        detail::MasterItem& item(pair.first->second);
+        bool createdItem = pair.second;
+
+        // we assume that the caller will send the Item now
+        item.isVersionSent = true;
+
+        // Find out if this is a brand new Item
+        if (createdItem)
+        {
+            // Set the expiration, queue for expiration and request a version reset
+            item.expiration = Aya::Time::now<Aya::Time::Fast>() + expirationTime;
+            expirationQueue.push(detail::PropertyKey(prop));
+            AYAASSERT(item.version == 0);
+            return SendVersionReset;
+        }
+
+        return item.version == 0 ? SendVersionReset : DontSendVersionReset;
+    }
+
+    // Call this function when an acknowledgement is received from the slave
+    void onReceivedAcknowledgement(Reflection::ConstProperty prop, int version)
+    {
+        Map::iterator iter = map.find(detail::PropertyKey(prop));
+        if (iter == map.end())
+            return;
+
+        iter->second.slaveVersion = version;
+    }
+
+    // Call this function when a property is received from the slave
+    FilterResult onReceivedPropertyChanged(Reflection::ConstProperty prop)
+    {
+        Map::iterator iter = map.find(detail::PropertyKey(prop));
+        if (iter == map.end())
+            return Accept;
+
+        detail::MasterItem& item(iter->second);
+        if (item.slaveVersion == item.version)
+            return Accept;
+
+        propertyRejectionCount++;
+        return Reject;
+    }
+
+    size_t itemCount() const
+    {
+        return Super::itemCount();
+    }
+
+    void setExpiration(Aya::Time::Interval expirationTime)
+    {
+        this->expirationTime = expirationTime;
+    }
+};
+
+class Slave : protected detail::Base<detail::SlaveItem>
+{
+    typedef detail::Base<detail::SlaveItem> Super;
+
+public:
+    unsigned int ackCount;
+    Slave()
+        : Super(Aya::Time::Interval(4))
+        , ackCount(0)
+    {
+    }
+
+    // Call this periodically to cull the database
+    void expireItems()
+    {
+        Super::expireItems();
+    }
+
+    // Call this function when the slave receives a property from the master
+    void onReceivedPropertyChanged(Reflection::ConstProperty prop, bool versionReset)
+    {
+        // get-or-create the Item
+        // TODO: use emplace when it becomes available
+        std::pair<Map::iterator, bool> pair = map.insert(Map::value_type(detail::PropertyKey(prop), detail::SlaveItem()));
+        detail::SlaveItem& item(pair.first->second);
+        bool createdItem = pair.second;
+
+        if (createdItem)
+        {
+            // Disabling this for now.
+            // AYAASSERT(versionReset);
+
+            // Set the expiration, queue for expiration and request a version reset
+            item.expiration = Aya::Time::now<Aya::Time::Fast>() + expirationTime;
+            expirationQueue.push(detail::PropertyKey(prop));
+        }
+        else
+        {
+            // touch the expiration time
+            item.expiration = Aya::Time::now<Aya::Time::Fast>() + expirationTime;
+        }
+
+        if (versionReset)
+            item.version = 0;
+        else
+            item.version++;
+
+        item.isAckSent = false;
+    }
+
+    // Call this function when the slave is about to send a property.
+    // If it returns SendAcknowledgement, then first send a message with the
+    // specified version number
+    typedef enum
+    {
+        SendAcknowledgement,
+        DontSendAcknowledgement
+    } PropertySendResult;
+    PropertySendResult onPropertySend(Reflection::ConstProperty prop, int& version)
+    {
+        Map::iterator iter = map.find(detail::PropertyKey(prop));
+        if (iter == map.end())
+            return DontSendAcknowledgement;
+
+        if (iter->second.isAckSent)
+            return DontSendAcknowledgement;
+
+        iter->second.isAckSent = true;
+        version = iter->second.version;
+        ackCount++;
+        return SendAcknowledgement;
+    }
+
+    size_t itemCount() const
+    {
+        return Super::itemCount();
+    }
+
+    void setExpiration(Aya::Time::Interval expirationTime)
+    {
+        this->expirationTime = expirationTime;
+    }
+};
+
+} // namespace PropSync
+} // namespace Network
+} // namespace Aya

engine/network/src/RakNetFast.hpp

@@ -0,0 +1,436 @@
+
+
+#pragma once
+
+// #include "Utility/Velocity.hpp"
+
+#include "RakNet/BitStream.hpp"
+
+// #include "Dictionary.hpp"
+#include "Util.hpp"
+
+namespace Aya
+{
+namespace Network
+{
+// --------------------------------------------------------------------
+// reverseEndian
+//
+// Reuse the numeric constants as much as possible to reduce constant value loads.
+
+template<unsigned int Bytes>
+struct ReverseEndianBytes
+{
+    template<class T>
+    static inline void reverseEndian(T& result)
+    {
+        BOOST_STATIC_ASSERT(sizeof(T) == 1);
+    }
+};
+
+template<>
+struct ReverseEndianBytes<2>
+{
+    template<class T>
+    static inline void reverseEndian(T& x)
+    {
+        uint16_t& result = (uint16_t&)x;
+        result = (result << 8) | (result >> 8);
+    }
+};
+
+template<>
+struct ReverseEndianBytes<4>
+{
+    template<class T>
+    static inline void reverseEndian(T& x)
+    {
+        uint32_t& result = (uint32_t&)x;
+        uint32_t val = ((result & 0x00FF00FF) << 8) | ((result >> 8) & 0x00FF00FF);
+        result = (val << 16) | (val >> 16);
+    }
+};
+
+template<>
+struct ReverseEndianBytes<8>
+{
+    template<class T>
+    static inline void reverseEndian(T& x)
+    {
+        uint64_t& result = (uint64_t&)x;
+        uint64_t val = (result << 32) | (result >> 32);
+        val = (val & 0x0000FFFF0000FFFF) << 16 | ((val >> 16) & 0x0000FFFF0000FFFF);
+        val = (val & 0x00FF00FF00FF00FF) << 8 | ((val >> 8) & 0x00FF00FF00FF00FF);
+        result = val;
+    }
+};
+
+template<class T>
+inline void reverseEndianBits(T& result, unsigned int bits)
+{
+    unsigned int msbPad = (-(signed int)bits) & 7; // == (8-(bits&7))&7.  High bits that were truncated off the MSB.
+
+    // Shift up by the amount clipped off the non-zero-MSB.
+    T shifted = result << msbPad;
+
+    // Right align the non-zero-MSB while it is still in the LSB and easy to locate.
+    result = (shifted & ~(T)0xff) | (result & ((T)0xff >> msbPad));
+
+    if ((sizeof(T) * 8 - 7) <= bits)
+    {
+        ReverseEndianBytes<sizeof(T)>::reverseEndian(result);
+    }
+    else
+    {
+        // Warning:  Taking the address of a parameter forces a register spill.
+        unsigned char* a = (unsigned char*)&result;
+        unsigned char* b = a + ((bits - 1) >> 3);
+        while (a < b)
+        {
+            unsigned char t = *a;
+            *a++ = *b;
+            *b-- = t;
+        }
+    }
+}
+
+// --------------------------------------------------------------------
+
+template<unsigned int Bytes>
+struct ReadFastBytes
+{
+    template<class T>
+    static inline void readFast(RakNet::BitStream& bitStream, T& result, unsigned int bits)
+    {
+        unsigned int readOffset = bitStream.GetReadOffset();
+
+        if (readOffset + bits > bitStream.GetNumberOfBitsUsed())
+            throw Aya::network_stream_exception("readFast past end");
+
+        const unsigned char* data = bitStream.GetData();
+        const unsigned char* pos = data + (readOffset >> 3);
+
+        unsigned int bitsOffset = readOffset & 7;
+        unsigned int firstByteBits = 8 - bitsOffset;
+
+        unsigned char firstByte = *pos & (0xff >> bitsOffset);
+
+        if (firstByteBits >= bits)
+        {
+            result = (T)(firstByte >> (firstByteBits - bits));
+            bitStream.SetReadOffset(readOffset + bits);
+            return;
+        }
+
+        T tmp = (T)firstByte;
+        ++pos;
+
+        unsigned int bitsRemaining = bits - firstByteBits; // 8 could be propagated out.
+        for (; bitsRemaining > 8; bitsRemaining -= 8)
+        {
+            tmp = (tmp << 8) | *pos;
+            ++pos;
+        }
+
+        tmp = (tmp << bitsRemaining) | (*pos >> (8 - bitsRemaining));
+        result = tmp;
+
+        bitStream.SetReadOffset(readOffset + bits);
+    }
+
+
+    template<unsigned int Bits, class T>
+    static inline void readFast(RakNet::BitStream& bitStream, T& result)
+    {
+        readFast(bitStream, result, Bits);
+    }
+};
+
+template<>
+struct ReadFastBytes<1>
+{
+    template<unsigned int Bits, class T>
+    static inline void readFast(RakNet::BitStream& bitStream, T& result)
+    {
+        BOOST_STATIC_ASSERT(Bits >= 1 && Bits <= 8);
+
+        unsigned int readOffset = bitStream.GetReadOffset();
+
+        if (readOffset + (Bits + 8) > bitStream.GetNumberOfBitsUsed())
+        {
+            // Avoid crashing due to reading an extra byte off the end of the buffer.
+            ReadFastBytes<0>::readFast(bitStream, result, Bits);
+        }
+        else
+        {
+            const unsigned char* data = bitStream.GetData();
+
+            unsigned int readOffsetBytes = readOffset >> 3;
+
+            unsigned char byte0 = data[readOffsetBytes + 0];
+            unsigned char byte1 = data[readOffsetBytes + 1];
+
+            result = (((((byte0 << 8) | byte1) << (readOffset & 7)) & 0xffff) >> (16 - Bits));
+
+            bitStream.SetReadOffset(readOffset + Bits);
+        }
+    }
+};
+
+template<>
+struct ReadFastBytes<2>
+{
+    template<unsigned int Bits, class T>
+    static inline void readFast(RakNet::BitStream& bitStream, T& result)
+    {
+        BOOST_STATIC_ASSERT(Bits >= 9 && Bits <= 16);
+
+        unsigned int readOffset = bitStream.GetReadOffset();
+
+        if (readOffset + (Bits + 8) > bitStream.GetNumberOfBitsUsed())
+        {
+            // Avoid crashing due to reading an extra byte off the end of the buffer.
+            ReadFastBytes<0>::readFast(bitStream, result, Bits);
+        }
+        else
+        {
+            const unsigned char* data = bitStream.GetData();
+
+            unsigned int readOffsetBytes = readOffset >> 3;
+
+            unsigned char byte0 = data[readOffsetBytes + 0];
+            unsigned char byte1 = data[readOffsetBytes + 1];
+            unsigned char byte2 = data[readOffsetBytes + 2];
+
+            result = ((((byte0 << 16) | (byte1 << 8) | byte2) << (readOffset & 7)) & 0xffffff) >> (24 - Bits);
+
+            bitStream.SetReadOffset(readOffset + Bits);
+        }
+    }
+};
+
+template<>
+struct ReadFastBytes<4>
+{
+    template<unsigned int Bits, class T>
+    static inline void readFast(RakNet::BitStream& bitStream, T& result)
+    {
+        BOOST_STATIC_ASSERT(Bits >= 17 && Bits <= 32);
+
+        unsigned int readOffset = bitStream.GetReadOffset();
+
+        if (readOffset + (Bits + 8) > bitStream.GetNumberOfBitsUsed())
+        {
+            // Avoid crashing due to reading an extra byte off the end of the buffer.
+            ReadFastBytes<0>::readFast(bitStream, result, Bits);
+        }
+        else
+        {
+            const unsigned char* data = bitStream.GetData();
+
+            unsigned int readOffsetBytes = readOffset >> 3;
+
+            unsigned char byte0 = data[readOffsetBytes + 0];
+            unsigned char byte1 = data[readOffsetBytes + 1];
+            unsigned char byte2 = data[readOffsetBytes + 2];
+            unsigned char byte3 = data[readOffsetBytes + 3];
+            unsigned char byte4 = data[readOffsetBytes + 4];
+
+            T tmp0 = (((((byte0 << 16) | (byte1 << 8) | byte2) << (readOffset & 7)) & 0xffffff) >> (24 - 16));
+            T tmp1 = (((((byte2 << 16) | (byte3 << 8) | byte4) << (readOffset & 7)) & 0xffffff) >> (24 - (Bits - 16)));
+            result = (tmp0 << 16) | tmp1;
+
+            bitStream.SetReadOffset(readOffset + Bits);
+        }
+    }
+};
+
+// --------------------------------------------------------------------
+//
+// The non-"T" versions replace ReadBits versions which did not do
+// endian swapping
+
+template<class T>
+inline void readFastN(RakNet::BitStream& bitStream, T& result, unsigned int bits)
+{
+    // Use default implementation for arbitrary bit-counts.
+    ReadFastBytes<0>::readFast(bitStream, result, bits);
+
+    reverseEndianBits(result, bits);
+}
+
+// The base template should only be needed for unusual numbers of bits.
+template<unsigned int Bits, class T>
+inline void readFastN(RakNet::BitStream& bitStream, T& result)
+{
+    ReadFastBytes<0>::readFast<Bits>(bitStream, result);
+
+    reverseEndianBits(result, Bits);
+}
+
+// --------------------------------------------------------------------
+// readFastT
+//
+// The "T" versions do not do endian swapping because the BitStream::Write
+// sends across network order/big-endian data.
+
+template<class T>
+inline void readFastT(RakNet::BitStream& bitStream, T& result)
+{
+    BOOST_STATIC_ASSERT(sizeof(T) == 0); // This is undefined.
+}
+
+#if defined(__linux)
+template<>
+inline void readFastT<unsigned long long>(RakNet::BitStream& bitStream, unsigned long long& result)
+{
+    bitStream.Read(result);
+}
+#endif
+
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, bool& result)
+{
+    unsigned int readOffset = bitStream.GetReadOffset();
+
+    if (readOffset + 1 > bitStream.GetNumberOfBitsUsed())
+        throw Aya::network_stream_exception("readFastBool past end");
+
+    const unsigned char* data = bitStream.GetData();
+
+    bool tmp = (data[readOffset >> 3] & (0x80 >> (readOffset & 7))) != 0;
+    result = tmp;
+
+    bitStream.SetReadOffset(readOffset + 1);
+}
+
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, float& result)
+{
+    union
+    {
+        float f;
+        uint32_t i;
+    } t;
+    ReadFastBytes<4>::readFast<32, uint32_t>(bitStream, t.i);
+    result = t.f;
+}
+
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, double& result)
+{
+    union
+    {
+        double d;
+        uint64_t i;
+    } t;
+    ReadFastBytes<0>::readFast<64, uint64_t>(bitStream, t.i);
+    result = t.d;
+}
+
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, int8_t& result)
+{
+    ReadFastBytes<1>::readFast<8, uint8_t>(bitStream, (uint8_t&)result);
+}
+
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, uint8_t& result)
+{
+    ReadFastBytes<1>::readFast<8, uint8_t>(bitStream, result);
+}
+
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, char& result)
+{
+    ReadFastBytes<1>::readFast<8, uint8_t>(bitStream, (uint8_t&)result);
+}
+
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, int16_t& result)
+{
+    ReadFastBytes<2>::readFast<16, uint16_t>(bitStream, (uint16_t&)result);
+}
+
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, uint16_t& result)
+{
+    ReadFastBytes<2>::readFast<16, uint16_t>(bitStream, result);
+}
+
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, int32_t& result)
+{
+    ReadFastBytes<4>::readFast<32, uint32_t>(bitStream, (uint32_t&)result);
+}
+
+#if !defined(__linux) && !defined(__APPLE__)
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, long& result)
+{
+#if defined(__LP64__)
+    BOOST_STATIC_ASSERT(sizeof(long) == sizeof(uint64_t));
+    ReadFastBytes<0>::readFast<64, uint64_t>(bitStream, (uint64_t&)result);
+#else
+    BOOST_STATIC_ASSERT(sizeof(long) == sizeof(uint32_t));
+    ReadFastBytes<4>::readFast<32, uint32_t>(bitStream, (uint32_t&)result);
+#endif
+}
+
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, unsigned long& result)
+{
+#if defined(__LP64__)
+    BOOST_STATIC_ASSERT(sizeof(unsigned long) == sizeof(uint64_t));
+    ReadFastBytes<0>::readFast<64, uint64_t>(bitStream, (uint64_t&)result);
+#else
+    BOOST_STATIC_ASSERT(sizeof(unsigned long) == sizeof(uint32_t));
+    ReadFastBytes<4>::readFast<32, uint32_t>(bitStream, (uint32_t&)result);
+#endif
+}
+#endif
+
+
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, uint32_t& result)
+{
+    ReadFastBytes<4>::readFast<32, uint32_t>(bitStream, result);
+}
+
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, int64_t& result)
+{
+    ReadFastBytes<0>::readFast<64, uint64_t>(bitStream, (uint64_t&)result);
+}
+
+template<>
+inline void readFastT(RakNet::BitStream& bitStream, uint64_t& result)
+{
+    ReadFastBytes<0>::readFast<64, uint64_t>(bitStream, result);
+}
+
+// --------------------------------------------------------------------
+
+inline void readVectorFast(RakNet::BitStream& bitStream, float& x, float& y, float& z)
+{
+    float magnitude;
+    readFastT(bitStream, magnitude);
+    if (magnitude > 0.00001f)
+    {
+        unsigned short c;
+        readFastT(bitStream, c);
+        x = (float(c) * (1.0f / 32767.5f) - 1.0f) * magnitude;
+
+        readFastT(bitStream, c);
+        y = (float(c) * (1.0f / 32767.5f) - 1.0f) * magnitude;
+
+        readFastT(bitStream, c);
+        z = (float(c) * (1.0f / 32767.5f) - 1.0f) * magnitude;
+    }
+    else
+    {
+        x = y = z = 0;
+    }
+}
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.ChangePropertyItem.cpp

@@ -0,0 +1,92 @@
+#include "Replicator.ChangePropertyItem.hpp"
+
+#include "Item.hpp"
+#include "Replicator.hpp"
+
+#include "Reflection/Property.hpp"
+
+#include "RakNet/BitStream.hpp"
+
+namespace Aya
+{
+namespace Network
+{
+
+DeserializedChangePropertyItem::DeserializedChangePropertyItem()
+{
+    type = Item::ItemTypeChangeProperty;
+}
+
+void DeserializedChangePropertyItem::process(Replicator& replicator)
+{
+    replicator.readChangedPropertyItem(this);
+}
+
+Replicator::ChangePropertyItem::ChangePropertyItem(
+    Replicator* replicator, const shared_ptr<const Instance>& instance, const Reflection::PropertyDescriptor& desc)
+    : PooledItem(*replicator)
+    , desc(desc)
+    , instance(instance)
+{
+}
+
+bool Replicator::ChangePropertyItem::write(RakNet::BitStream& bitStream)
+{
+    replicator.pendingChangedPropertyItems.erase(Aya::Reflection::ConstProperty(desc, instance.get()));
+
+    // Check to see if we are still replicating this object
+    if (!replicator.isReplicationContainer(instance.get()))
+        return true;
+
+    // Write value
+    replicator.writeChangedProperty(instance.get(), desc, bitStream);
+    return true;
+}
+
+shared_ptr<DeserializedItem> Replicator::ChangePropertyItem::read(Replicator& replicator, RakNet::BitStream& inBitstream)
+{
+    shared_ptr<DeserializedChangePropertyItem> deserializedData(new DeserializedChangePropertyItem());
+
+    int start = inBitstream.GetReadOffset();
+    shared_ptr<Instance> deserializedInstance;
+    replicator.deserializeInstanceRef(inBitstream, deserializedInstance, deserializedData->id);
+    deserializedData->instance = deserializedInstance;
+
+    // Read the property name
+    unsigned int propId = replicator.propDictionary.receive(inBitstream, deserializedData->propertyDescriptor, true);
+
+    if (replicator.ProcessOutdatedChangedProperty(
+            inBitstream, deserializedData->id, deserializedInstance.get(), deserializedData->propertyDescriptor, propId))
+    {
+        return shared_ptr<DeserializedChangePropertyItem>();
+    }
+
+    if (!deserializedData->propertyDescriptor)
+        throw Aya::runtime_error("Replicator readChangedProperty NULL descriptor");
+
+    if (deserializedInstance && !deserializedInstance->getDescriptor().isA(deserializedData->propertyDescriptor->owner))
+    {
+        throw Aya::runtime_error("Replication: Bad re-binding prop %s-%s << %s", deserializedInstance->getClassName().c_str(),
+            deserializedData->propertyDescriptor->name.c_str(), RakNetAddressToString(replicator.remotePlayerId).c_str());
+    }
+
+    if (!replicator.isServerReplicator())
+    {
+        inBitstream >> deserializedData->versionReset;
+    }
+
+    Reflection::Property prop = Reflection::Property(*deserializedData->propertyDescriptor, deserializedInstance.get());
+    replicator.deserializePropertyValue(inBitstream, prop, true /*useDictionary*/, false, &deserializedData->value);
+
+    if (replicator.settings().trackDataTypes)
+    {
+        replicator.replicatorStats.incrementPacketsReceived(deserializedData->propertyDescriptor->category.str);
+        replicator.replicatorStats.samplePacketsReceived(
+            deserializedData->propertyDescriptor->category.str, (inBitstream.GetReadOffset() - start) / 8);
+    }
+
+    return deserializedData;
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.ChangePropertyItem.hpp

@@ -0,0 +1,48 @@
+#pragma once
+
+#include "Item.hpp"
+#include "Replicator.hpp"
+
+namespace Aya
+{
+
+class Instance;
+namespace Reflection
+{
+class PropertyDescriptor;
+}
+
+namespace Network
+{
+
+
+class DeserializedChangePropertyItem : public DeserializedItem
+{
+public:
+    Aya::Guid::Data id;
+    shared_ptr<Instance> instance;
+    const Reflection::PropertyDescriptor* propertyDescriptor;
+    Reflection::Variant value;
+
+    bool versionReset; // client only
+
+    DeserializedChangePropertyItem();
+    ~DeserializedChangePropertyItem() {}
+
+    /*implement*/ void process(Replicator& replicator);
+};
+
+class Replicator::ChangePropertyItem : public PooledItem
+{
+    const Reflection::PropertyDescriptor& desc;
+    const shared_ptr<const Instance> instance;
+
+public:
+    ChangePropertyItem(Replicator* replicator, const shared_ptr<const Instance>& instance, const Reflection::PropertyDescriptor& desc);
+
+    /*implement*/ virtual bool write(RakNet::BitStream& bitStream);
+    static shared_ptr<DeserializedItem> read(Replicator& replicator, RakNet::BitStream& bitStream);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.DeleteInstanceItem.cpp

@@ -0,0 +1,104 @@
+#include "Replicator.DeleteInstanceItem.hpp"
+
+#include "Item.hpp"
+#include "NetworkPacketCache.hpp"
+#include "Replicator.hpp"
+#include "ReplicatorStats.hpp"
+#include "Streaming.hpp"
+#include "Util.hpp"
+
+#include "Tree/Instance.hpp"
+
+
+#include "RakNet/BitStream.hpp"
+
+#include <string>
+
+namespace Aya
+{
+namespace Network
+{
+
+DeserializedDeleteInstanceItem::DeserializedDeleteInstanceItem()
+{
+    type = Item::ItemTypeDelete;
+}
+
+void DeserializedDeleteInstanceItem::process(Replicator& replicator)
+{
+    replicator.readInstanceDeleteItem(this);
+}
+
+Replicator::DeleteInstanceItem::DeleteInstanceItem(Replicator* replicator, const shared_ptr<const Instance>& instance)
+    : PooledItem(*replicator)
+    , id(replicator->extractId(instance.get()))
+{
+    if (replicator->settings().printInstances)
+    {
+        details.reset(new InstanceDetails());
+        details->className = instance->getClassName().c_str();
+        details->guid = instance->getGuid().readableString();
+    }
+
+    // remove entry from instance cache
+    if (replicator->instancePacketCache)
+        replicator->instancePacketCache->remove(instance.get());
+}
+
+bool Replicator::DeleteInstanceItem::write(RakNet::BitStream& bitStream)
+{
+    if (!id.valid)
+        Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE,
+            "Replication: ~NULL >> %s", // remove player always on right
+            RakNetAddressToString(replicator.remotePlayerId).c_str());
+    else
+    {
+        try
+        {
+            int byteStart = bitStream.GetNumberOfBytesUsed();
+            writeItemType(bitStream, ItemTypeDelete);
+
+            // Write the GUID
+            replicator.sendId(bitStream, id);
+
+            if (details)
+                Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE, "Replication: ~%s:%s >> %s, bytes: %d", details->className.c_str(),
+                    details->guid.c_str(), RakNetAddressToString(replicator.remotePlayerId).c_str(), bitStream.GetNumberOfBytesUsed() - byteStart);
+
+            if (replicator.settings().trackDataTypes)
+            {
+                replicator.replicatorStats.incrementPacketsSent(ReplicatorStats::PACKET_TYPE_InstanceDelete);
+                replicator.replicatorStats.samplePacketsSent(
+                    ReplicatorStats::PACKET_TYPE_InstanceDelete, bitStream.GetNumberOfBytesUsed() - byteStart);
+            }
+        }
+        catch (std::runtime_error& e)
+        {
+            Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE, "Replication: ~%s >> %s, %s",
+                "UnknownInstance", // instance->getClassName().c_str(),
+                RakNetAddressToString(replicator.remotePlayerId).c_str(), e.what());
+        }
+    }
+    return true;
+}
+
+shared_ptr<DeserializedItem> Replicator::DeleteInstanceItem::read(Replicator& replicator, RakNet::BitStream& inBitstream)
+{
+    shared_ptr<DeserializedDeleteInstanceItem> deserializedData(new DeserializedDeleteInstanceItem());
+
+    int start = inBitstream.GetReadOffset();
+
+    replicator.deserializeId(inBitstream, deserializedData->id);
+
+    if (replicator.settings().trackDataTypes)
+    {
+        replicator.replicatorStats.incrementPacketsReceived(ReplicatorStats::PACKET_TYPE_InstanceDelete);
+        replicator.replicatorStats.samplePacketsReceived(ReplicatorStats::PACKET_TYPE_InstanceDelete, (inBitstream.GetReadOffset() - start) / 8);
+    }
+
+    return deserializedData;
+}
+
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.DeleteInstanceItem.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "Item.hpp"
+#include "Replicator.hpp"
+#include "Streaming.hpp"
+
+#include <boost/scoped_ptr.hpp>
+
+namespace Aya
+{
+
+class Instance;
+
+namespace Network
+{
+
+class DeserializedDeleteInstanceItem : public DeserializedItem
+{
+public:
+    Aya::Guid::Data id;
+
+    DeserializedDeleteInstanceItem();
+    ~DeserializedDeleteInstanceItem() {}
+
+    /*implement*/ void process(Replicator& replicator);
+};
+
+class Replicator::DeleteInstanceItem : public PooledItem
+{
+    const IdSerializer::Id id;
+    struct InstanceDetails
+    {
+        std::string className;
+        std::string guid;
+    };
+    boost::scoped_ptr<InstanceDetails> details;
+
+public:
+    DeleteInstanceItem(Replicator* replicator, const shared_ptr<const Instance>& instance);
+
+    /*implement*/ virtual bool write(RakNet::BitStream& bitStream);
+    static shared_ptr<DeserializedItem> read(Replicator& replicator, RakNet::BitStream& bitStream);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.EventInvocationItem.cpp

@@ -0,0 +1,121 @@
+#include "Replicator.EventInvocationItem.hpp"
+
+#include "Item.hpp"
+#include "NetworkSettings.hpp"
+#include "Replicator.hpp"
+#include "Util.hpp"
+
+#include "Reflection/Event.hpp"
+#include "Utility/StandardOut.hpp"
+
+
+#include "RakNet/BitStream.hpp"
+
+namespace Aya
+{
+namespace Network
+{
+
+DeserializedEventInvocationItem::DeserializedEventInvocationItem()
+{
+    type = Item::ItemTypeEventInvocation;
+}
+
+void DeserializedEventInvocationItem::process(Replicator& replicator)
+{
+    replicator.readEventInvocationItem(this);
+}
+
+Replicator::EventInvocationItem::EventInvocationItem(Replicator* replicator, const shared_ptr<Instance>& instance,
+    const Reflection::EventDescriptor& desc, const Reflection::EventArguments& arguments)
+    : PooledItem(*replicator)
+    , desc(desc)
+    , instance(instance)
+    , arguments(arguments)
+{
+}
+
+bool Replicator::EventInvocationItem::write(RakNet::BitStream& bitStream)
+{
+    // Check to see if we are still replicating this object
+    if (!replicator.isReplicationContainer(instance.get()))
+        return true;
+
+    DescriptorSender<Aya::Reflection::EventDescriptor>::IdContainer idContainer = replicator.eventDictionary.getId(&desc);
+    if (idContainer.outdated)
+    {
+        return true;
+    }
+
+    if (replicator.isEventRemoved(instance.get(), desc.name))
+    {
+        return true;
+    }
+
+    int byteStart = bitStream.GetNumberOfBytesUsed();
+    writeItemType(bitStream, ItemTypeEventInvocation);
+
+    // Write the GUID
+    replicator.serializeId(bitStream, instance.get());
+
+    // Write property name
+    replicator.eventDictionary.send(bitStream, idContainer.id);
+
+    // Write value
+    replicator.serializeEventInvocation(Reflection::EventInvocation(Reflection::Event(desc, instance), arguments), bitStream);
+
+    if (replicator.settings().printEvents)
+    {
+        Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE, "Replication: %s-%s.%s >> %s, bytes: %d", instance->getClassName().c_str(),
+            instance->getGuid().readableString().c_str(), desc.name.c_str(), RakNetAddressToString(replicator.remotePlayerId).c_str(),
+            bitStream.GetNumberOfBytesUsed() - byteStart);
+    }
+
+    if (replicator.settings().trackDataTypes)
+    {
+        replicator.replicatorStats.incrementPacketsSent(ReplicatorStats::PACKET_TYPE_Event);
+        replicator.replicatorStats.samplePacketsSent(ReplicatorStats::PACKET_TYPE_Event, bitStream.GetNumberOfBytesUsed() - byteStart);
+    }
+
+    return true;
+}
+
+shared_ptr<DeserializedItem> Replicator::EventInvocationItem::read(Replicator& replicator, RakNet::BitStream& inBitstream)
+{
+    shared_ptr<DeserializedEventInvocationItem> deserializedData(new DeserializedEventInvocationItem());
+
+    shared_ptr<Instance> deseiralizedInstance;
+    Aya::Guid::Data id;
+    replicator.deserializeInstanceRef(inBitstream, deseiralizedInstance, id);
+
+    // Read the event name
+    unsigned int eventId = replicator.eventDictionary.receive(inBitstream, deserializedData->eventDescriptor, true);
+
+    if (replicator.ProcessOutdatedEventInvocation(inBitstream, id, deseiralizedInstance.get(), deserializedData->eventDescriptor, eventId))
+    {
+        return shared_ptr<DeserializedEventInvocationItem>();
+    }
+
+    if (deseiralizedInstance && !deseiralizedInstance->getDescriptor().isA(deserializedData->eventDescriptor->owner))
+    {
+        throw Aya::runtime_error("Replication: Bad re-binding event %s-%s << %s", deseiralizedInstance->getClassName().c_str(),
+            deserializedData->eventDescriptor->name.c_str(), RakNetAddressToString(replicator.remotePlayerId).c_str());
+    }
+
+    if (replicator.settings().printEvents)
+        Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE,
+            "Replication: %s-%s.%s << %s", // remote player always on right side
+            deseiralizedInstance ? deseiralizedInstance->getClassName().c_str() : "?", id.readableString().c_str(),
+            deserializedData->eventDescriptor->name.c_str(), RakNetAddressToString(replicator.remotePlayerId).c_str());
+
+    deserializedData->instance = deseiralizedInstance;
+
+    deserializedData->eventInvocation.reset(
+        new Reflection::EventInvocation(Reflection::Event(*deserializedData->eventDescriptor, deserializedData->instance)));
+    replicator.deserializeEventInvocation(inBitstream, *deserializedData->eventInvocation, false);
+
+    return deserializedData;
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.EventInvocationItem.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "Item.hpp"
+#include "Replicator.hpp"
+
+#include "Reflection/Event.hpp"
+
+namespace Aya
+{
+namespace Network
+{
+
+class DeserializedEventInvocationItem : public DeserializedItem
+{
+public:
+    shared_ptr<Instance> instance;
+    const Reflection::EventDescriptor* eventDescriptor;
+    shared_ptr<Reflection::EventInvocation> eventInvocation;
+
+    DeserializedEventInvocationItem();
+    ~DeserializedEventInvocationItem() {}
+
+    /*implement*/ void process(Replicator& replicator);
+};
+
+class Replicator::EventInvocationItem : public PooledItem
+{
+    const Reflection::EventDescriptor& desc;
+    const shared_ptr<Instance> instance;
+    const Reflection::EventArguments arguments;
+
+public:
+    EventInvocationItem(Replicator* replicator, const shared_ptr<Instance>& instance, const Reflection::EventDescriptor& desc,
+        const Reflection::EventArguments& arguments);
+
+    /*implement*/ virtual bool write(RakNet::BitStream& bitStream);
+    static shared_ptr<DeserializedItem> read(Replicator& replicator, RakNet::BitStream& bitStream);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.GCJob.cpp

@@ -0,0 +1,532 @@
+#include "Replicator.GCJob.hpp"
+
+#include "ClientReplicator.hpp"
+#include "Compressor.hpp"
+#include "Player.hpp"
+#include "Util.hpp"
+
+#include "DrawAdorn.hpp"
+#include "DenseHash.hpp"
+
+#include "DataModel/Workspace.hpp"
+
+#include "DataModel/DataModel.hpp"
+
+#include "DataModel/PartInstance.hpp"
+
+#include "World/ContactManager.hpp"
+#include "World/ContactManagerSpatialHash.hpp"
+#include "Utility/ScopedAssign.hpp"
+
+#include "Utility/StreamRegion.hpp"
+
+#include "Base/Adorn.hpp"
+#include "DataModel/JointsService.hpp"
+
+#include "NetworkProfiler.hpp"
+#include "Humanoid/Humanoid.hpp"
+
+// #include "World/Contact.hpp"
+// #include "Kernel/Body.hpp"
+
+// #include "Kernel/SimBody.hpp"
+
+
+DYNAMIC_LOGGROUP(PartStreamingRequests)
+DYNAMIC_FASTINTVARIABLE(PartStreamingGCMinRegionLength, 2);
+// Default value: each id takes about 5 bytes (1 byte for scope, 4 bytes for id)
+// and expected mtu size ~1500, so a little less than 300 ids fit in one mtu.
+FASTINTVARIABLE(StreamOutCompressionIdListLengthThreshold, 250);
+DYNAMIC_FASTINTVARIABLE(JoinDataCompressionLevel, 1)
+DYNAMIC_FASTINTVARIABLE(JoinDataBonus, 0)
+
+using namespace Aya;
+using namespace Aya::Network;
+
+namespace
+{
+bool compRegionDistance(const RegionsMap::iterator a, const RegionsMap::iterator b)
+{
+    // far to near, in axis
+    return a->second.streamDistance > b->second.streamDistance;
+}
+} // namespace
+
+class ClientReplicator::GCJob::RegionRemovalItem : public Item
+{
+    StreamRegion::Id region;
+    std::vector<Guid::Data> collectedInstanceGuids;
+
+public:
+    RegionRemovalItem(Replicator* replicator, StreamRegion::Id id)
+        : Item(*replicator)
+        , region(id)
+    {
+    }
+    bool addInstance(shared_ptr<Instance> partInstance)
+    {
+        Guid::Data data;
+        partInstance->getGuid().extract(data);
+        if (std::find(collectedInstanceGuids.begin(), collectedInstanceGuids.end(), data) == collectedInstanceGuids.end())
+        {
+            collectedInstanceGuids.push_back(data);
+            return true;
+        }
+        return false;
+    }
+    bool write(RakNet::BitStream& bitStream)
+    {
+        writeItemType(bitStream, ItemTypeRegionRemoval);
+        bitStream << region;
+        boost::uint32_t numberOfCollectedGuids = collectedInstanceGuids.size();
+        bitStream << numberOfCollectedGuids;
+
+        bool useCompression = FInt::StreamOutCompressionIdListLengthThreshold >= 0 &&
+                              numberOfCollectedGuids > ((unsigned int)FInt::StreamOutCompressionIdListLengthThreshold);
+
+        bitStream << useCompression;
+
+        if (useCompression)
+        {
+            RakNet::BitStream uncompressedBuffer;
+
+            for (boost::uint32_t i = 0; i < collectedInstanceGuids.size(); ++i)
+            {
+                replicator.serializeId(uncompressedBuffer, collectedInstanceGuids[i]);
+            }
+
+            Replicator::compressBitStream(uncompressedBuffer, bitStream, DFInt::JoinDataCompressionLevel);
+        }
+        else
+        {
+            for (boost::uint32_t i = 0; i < collectedInstanceGuids.size(); ++i)
+            {
+                replicator.serializeId(bitStream, collectedInstanceGuids[i]);
+            }
+        }
+
+        return true;
+    }
+};
+
+class ClientReplicator::GCJob::InstanceRemovalItem : public Item
+{
+public:
+    InstanceRemovalItem(Replicator* replicator, Guid::Data id)
+        : Item(*replicator)
+        , id(id)
+    {
+    }
+    bool write(RakNet::BitStream& bitStream)
+    {
+        writeItemType(bitStream, ItemTypeInstanceRemoval);
+        replicator.serializeId(bitStream, id);
+        return true;
+    }
+
+private:
+    Guid::Data id;
+};
+
+ClientReplicator::GCJob::GCJob(Replicator& replicator)
+    : ReplicatorJob("Replicator GC Job", replicator, DataModelJob::Write)
+    , numSorted(0)
+    , numRegionToGC(0)
+    , renderingDistance(NULL)
+    , renderingRegionDistance(0x7FFF)
+    , gcRegionDistance(0x7FFF)
+    , maxRegionDistance(0x7FFF)
+    , lastPlayerPosition(Vector3::zero())
+    , centerRegion(Vector3int32::maxInt())
+{
+    cyclicExecutive = true;
+
+    if (Workspace* workspace = Workspace::findWorkspace(&replicator))
+    {
+        if (World* world = workspace->getWorld())
+        {
+            spatialHash = world->getContactManager()->getSpatialHash();
+        }
+        renderingDistance = &workspace->renderingDistance;
+    }
+
+    AYAASSERT(spatialHash);
+    if (spatialHash)
+    {
+        spatialHash->registerCoarseMovementCallback(this);
+    }
+}
+
+ClientReplicator::GCJob::~GCJob()
+{
+    unregisterCoarseMovementCallback();
+}
+
+void ClientReplicator::GCJob::unregisterCoarseMovementCallback()
+{
+    if (Workspace* workspace = Workspace::findWorkspace(replicator.get()))
+    {
+        if (World* world = workspace->getWorld())
+        {
+            spatialHash = world->getContactManager()->getSpatialHash();
+            if (spatialHash)
+            {
+                StandardOut::singleton()->printf(MESSAGE_INFO, "unregisterCoarseMovementCallback() from ~GCJob()");
+                spatialHash->unregisterCoarseMovementCallback(this);
+            }
+        }
+    }
+}
+
+TaskScheduler::Job::Error ClientReplicator::GCJob::error(const Stats& stats)
+{
+    // TODO: compute error based on need to gc
+    return computeStandardError(stats, replicator->settings().getDataGCRate());
+}
+
+void ClientReplicator::GCJob::evaluateNumRegionToGC()
+{
+    if (numRegionToGC <= 0)
+    {
+        numRegionToGC = streamedRegionList.size() / 2;
+        int minGcSize = StreamRegion::Constants::kMinNumPlayableRegion;
+        if (numRegionToGC < minGcSize)
+        {
+            numRegionToGC = streamedRegionList.size() - minGcSize;
+            if (numRegionToGC < 0)
+            {
+                numRegionToGC = 0;
+            }
+        }
+    }
+}
+
+TaskScheduler::StepResult ClientReplicator::GCJob::stepDataModelJob(const Stats& stats)
+{
+    bool tryGC = false;
+    bool forceGC = false;
+
+    if (!replicator)
+        return TaskScheduler::Done;
+
+    // get the player CFrame if they are alive
+    CoordinateFrame cf;
+    bool playerIsDead = true;
+    if (const Player* player = replicator->findTargetPlayer())
+        if (const ModelInstance* characterModel = player->getConstCharacter())
+            if (const Humanoid* h = characterModel->findConstFirstModifierOfType<Humanoid>())
+                if (!h->getDead())
+                {
+                    playerIsDead = false;
+                    if (const PartInstance* head = Humanoid::getConstHeadFromCharacter(characterModel))
+                        cf = head->getCoordinateFrame();
+                }
+
+    ClientReplicator* clientRep = replicator->fastDynamicCast<ClientReplicator>();
+    AYAASSERT(clientRep);
+    clientRep->updateMemoryStats();
+    if (!playerIsDead && (clientRep->getMemoryLevel() <= Aya::MemoryStats::MEMORYLEVEL_ALL_CRITICAL_LOW))
+    {
+        // Memory level too low, force GC
+        tryGC = true;
+        forceGC = true;
+    }
+
+    // sample the rendering region distance
+    AYAASSERT(renderingDistance);
+    if (renderingDistance)
+    {
+        AYAASSERT((*renderingDistance) / StreamRegion::Id::streamGridCellSizeInStuds() < 32767.f);
+        float oldRenderingRegionDistance = renderingRegionDistance;
+        short newRenderingRegionDistance = (short)((*renderingDistance) / StreamRegion::Id::streamGridCellSizeInStuds()) + 1;
+        // smoothing the variance by weight averaging the final value
+        renderingRegionDistance = (renderingRegionDistance * 2.0f + newRenderingRegionDistance) / 3.0f;
+        if (renderingRegionDistance != oldRenderingRegionDistance)
+        {
+            FASTLOG1F(DFLog::PartStreamingRequests, "Rendering region distance: %f", renderingRegionDistance);
+        }
+    }
+
+    // check current player location, reset stream center if needed
+    if (!playerIsDead)
+    {
+        StreamRegion::Id playerRegionId = StreamRegion::regionContainingWorldPosition(cf.translation);
+        if (centerRegion != playerRegionId)
+        {
+            int regionSizeSquared = StreamRegion::Id::streamGridCellSizeInStuds() * StreamRegion::Id::streamGridCellSizeInStuds();
+            if (forceGC || (cf.translation - lastPlayerPosition).squaredMagnitude() >= regionSizeSquared)
+            {
+                FASTLOG1F(DFLog::PartStreamingRequests, "Last player pos translation squared: %.0f",
+                    (cf.translation - lastPlayerPosition).squaredMagnitude());
+
+                // recompute distance
+                for (RegionList::iterator iter = streamedRegionList.begin(); iter != streamedRegionList.end(); iter++)
+                {
+                    (*iter)->second.computeRegionDistance(playerRegionId);
+                }
+
+                // TODO: nth_element sort?
+                streamedRegionList.sort(compRegionDistance);
+                numSorted = streamedRegionList.size();
+                lastPlayerPosition = cf.translation;
+                tryGC = true;
+
+                centerRegion = playerRegionId;
+            }
+        }
+    }
+
+    if (pendingGC())
+    {
+        tryGC = true;
+    }
+
+    DataModel::scoped_write_request request(replicator.get());
+
+    if (tryGC)
+    {
+        if (clientRep->needGC())
+        {
+            evaluateNumRegionToGC();
+            if (numRegionToGC > 0)
+            {
+                pendingRemovalPartInstances.clear();
+                // if GC is needed, check to see if new regions arrived since last sort, GC should always use a sorted list
+                if ((numSorted < (int)streamedRegionList.size()))
+                {
+                    // recompute distance for newly arrived regions using cached player position
+                    StreamRegion::Id lastPlayerRegionId = StreamRegion::regionContainingWorldPosition(lastPlayerPosition);
+                    int numNewRegions = streamedRegions.size() - numSorted;
+                    RegionList::iterator iter = streamedRegionList.end();
+                    std::advance(iter, -numNewRegions);
+                    for (; iter != streamedRegionList.end(); iter++)
+                    {
+                        (*iter)->second.computeRegionDistance(lastPlayerRegionId);
+                    }
+
+                    streamedRegionList.sort(compRegionDistance);
+                    numSorted = streamedRegionList.size();
+                }
+                FASTLOG1(DFLog::PartStreamingRequests, "Not enough memory: %u.  Garbage collecting some items.", MemoryStats::freeMemoryBytes());
+
+                // local delete
+                Time t = Time::now<Time::Fast>() + Time::Interval(0.2 / replicator->settings().getDataGCRate());
+                while (numSorted > 0 && numRegionToGC > 0)
+                {
+                    StreamRegion::Id regionId = streamedRegionList.front()->second.id;
+                    StreamRegion::Id characterRegionId = StreamRegion::regionContainingWorldPosition(cf.translation);
+                    int regionDistance = StreamRegion::Id::getRegionLongestAxisDistance(regionId, characterRegionId);
+
+                    if (!forceGC && (!clientRep->needGC() || regionDistance <= DFInt::PartStreamingGCMinRegionLength))
+                    {
+                        numRegionToGC = 0;
+                        tryGC = false;
+                        FASTLOG1(DFLog::PartStreamingRequests, "GC Job completed.  Memory free: %u", MemoryStats::freeMemoryBytes());
+                        break;
+                    }
+
+                    if (regionDistance < gcRegionDistance)
+                    {
+                        gcRegionDistance = regionDistance - 1;
+                        AYAASSERT(gcRegionDistance > 0);
+                    }
+
+                    // remove region from containers
+                    streamedRegionList.pop_front();
+                    streamedRegions.erase(regionId);
+                    numSorted--;
+                    numRegionToGC--;
+
+                    // TODO: batch regions into 1 item
+                    RegionRemovalItem* regionRemoveItem = new RegionRemovalItem(replicator.get(), regionId);
+
+                    gcRegion(regionId, regionRemoveItem);
+
+                    clientRep->pendingItems.push_back(regionRemoveItem);
+
+                    // break if over time budget
+                    if (Time::now<Time::Fast>() > t)
+                    {
+                        if (forceGC)
+                        {
+                            // if we are in forced GC mode, we only break if memory is above critical level
+                            clientRep->updateMemoryStats();
+                            if (clientRep->getMemoryLevel() >= Aya::MemoryStats::MEMORYLEVEL_LIMITED)
+                            {
+                                break;
+                            }
+                        }
+                        else
+                        {
+                            break;
+                        }
+                    }
+                }
+
+                if (pendingRemovalPartInstances.size() > 0)
+                {
+                    CPUPROFILER_START(NetworkProfiler::PROFILER_jointRemoval);
+                    // gc invalid joints
+                    ClientReplicator* cRep = static_cast<ClientReplicator*>(replicator.get());
+                    if (JointsService* jointsService = ServiceProvider::find<JointsService>(cRep))
+                    {
+                        jointsService->visitDescendants(
+                            boost::bind(&ClientReplicator::streamOutAutoJointHelper, cRep, pendingRemovalPartInstances, _1));
+                    }
+                    CPUPROFILER_STEP(NetworkProfiler::PROFILER_jointRemoval);
+                    for (std::vector<shared_ptr<PartInstance>>::iterator iter = pendingRemovalPartInstances.begin();
+                        iter != pendingRemovalPartInstances.end(); iter++)
+                    {
+                        shared_ptr<PartInstance> partInstance = *iter;
+                        {
+                            if (partInstance)
+                            {
+                                Aya::ScopedAssign<Instance*> assign(clientRep->removingInstance, partInstance.get());
+                                partInstance->setParent(NULL);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        else
+        {
+            // if gc is not needed, gradually increase the gc radius
+            updateGcRegionDistance();
+        }
+    }
+
+    setMaxSimulationRadius((float)(gcRegionDistance - 2.5f) * StreamRegion::Id::streamGridCellSizeInStuds());
+
+    return TaskScheduler::Stepped;
+}
+
+void ClientReplicator::GCJob::insertRegion(const StreamRegion::Id& id)
+{
+    std::pair<RegionsMap::iterator, bool> result = streamedRegions.insert(std::make_pair(id, RegionInfo(id)));
+    if (result.second)
+    {
+        // new region
+        streamedRegionList.push_back(result.first);
+    }
+}
+
+void ClientReplicator::GCJob::setMaxSimulationRadius(float radius)
+{
+    if (Player* player = replicator->findTargetPlayer())
+    {
+        if (player->getMaxSimulationRadius() > radius)
+            player->setMaxSimulationRadius(radius);
+    }
+}
+
+void ClientReplicator::GCJob::coarsePrimitiveMovement(Primitive* p, const UpdateInfo& info)
+{
+
+    StreamRegion::IdExtents oldRegionExtents, newRegionExtents;
+    if (!StreamRegion::coarseMovementCausesStreamRegionChange(info, &oldRegionExtents, &newRegionExtents))
+    {
+        return;
+    }
+
+    shared_ptr<PartInstance> instance = shared_from(PartInstance::fromPrimitive(p));
+    if (!newRegionExtents.intersectsContainer(streamedRegions))
+    {
+        // Mark the region as streamed so it can be GC'ed when needed
+        insertRegion(StreamRegion::regionContainingWorldPosition(instance->getCoordinateFrame().translation));
+    }
+}
+
+void ClientReplicator::GCJob::gcRegion(const StreamRegion::Id& regionId, RegionRemovalItem* removeItem)
+{
+    DenseHashSet<Primitive*> found(NULL);
+    spatialHash->getPrimitivesOverlapping(StreamRegion::extentsFromRegionId(regionId), found);
+
+    for (DenseHashSet<Primitive*>::const_iterator iter = found.begin(); iter != found.end(); ++iter)
+    {
+        // This assumes that the streamed regions set has already been updated
+        StreamRegion::IdExtents regionExtents =
+            StreamRegion::regionExtentsFromContactManagerLevelAndExtents((*iter)->getSpatialNodeLevel(), (*iter)->getOldSpatialExtents());
+
+        PartInstance* part = PartInstance::fromPrimitive(*iter);
+        if (!regionExtents.intersectsContainer(streamedRegions) && !replicator->isInstanceAChildOfClientsCharacterModel(part))
+        {
+            gcPartInstance(part, removeItem);
+        }
+    }
+
+    OneQuarterClusterChunkCellIterator cellIterator;
+    cellIterator.setToStartOfStreamRegion(regionId);
+    while (cellIterator.size())
+    {
+        Vector3int16 cellPos;
+        cellIterator.pop(&cellPos);
+        replicator->fastDynamicCast<ClientReplicator>()->streamOutTerrain(cellPos);
+    }
+}
+
+void ClientReplicator::GCJob::gcPartInstance(PartInstance* part, RegionRemovalItem* removeItem)
+{
+    AYAASSERT_SLOW(!replicator->isInstanceAChildOfClientsCharacterModel(part));
+
+    pendingRemovalPartInstances.push_back(shared_from(part));
+
+    if (removeItem->addInstance(shared_from(part)))
+    {
+        part->visitDescendants(boost::bind(&RegionRemovalItem::addInstance, removeItem, _1));
+        replicator->fastDynamicCast<ClientReplicator>()->streamOutInstance(part, false);
+    }
+}
+
+void ClientReplicator::GCJob::render3dAdorn(Adorn* adorn)
+{
+    for (RegionList::reverse_iterator iter = streamedRegionList.rbegin(); iter != streamedRegionList.rend(); iter++)
+    {
+        Extents ext = StreamRegion::extentsFromRegionId((*iter)->second.id);
+        DrawAdorn::outlineBox(adorn, AABox(ext.min(), ext.max()), Color4(Color3::red(), 0.5f));
+    }
+}
+
+bool ClientReplicator::GCJob::updateMaxRegionDistance()
+{
+    if (renderingDistanceUpdateTimer.delta().seconds() > kRenderingDistanceUpdateInterval)
+    {
+        renderingDistanceUpdateTimer.reset();
+        short newMaxRegionDistance = (renderingRegionDistance < gcRegionDistance ? renderingRegionDistance : gcRegionDistance);
+        if (maxRegionDistance != newMaxRegionDistance)
+        {
+            maxRegionDistance = newMaxRegionDistance;
+            return true;
+        }
+        else
+            return false;
+    }
+    else
+    {
+        return false;
+    }
+}
+
+void ClientReplicator::GCJob::updateGcRegionDistance()
+{
+    if (maxRegionExpansionTimer.delta().seconds() > kMaxRegionExpansionTimerLimit)
+    {
+        maxRegionExpansionTimer.reset();
+        if (gcRegionDistance < 0x7FFF)
+            gcRegionDistance++;
+    }
+}
+
+void ClientReplicator::GCJob::notifyServerGcingInstanceAndDescendants(shared_ptr<Instance> instance)
+{
+    Guid::Data data;
+    instance->getGuid().extract(data);
+    InstanceRemovalItem* instanceRemovalItem = new InstanceRemovalItem(replicator.get(), data);
+    ClientReplicator* clientRep = replicator->fastDynamicCast<ClientReplicator>();
+    clientRep->pendingItems.push_back(instanceRemovalItem);
+
+    // important: use visit children (_not_ visitDescendants) because this function
+    // is recurring down one step at a time. If visitDescendants is called we can queue
+    // multiple removal items for grandchildren and great-grandchildren.
+    instance->visitChildren(boost::bind(&ClientReplicator::GCJob::notifyServerGcingInstanceAndDescendants, this, _1));
+}

engine/network/src/Replicator.GCJob.hpp

@@ -0,0 +1,125 @@
+#pragma once
+
+#include "ClientReplicator.hpp"
+#include "Util.hpp"
+#include "World/ContactManagerSpatialHash.hpp"
+#include "Utility/StreamRegion.hpp"
+
+#include "Base/IAdornable.hpp"
+#include <vector>
+
+
+namespace Aya
+{
+
+namespace Network
+{
+
+static const float kMaxRegionExpansionTimerLimit = 10.f;
+static const float kRenderingDistanceUpdateInterval = 1.f;
+struct RegionInfo
+{
+    float distance;
+    int streamDistance;
+    StreamRegion::Id id;
+
+    explicit RegionInfo(StreamRegion::Id id)
+        : id(id)
+    {
+    }
+
+    bool operator==(const RegionInfo& other) const
+    {
+        return id == other.id;
+    }
+
+    void computeRegionDistance(const StreamRegion::Id& focus)
+    {
+        streamDistance = StreamRegion::Id::getRegionLongestAxisDistance(id, focus);
+    }
+};
+
+typedef boost::unordered_map<StreamRegion::Id, RegionInfo> RegionsMap;
+typedef std::list<RegionsMap::iterator> RegionList;
+
+class ClientReplicator::GCJob
+    : public ReplicatorJob
+    , public ContactManagerSpatialHash::CoarseMovementCallback
+{
+    ContactManagerSpatialHash* spatialHash;
+    RegionsMap streamedRegions;
+    RegionList streamedRegionList;
+
+    Vector3 lastPlayerPosition;
+    int numSorted;
+    int numRegionToGC;
+    float* renderingDistance;
+    float renderingRegionDistance;
+    short gcRegionDistance;
+    short maxRegionDistance;
+    StreamRegion::Id centerRegion;
+
+    Aya::Timer<Aya::Time::Fast> maxRegionExpansionTimer;
+    Aya::Timer<Aya::Time::Fast> renderingDistanceUpdateTimer;
+
+    std::vector<shared_ptr<PartInstance>> pendingRemovalPartInstances;
+
+    class RegionRemovalItem;
+    class InstanceRemovalItem;
+
+public:
+    GCJob(Replicator& replicator);
+    virtual ~GCJob();
+    void insertRegion(const StreamRegion::Id& id);
+
+    // implement CoarseMovementCallback
+    virtual void coarsePrimitiveMovement(Primitive* p, const UpdateInfo& info);
+
+    void render3dAdorn(Adorn* adorn);
+    bool pendingGC()
+    {
+        return (numRegionToGC > 0);
+    }
+    void evaluateNumRegionToGC();
+
+    short getMaxRegionDistance()
+    {
+        return maxRegionDistance;
+    }
+    bool updateMaxRegionDistance();
+    void updateGcRegionDistance();
+    void notifyServerGcingInstanceAndDescendants(shared_ptr<Instance> instance);
+
+    void unregisterCoarseMovementCallback();
+
+    // debug
+    int getNumRegionsToGC()
+    {
+        return numRegionToGC;
+    }
+    short getGCDistance()
+    {
+        return gcRegionDistance;
+    }
+    int getNumRegions()
+    {
+        return streamedRegionList.size();
+    }
+
+private:
+    virtual Time::Interval sleepTime(const Stats& stats)
+    {
+        return computeStandardSleepTime(stats, replicator->settings().getDataGCRate());
+    }
+    virtual Error error(const Stats& stats);
+    virtual TaskScheduler::StepResult stepDataModelJob(const Stats& stats);
+
+    void gcRegion(const StreamRegion::Id& regionId, RegionRemovalItem* removeItem);
+    void gcPartInstance(PartInstance* partInstance, RegionRemovalItem* removeItem);
+
+    void setMaxSimulationRadius(float radius);
+};
+
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.ItemSender.cpp

@@ -0,0 +1,83 @@
+#include "Replicator.ItemSender.hpp"
+
+#include "ConcurrentRakPeer.hpp"
+#include "Item.hpp"
+#include "Replicator.hpp"
+
+#include "RakNet/BitStream.hpp"
+
+#include <boost/noncopyable.hpp>
+#include <boost/shared_ptr.hpp>
+
+namespace Aya
+{
+namespace Network
+{
+
+void Replicator::ItemSender::openPacket()
+{
+    if (!bitStream)
+    {
+        bitStream.reset(new RakNet::BitStream());
+        *bitStream << (unsigned char)ID_DATA;
+    }
+}
+
+void Replicator::ItemSender::closePacket()
+{
+    if (bitStream)
+    {
+        const int bytes = bitStream->GetNumberOfBytesUsed();
+        if (bytes > 0)
+        {
+            // Write the "end" id (non-compact)
+            Item::writeItemType(*bitStream, Item::ItemTypeEnd);
+
+            rakPeer->Send(bitStream, packetPriority, DATAMODEL_RELIABILITY, DATA_CHANNEL, replicator.remotePlayerId, false);
+            replicator.replicatorStats.dataPacketsSent.sample();
+            replicator.replicatorStats.dataPacketsSentSize.sample(bytes);
+        }
+        bitStream.reset();
+    }
+}
+
+Replicator::ItemSender::ItemSender(Replicator& replicator, ConcurrentRakPeer* rakPeer)
+    : replicator(replicator)
+    , rakPeer(rakPeer)
+    , maxStreamSize(replicator.getAdjustedMtuSize()) // guesstimate
+    , sentItems(false)
+    , packetPriority(replicator.settings().getDataSendPriority())
+{
+}
+
+Replicator::ItemSender::~ItemSender()
+{
+    closePacket();
+}
+
+Replicator::ItemSender::SendStatus Replicator::ItemSender::send(Item& item)
+{
+
+    if (bitStream && bitStream->GetNumberOfBytesUsed() >= static_cast<int>(maxStreamSize))
+        return SEND_BITSTREAM_FULL;
+
+    openPacket();
+
+    if (!item.write(*bitStream))
+        return SEND_BITSTREAM_FULL;
+
+    sentItems = true;
+
+    return SEND_OK;
+}
+
+int Replicator::ItemSender::getNumberOfBytesUsed() const
+{
+    if (bitStream)
+        return bitStream->GetNumberOfBytesUsed();
+
+    return 0;
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.ItemSender.hpp

@@ -0,0 +1,45 @@
+#pragma once
+
+#include "Replicator.hpp"
+
+#include "RakNet/BitStream.hpp"
+
+#include <boost/noncopyable.hpp>
+#include <boost/shared_ptr.hpp>
+
+namespace Aya
+{
+namespace Network
+{
+
+class ConcurrentRakPeer;
+class Item;
+
+class Replicator::ItemSender : boost::noncopyable
+{
+    Replicator& replicator;
+    ConcurrentRakPeer* rakPeer;
+    shared_ptr<RakNet::BitStream> bitStream;
+    PacketPriority packetPriority;
+
+    void openPacket();
+    void closePacket();
+    const unsigned int maxStreamSize;
+
+public:
+    bool sentItems;
+    ItemSender(Replicator& replicator, ConcurrentRakPeer* rakPeer);
+    ~ItemSender();
+
+    typedef enum
+    {
+        SEND_BITSTREAM_FULL = 0,
+        SEND_OK
+    } SendStatus;
+
+    SendStatus send(Item& item);
+    int getNumberOfBytesUsed() const;
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.JoinDataItem.cpp

@@ -0,0 +1,239 @@
+#include "Replicator.JoinDataItem.hpp"
+#include "Replicator.NewInstanceItem.hpp"
+
+SYNCHRONIZED_FASTFLAGVARIABLE(NetworkAlignJoinData, true) // 223
+
+namespace Aya
+{
+namespace Network
+{
+
+DeserializedJoinDataItem::DeserializedJoinDataItem()
+    : numInstances(0)
+{
+    type = Item::ItemTypeJoinData;
+}
+
+void DeserializedJoinDataItem::process(Replicator& replicator)
+{
+    replicator.readJoinDataItem(this);
+}
+
+bool Replicator::JoinDataItem::canUseCache(const Instance* instance)
+{
+    const PartInstance* part = instance->fastDynamicCast<PartInstance>();
+    if (part)
+    {
+        if (NetworkOwner::isClient(part->getNetworkOwner()))
+            return true;
+        else if (part->getSleeping())
+        {
+            // when a part is simulated by the server, physics changes does not trigger prop change signal,
+            // therefore causing cache to not mark itself dirty. This check prevent cache usage if part is simulated
+            // by the server and awake.
+            return true;
+        }
+
+        return false;
+    }
+
+    // Script instances have a ProtectedString property that switches the replication format based on protocol version
+    // We can't cache these instances between different replicators until we can have older clients
+    BOOST_STATIC_ASSERT(NETWORK_PROTOCOL_VERSION_MIN < 28); // Remove the if check when min protocol version is 28
+    if (instance->isA<BaseScript>() || instance->isA<ModuleScript>())
+        return false;
+
+    // can use cache if not a part instance
+    return true;
+}
+
+
+bool Replicator::JoinDataItem::writeInstance(const Instance* instance, RakNet::BitStream& bitStream)
+{
+    // If the class is outdated, we still write the instance, because the property and event serializers will take care of the changes
+    // But if the class is removed, we simply bypass it
+    if (replicator.isClassRemoved(instance))
+    {
+        return false;
+    }
+
+    DescriptorSender<Aya::Reflection::ClassDescriptor>::IdContainer idContainer = replicator.classDictionary.getId(&instance->getDescriptor());
+
+    // Write GUID
+    replicator.serializeIdWithoutDictionary(bitStream, instance);
+
+    if (replicator.settings().printInstances)
+        Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE, // remote player always on right
+            "Replication NewInstance::write: %s:%s:%s >> %s", instance->getClassName().c_str(), instance->getGuid().readableString().c_str(),
+            instance->getName().c_str(), RakNetAddressToString(replicator.remotePlayerId).c_str());
+
+    // Write ClassName, ok to use dictionary here because classDictionary entries are already sent to the client
+    replicator.classDictionary.send(bitStream, idContainer.id);
+
+    // Write ownership flag
+    bool deleteOnDisconnect = replicator.remoteDeleteOnDisconnect(instance);
+    bitStream << deleteOnDisconnect;
+
+    // Write all properties
+    replicator.writeProperties(instance, bitStream, PropertyCacheType_All, false /*useDictionary*/);
+
+    // Write the Parent property
+    Reflection::ConstProperty property(Instance::propParent, instance);
+    replicator.serializePropertyValue(property, bitStream, false /*useDictionary*/);
+
+    // This maximizes the compression ratio and improves instance packet cache update performance
+    bitStream.AlignWriteToByteBoundary();
+
+    return true;
+}
+
+
+size_t Replicator::JoinDataItem::writeInstances(RakNet::BitStream& bitStream)
+{
+    // preallocate memory for bitstream using estimated size
+    RakNet::BitStream dataBitstream(instances.size() * 256);
+
+    unsigned int numWritten = 0;
+    while (!instances.empty() && (0 == maxInstancesToWrite || numWritten < maxInstancesToWrite))
+    {
+        const Instance* instance = instances.front().get();
+        if (!replicator.removeFromPendingNewInstances(instance))
+        {
+            instances.pop_front();
+            continue;
+        }
+
+        bool instanceWritten;
+        if (replicator.instancePacketCache && canUseCache(instance)) // use cache
+        {
+            unsigned int startBit = dataBitstream.GetWriteOffset();
+
+            // try fetch from cache.
+            if (!replicator.instancePacketCache->fetchIfUpToDate(instance, dataBitstream, true))
+            {
+                // instance dirty or cached bitstream was not set
+                instanceWritten = writeInstance(instance, dataBitstream);
+                dataBitstream.SetReadOffset(startBit);
+
+                // update the cache
+                replicator.instancePacketCache->update(
+                    instance, dataBitstream, dataBitstream.GetNumberOfBitsUsed() - startBit, true /*isInitalJoinData*/);
+            }
+            else
+            {
+                // fetched from cache
+                instanceWritten = true;
+
+                if (replicator.settings().printInstances)
+                    Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE, // remote player always on right
+                        "Replication NewInstance::write from cache: %s:%s:%s >> %s, %d bits", instance->getClassName().c_str(),
+                        instance->getGuid().readableString().c_str(), instance->getName().c_str(),
+                        RakNetAddressToString(replicator.remotePlayerId).c_str(), dataBitstream.GetWriteOffset() - startBit);
+            }
+        }
+        else
+        {
+            instanceWritten = writeInstance(instance, dataBitstream);
+        }
+
+        if (instanceWritten)
+        {
+            ++numWritten;
+        }
+        instances.pop_front();
+
+        if (sendBytesPerStep > 0)
+        {
+            // estimate data size after compression
+            if (dataBitstream.GetNumberOfBytesUsed() >= (sendBytesPerStep * ESTIMATED_COMPRESSION_RATIO))
+                break;
+        }
+        else
+        {
+            if (instanceWritten)
+            {
+                break; // 1 instance at a time
+            }
+        }
+    }
+
+    // This makes sure we're compressing to a byte-aligned stream to maximize writing performance
+    bitStream.AlignWriteToByteBoundary();
+
+    bitStream << numWritten;
+
+    // compress the data and write to bitstream
+    if (numWritten > 0)
+        Replicator::compressBitStream(dataBitstream, bitStream, DFInt::JoinDataCompressionLevel);
+
+    instancesWrittenOverLifetime += numWritten;
+    FASTLOG2(DFLog::NetworkJoin, "JoinDataItem(0x%p) Finished writing %u instances", this, numWritten);
+
+    return numWritten;
+}
+
+void Replicator::JoinDataItem::addInstance(shared_ptr<const Instance> instance)
+{
+    Aya::mutex::scoped_lock lock(replicator.pendingInstancesMutex);
+    replicator.pendingNewInstances.insert(instance.get());
+
+    instances.push_back(instance);
+
+    // create an entry in new instance bitstream cache
+    if (replicator.instancePacketCache && !Replicator::isTopContainer(instance.get()))
+        replicator.instancePacketCache->insert(instance.get());
+}
+
+
+bool Replicator::JoinDataItem::write(RakNet::BitStream& bitStream)
+{
+    AYAASSERT(sendBytesPerStep > 0);
+
+    Timer<Time::Fast> writeTimer;
+    ++timesWriteCalled;
+
+    writeItemType(bitStream, ItemTypeJoinData);
+    writeInstances(bitStream);
+
+    if (DFInt::JoinDataBonus)
+        writeBonus(bitStream, DFInt::JoinDataBonus);
+
+    FASTLOG1F(DFLog::NetworkJoin, "\tTime: %f ms", writeTimer.delta().msec());
+
+    return instances.empty();
+}
+
+shared_ptr<DeserializedItem> Replicator::JoinDataItem::read(Replicator& replicator, RakNet::BitStream& inBitstream)
+{
+    shared_ptr<DeserializedJoinDataItem> deserializedData(new DeserializedJoinDataItem());
+
+    inBitstream.AlignReadToByteBoundary();
+
+    unsigned int count;
+    inBitstream >> count;
+
+    if (count == 0)
+        return shared_ptr<DeserializedJoinDataItem>();
+
+    RakNet::BitStream bitstream;
+    Replicator::decompressBitStream(inBitstream, bitstream);
+
+    int numInstances = 0;
+    deserializedData->instanceInfos.resize(count);
+    for (unsigned int i = 0; i < count; i++)
+    {
+        if (!NewInstanceItem::read(replicator, bitstream, true, deserializedData->instanceInfos[numInstances]))
+            deserializedData->instanceInfos[numInstances].reset();
+        else
+            numInstances++;
+
+        bitstream.AlignReadToByteBoundary();
+    }
+
+    deserializedData->numInstances = numInstances;
+
+    return deserializedData;
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.JoinDataItem.hpp

@@ -0,0 +1,111 @@
+#pragma once
+
+#include "Replicator.hpp"
+#include "Utility/StandardOut.hpp"
+
+#include "NetworkPacketCache.hpp"
+
+#include "NetworkOwner.hpp"
+
+#include "Util.hpp"
+#include "Compressor.hpp"
+#include "Script/script.hpp"
+#include "Script/ModuleScript.hpp"
+#include "Replicator.NewInstanceItem.hpp"
+
+#include "time.hpp"
+
+#include "Utility/VarInt.hpp"
+
+
+#define ESTIMATED_COMPRESSION_RATIO 5.0f
+
+DYNAMIC_LOGGROUP(NetworkJoin)
+DYNAMIC_FASTINT(JoinDataCompressionLevel)
+
+DYNAMIC_FASTINT(JoinDataBonus)
+
+namespace Aya
+{
+namespace Network
+{
+
+class DeserializedJoinDataItem : public DeserializedItem
+{
+public:
+    int numInstances;
+    std::vector<DeserializedNewInstanceItem> instanceInfos;
+
+    DeserializedJoinDataItem();
+    ~DeserializedJoinDataItem() {}
+
+    /*implement*/ void process(Replicator& replicator);
+};
+
+class Replicator::JoinDataItem : public Item
+{
+    std::list<shared_ptr<const Instance>> instances;
+    int sendBytesPerStep;
+    unsigned timesWriteCalled;
+    size_t instancesWrittenOverLifetime;
+    size_t maxInstancesToWrite;
+
+    bool canUseCache(const Instance* instance);
+
+protected:
+    bool writeInstance(const Instance* instance, RakNet::BitStream& bitStream);
+
+    size_t writeInstances(RakNet::BitStream& bitStream);
+
+    void writeBonus(RakNet::BitStream& bitStream, unsigned int bytes)
+    {
+        unsigned char buf[64] = {};
+
+        while (bytes >= sizeof(buf))
+        {
+            bitStream.WriteAlignedBytes(buf, sizeof(buf));
+            bytes -= sizeof(buf);
+        }
+    }
+
+
+
+public:
+    JoinDataItem(Replicator* replicator)
+        : Item(*replicator)
+        , sendBytesPerStep(-1)
+        , timesWriteCalled(0)
+        , instancesWrittenOverLifetime(0)
+        , maxInstancesToWrite(0)
+    {
+        FASTLOG1(DFLog::NetworkJoin, "Created JoinDataItem(0x%p)", this);
+    }
+    virtual ~JoinDataItem()
+    {
+        FASTLOG3(
+            DFLog::NetworkJoin, "~JoinDataItem(0x%p) handled %u instances over %u invocations", this, instancesWrittenOverLifetime, timesWriteCalled);
+    }
+    void setMaxInstancesToWrite(size_t num)
+    {
+        maxInstancesToWrite = num;
+    }
+    bool empty() const
+    {
+        return instances.empty();
+    }
+    size_t size() const
+    {
+        return instances.size();
+    }
+    void setBytesPerStep(int numBytes)
+    {
+        sendBytesPerStep = numBytes;
+    }
+    void addInstance(shared_ptr<const Instance> instance);
+
+    /*implement*/ virtual bool write(RakNet::BitStream& bitStream);
+    static shared_ptr<DeserializedItem> read(Replicator& replicator, RakNet::BitStream& bitStream);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.MarkerItem.cpp

@@ -0,0 +1,60 @@
+#include "Replicator.MarkerItem.hpp"
+
+#include "Item.hpp"
+#include "Replicator.hpp"
+#include "Util.hpp"
+#include "Replicator.StreamJob.hpp"
+
+#include "RakNet/BitStream.hpp"
+#include <string>
+
+DYNAMIC_LOGGROUP(NetworkJoin)
+
+namespace Aya
+{
+namespace Network
+{
+
+Replicator::MarkerItem::MarkerItem(Replicator* replicator, int id)
+    : Item(*replicator)
+    , id(id)
+{
+}
+
+bool Replicator::MarkerItem::write(RakNet::BitStream& bitStream)
+{
+    if (!replicator.isInitialDataSent())
+        return false;
+
+    writeItemType(bitStream, ItemTypeMarker);
+
+    bitStream << id;
+    if (replicator.settings().printInstances)
+    {
+        Aya::StandardOut::singleton()->printf(
+            Aya::MESSAGE_SENSITIVE, "Replication: Sending marker %d to %s", id, RakNetAddressToString(replicator.remotePlayerId).c_str());
+    }
+
+    replicator.onSentMarker(id);
+    FASTLOG1F(DFLog::NetworkJoin, "MarkerItem %ld sent", id);
+
+    return true;
+}
+
+shared_ptr<DeserializedItem> Replicator::MarkerItem::read(Replicator& replicator, RakNet::BitStream& bitStream)
+{
+    shared_ptr<DeserializedMarkerItem> deserializedData(new DeserializedMarkerItem());
+
+    bitStream >> deserializedData->id;
+
+    if (replicator.settings().printInstances)
+    {
+        Aya::StandardOut::singleton()->printf(
+            Aya::MESSAGE_SENSITIVE, "Received marker %d from %s", deserializedData->id, RakNetAddressToString(replicator.remotePlayerId).c_str());
+    }
+
+    return deserializedData;
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.MarkerItem.hpp

@@ -0,0 +1,40 @@
+#pragma once
+
+#include "Item.hpp"
+#include "Replicator.hpp"
+
+namespace Aya
+{
+namespace Network
+{
+
+class DeserializedMarkerItem : public DeserializedItem
+{
+public:
+    int id;
+
+    DeserializedMarkerItem()
+    {
+        type = Item::ItemTypeMarker;
+    }
+    ~DeserializedMarkerItem() {}
+    /*implement*/ void process(Replicator& replicator)
+    {
+        replicator.readMarkerItem(this);
+    }
+};
+
+class Replicator::MarkerItem : public Item
+{
+    int id;
+
+public:
+    MarkerItem(Replicator* replicator, int id);
+
+    /*implement*/ virtual bool write(RakNet::BitStream& bitStream);
+
+    static shared_ptr<DeserializedItem> read(Replicator& replicator, RakNet::BitStream& bitStream);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.NewInstanceItem.cpp

@@ -0,0 +1,236 @@
+#include "Replicator.NewInstanceItem.hpp"
+
+#include "Item.hpp"
+#include "Replicator.hpp"
+#include "ReplicatorStats.hpp"
+#include "Security/FuzzyTokens.hpp"
+#include "Players.hpp"
+#include "FastLog.hpp"
+#include "NetworkPacketCache.hpp"
+
+#include "DataModel/DataModel.hpp"
+
+#include "DataModel/HackDefines.hpp"
+
+
+#include "RakNet/RakNetTime.hpp"
+#include "RakNet/BitStream.hpp"
+
+
+
+namespace Aya
+{
+namespace Network
+{
+
+DeserializedNewInstanceItem::DeserializedNewInstanceItem()
+    : classDescriptor(NULL)
+{
+    type = Item::ItemTypeNew;
+}
+
+void DeserializedNewInstanceItem::process(Replicator& replicator)
+{
+    replicator.readInstanceNewItem(this, false);
+}
+
+Replicator::NewInstanceItem::NewInstanceItem(Replicator* replicator, shared_ptr<const Instance> instance)
+    : PooledItem(*replicator)
+    , instance(instance)
+    , useStoredParentGuid(false)
+{
+    // check if the flag is on, and that this isn't the special case for
+    // initial player sync.
+    // HACK -- Currently the client replicator creates a NewInstanceItem for
+    // its local player before it has received the correct GUID for the
+    // Players service from the server. So, bypass the parent desync bug
+    // fix if the new instance is a Player under Players.
+    useStoredParentGuid =
+        !(Instance::fastSharedDynamicCast<const Player>(instance) && Instance::fastDynamicCast<const Players>(instance->getParent()));
+
+    if (!replicator->isLegalSendInstance(instance.get()))
+        return;
+
+    Aya::mutex::scoped_lock lock(replicator->pendingInstancesMutex);
+    replicator->pendingNewInstances.insert(instance.get());
+
+    // create an entry in new instance bitstream cache
+    if (replicator->instancePacketCache && !Replicator::isTopContainer(instance.get()))
+        replicator->instancePacketCache->insert(instance.get());
+
+    if (useStoredParentGuid)
+    {
+        instance->getParent()->getGuid().extract(parentIdAtItemCreation);
+    }
+}
+
+bool Replicator::NewInstanceItem::write(RakNet::BitStream& bitStream)
+{
+    // If the class is outdated, we still write the instance, because the property and event serializers will take care of the changes
+    // But if the class is removed, we simply bypass it
+    if (replicator.isClassRemoved(instance.get()))
+    {
+        return true;
+    }
+    DescriptorSender<Aya::Reflection::ClassDescriptor>::IdContainer idContainer = replicator.classDictionary.getId(&instance->getDescriptor());
+
+    int byteStart = bitStream.GetNumberOfBytesUsed();
+    if (!replicator.removeFromPendingNewInstances(instance.get()))
+        return true;
+
+    writeItemType(bitStream, ItemTypeNew);
+
+    // Write the GUID
+    replicator.serializeId(bitStream, instance.get());
+
+    if (replicator.settings().printInstances)
+    {
+        Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE, // remote player always on right
+            "Replication NewInstance::write: %s:%s:%s >> %s", instance->getClassName().c_str(), instance->getGuid().readableString().c_str(),
+            instance->getName().c_str(), RakNetAddressToString(replicator.remotePlayerId).c_str());
+    }
+    if (replicator.settings().trackDataTypes)
+    {
+        replicator.replicatorStats.incrementPacketsSent(ReplicatorStats::PACKET_TYPE_InstanceNew);
+        replicator.replicatorStats.samplePacketsSent(ReplicatorStats::PACKET_TYPE_InstanceNew, bitStream.GetNumberOfBytesUsed() - byteStart);
+    }
+
+    // Write ClassName
+    replicator.classDictionary.send(bitStream, idContainer.id);
+
+    // Write ownership flag
+    bool deleteOnDisconnect = replicator.remoteDeleteOnDisconnect(instance.get());
+    bitStream << deleteOnDisconnect;
+
+    // write non cacheable properties first
+    replicator.writeProperties(instance.get(), bitStream, Replicator::PropertyCacheType_NonCacheable, true);
+
+    if (replicator.instancePacketCache) // use cache
+    {
+        unsigned int startBit = bitStream.GetWriteOffset();
+
+        // try fetch from cache.
+        if (!replicator.instancePacketCache->fetchIfUpToDate(instance.get(), bitStream, false))
+        {
+            // instance dirty or cached bitstream was not set
+
+            replicator.writeProperties(instance.get(), bitStream, Replicator::PropertyCacheType_Cacheable, true);
+
+            bitStream.SetReadOffset(startBit);
+
+            // update the cache
+            replicator.instancePacketCache->update(instance.get(), bitStream, bitStream.GetNumberOfBitsUsed() - startBit, false);
+        }
+        else
+        {
+            if (replicator.settings().printBits)
+            {
+                Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE, "   write from cache %d bits", bitStream.GetWriteOffset() - startBit);
+            }
+        }
+    }
+    else // not using cache
+        replicator.writeProperties(instance.get(), bitStream, Replicator::PropertyCacheType_Cacheable, true);
+
+    // Write the Parent property
+    if (useStoredParentGuid)
+    {
+        replicator.serializeId(bitStream, parentIdAtItemCreation);
+    }
+    else
+    {
+        Reflection::ConstProperty property(Instance::propParent, instance.get());
+        replicator.serializePropertyValue(property, bitStream, true /*useDictionary*/);
+    }
+
+    return true;
+}
+
+bool Replicator::NewInstanceItem::read(
+    Replicator& replicator, RakNet::BitStream& bitStream, bool isJoinData, DeserializedNewInstanceItem& deserializedItem)
+{
+    if (isJoinData)
+        replicator.deserializeIdWithoutDictionary(bitStream, deserializedItem.id);
+    else
+        replicator.deserializeId(bitStream, deserializedItem.id);
+
+    // Get the class and construct the object
+    unsigned int classId = replicator.classDictionary.receive(bitStream, deserializedItem.classDescriptor,
+        false /*we don't do version check for class here, if the properties or events of the class are changed, they will be handled later*/);
+
+    if (replicator.ProcessOutdatedInstance(bitStream, isJoinData, deserializedItem.id, deserializedItem.classDescriptor, classId))
+    {
+        return false;
+    }
+
+    bool newInstance = false;
+    shared_ptr<Instance> i;
+    if (replicator.guidRegistry->lookupByGuid(deserializedItem.id, i))
+    {
+        // We got back an object we've already seen
+        if (i == 0)
+            throw Aya::runtime_error("readInstanceNew got a null object (guid %s)", deserializedItem.id.readableString(32).c_str());
+
+        deserializedItem.instance = i;
+
+        if (deserializedItem.instance->getDescriptor() != *deserializedItem.classDescriptor)
+        {
+            std::string message = Aya::format("Replication: Bad re-binding in deserialize new instance %s-%s << %s, %s-%s",
+                deserializedItem.classDescriptor->name.c_str(), deserializedItem.id.readableString().c_str(),
+                RakNetAddressToString(replicator.remotePlayerId).c_str(), deserializedItem.instance->getClassName().c_str(),
+                deserializedItem.id.readableString().c_str());
+
+            Aya::StandardOut::singleton()->printf(Aya::MESSAGE_SENSITIVE, "%s", message.c_str());
+
+            throw Aya::runtime_error("%s", message.c_str());
+        }
+    }
+    else
+    {
+        deserializedItem.instance = Creatable<Instance>::createByName(deserializedItem.classDescriptor->name, Aya::ReplicationCreator);
+        if (!deserializedItem.instance)
+        {
+            std::string message = format("Replication: Can't create object of type %s", deserializedItem.classDescriptor->name.c_str());
+            Aya::StandardOut::singleton()->print(Aya::MESSAGE_ERROR, message);
+            throw std::runtime_error(message);
+        }
+
+        replicator.guidRegistry->assignGuid(deserializedItem.instance.get(), deserializedItem.id);
+        newInstance = true;
+    }
+
+    bitStream >> deserializedItem.deleteOnDisconnect;
+
+    // Write properties directly into newly created instances since it's not in datamodel yet, otherwise write to variant array to be set later.
+    if (isJoinData)
+    {
+        replicator.readProperties(
+            bitStream, deserializedItem.instance.get(), PropertyCacheType_All, false, false, newInstance ? NULL : &deserializedItem.propValueList);
+        replicator.deserializeIdWithoutDictionary(bitStream, deserializedItem.parentId);
+    }
+    else
+    {
+        replicator.readProperties(bitStream, deserializedItem.instance.get(), PropertyCacheType_NonCacheable, true, false,
+            newInstance ? NULL : &deserializedItem.propValueList);
+        replicator.readProperties(bitStream, deserializedItem.instance.get(), PropertyCacheType_Cacheable, true, false,
+            newInstance ? NULL : &deserializedItem.propValueList);
+        replicator.deserializeId(bitStream, deserializedItem.parentId);
+    }
+
+    replicator.guidRegistry->lookupByGuid(deserializedItem.parentId, deserializedItem.parent);
+
+    return true;
+}
+
+shared_ptr<DeserializedItem> Replicator::NewInstanceItem::read(Replicator& replicator, RakNet::BitStream& bitStream, bool isJoinData)
+{
+    shared_ptr<DeserializedNewInstanceItem> deserializedData(new DeserializedNewInstanceItem());
+
+    if (!read(replicator, bitStream, isJoinData, *deserializedData))
+        return shared_ptr<DeserializedNewInstanceItem>();
+
+    return deserializedData;
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.NewInstanceItem.hpp

@@ -0,0 +1,71 @@
+
+
+#pragma once
+
+#include "Replicator.hpp"
+
+namespace Aya
+{
+namespace Network
+{
+
+struct PropValuePair
+{
+    const Reflection::PropertyDescriptor* descriptor;
+    Reflection::Variant value;
+
+    PropValuePair(const Reflection::PropertyDescriptor& descriptor, const Reflection::Variant& _value)
+        : descriptor(&descriptor)
+        , value(_value)
+    {
+    }
+};
+
+class DeserializedNewInstanceItem : public DeserializedItem
+{
+public:
+    typedef std::vector<PropValuePair> PropValuePairList;
+    PropValuePairList propValueList; // for non creatable instances
+
+    Aya::Guid::Data id;
+    Aya::Guid::Data parentId;
+    const Reflection::ClassDescriptor* classDescriptor;
+
+    shared_ptr<Instance> instance;
+    shared_ptr<Instance> parent;
+    bool deleteOnDisconnect;
+
+    DeserializedNewInstanceItem();
+    ~DeserializedNewInstanceItem(){};
+
+    void reset()
+    {
+        propValueList.clear();
+        instance.reset();
+        parent.reset();
+        id.scope.setNull();
+        parentId.scope.setNull();
+        classDescriptor = NULL;
+    }
+
+    /*implement*/ void process(Replicator& replicator);
+};
+
+class Replicator::NewInstanceItem : public PooledItem
+{
+    shared_ptr<const Instance> instance;
+    bool useStoredParentGuid;
+    Aya::Guid::Data parentIdAtItemCreation;
+
+    bool deleteOnDisconnect;
+
+public:
+    NewInstanceItem(Replicator* replicator, shared_ptr<const Instance> instance);
+
+    bool write(RakNet::BitStream& bitStream);
+    static bool read(Replicator& replicator, RakNet::BitStream& bitStream, bool isJoinData, DeserializedNewInstanceItem& deserializedData);
+    static shared_ptr<DeserializedItem> read(Replicator& replicator, RakNet::BitStream& bitStream, bool isJoinData);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.PingBackItem.cpp

@@ -0,0 +1,71 @@
+#include "Replicator.PingBackItem.hpp"
+
+#include "Item.hpp"
+#include "Replicator.hpp"
+#include "ReplicatorStats.hpp"
+#include "DataModel/DataModel.hpp"
+
+#include "DataModel/HackDefines.hpp"
+
+
+#include "RakNet/RakNetTime.hpp"
+#include "RakNet/BitStream.hpp"
+
+
+
+namespace Aya
+{
+namespace Network
+{
+
+Replicator::PingBackItem::PingBackItem(Replicator* replicator, RakNet::Time time, unsigned int extraStats)
+    : PooledItem(*replicator)
+    , time(time)
+    , extraStats(extraStats)
+{
+}
+
+bool Replicator::PingBackItem::write(RakNet::BitStream& bitStream)
+{
+
+    int byteStart = bitStream.GetNumberOfBytesUsed();
+
+    writeItemType(bitStream, ItemTypePingBack);
+    bitStream << true;
+#if !defined(__linux) && !defined(__APPLE__)
+    bitStream << time;
+#else
+    bitStream << static_cast<unsigned long long>(time);
+#endif
+
+#if !defined(LOVE_ALL_ACCESS)
+    unsigned int sendStats = DataModel::sendStats | DataModel::get(&replicator)->allHackFlagsOredTogether();
+#else
+    unsigned int sendStats = 0;
+#endif
+    bitStream << sendStats;
+
+    if (replicator.canUseProtocolVersion(34))
+    {
+#if !defined(AYA_SERVER) && !defined(AYA_STUDIO)
+        if (time & 0x20)
+        {
+            extraStats = ~extraStats;
+        }
+
+#endif
+        bitStream.Write(static_cast<uint32_t>(extraStats));
+    }
+
+    if (replicator.settings().trackDataTypes)
+    {
+        replicator.replicatorStats.incrementPacketsSent(ReplicatorStats::PACKET_TYPE_Ping);
+        replicator.replicatorStats.samplePacketsSent(ReplicatorStats::PACKET_TYPE_Ping, bitStream.GetNumberOfBytesUsed() - byteStart);
+    }
+
+    
+    return true;
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.PingBackItem.hpp

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "Item.hpp"
+#include "Replicator.hpp"
+
+#include "RakNet/RakNetTime.hpp"
+
+namespace Aya
+{
+namespace Network
+{
+
+class Replicator::PingBackItem : public PooledItem
+{
+    RakNet::Time time;
+    unsigned int extraStats;
+
+public:
+    PingBackItem(Replicator* replicator, RakNet::Time time, unsigned int extraStats);
+
+    /*implement*/ virtual bool write(RakNet::BitStream& bitStream);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.PingItem.cpp

@@ -0,0 +1,137 @@
+#include "Replicator.PingItem.hpp"
+
+#include "Item.hpp"
+#include "Replicator.hpp"
+#include "ReplicatorStats.hpp"
+#include "Security/FuzzyTokens.hpp"
+#include "Security/ApiSecurity.hpp"
+#include "DataModel/HackDefines.hpp"
+
+
+#include "FastLog.hpp"
+#include "DataModel/DataModel.hpp"
+
+
+#include "RakNet/RakNetTime.hpp"
+#include "RakNet/BitStream.hpp"
+
+
+
+namespace Aya
+{
+namespace Network
+{
+
+DeserializedPingItem::DeserializedPingItem()
+    : extraStats(0)
+{
+    type = Item::ItemTypePing;
+}
+
+void DeserializedPingItem::process(Replicator& replicator)
+{
+    replicator.readDataPingItem(this);
+}
+
+Replicator::PingItem::PingItem(Replicator* replicator, RakNet::Time time, unsigned int extraStats)
+    : PooledItem(*replicator)
+    , time(time)
+    , extraStats(extraStats)
+{
+}
+
+bool Replicator::PingItem::write(RakNet::BitStream& bitStream)
+{
+    boost::scoped_ptr<unsigned int> moreStatsCopy(new unsigned int);
+
+    int byteStart = bitStream.GetNumberOfBytesUsed();
+
+    writeItemType(bitStream, ItemTypePing);
+
+    unsigned int moreStats = 0;
+#if !defined(LOVE_ALL_ACCESS) && !defined(AYA_STUDIO) && !defined(AYA_PLATFORM_DURANGO)
+    unsigned int sendStats =
+        Tokens::simpleToken | DataModel::perfStats | DataModel::sendStats | DataModel::get(&replicator)->allHackFlagsOredTogether();
+    // I'm sorry for this.  The values need to be spread out in memory and thus can't be an array.
+    moreStats |= Aya::Security::getHackFlag<LINE_RAND4>(Aya::Security::hackFlag0);
+    moreStats |= Aya::Security::getHackFlag<LINE_RAND4>(Aya::Security::hackFlag1);
+    moreStats |= Aya::Security::getHackFlag<LINE_RAND4>(Aya::Security::hackFlag2);
+    moreStats |= Aya::Security::getHackFlag<LINE_RAND4>(Aya::Security::hackFlag3);
+    moreStats |= Aya::Security::getHackFlag<LINE_RAND4>(Aya::Security::hackFlag4);
+    moreStats |= Aya::Security::getHackFlag<LINE_RAND4>(Aya::Security::hackFlag5);
+    moreStats |= Aya::Security::getHackFlag<LINE_RAND4>(Aya::Security::hackFlag6);
+    moreStats |= Aya::Security::getHackFlag<LINE_RAND4>(Aya::Security::hackFlag7);
+    moreStats |= Aya::Security::getHackFlag<LINE_RAND4>(Aya::Security::hackFlag8);
+    moreStats |= Aya::Security::getHackFlag<LINE_RAND4>(Aya::Security::hackFlag9);
+    moreStats |= Aya::Security::getHackFlag<LINE_RAND4>(Aya::Security::hackFlag10);
+    moreStats |= Aya::Security::getHackFlag<LINE_RAND4>(Aya::Security::hackFlag11);
+    moreStats |= Aya::Security::getHackFlag<LINE_RAND4>(Aya::Security::hackFlag12);
+    moreStats |= sendStats;
+    *moreStatsCopy = moreStats;
+#endif
+    bitStream << false;
+#if !defined(__linux) && !defined(__APPLE__)
+    bitStream << time;
+#else
+    bitStream << static_cast<unsigned long long>(time);
+#endif
+
+    bitStream.Write(static_cast<uint32_t>(moreStats));
+    if (replicator.canUseProtocolVersion(34))
+    {
+#if !defined(AYA_SERVER) && !defined(AYA_STUDIO)
+        if (time & 0x20)
+        {
+            extraStats = ~extraStats;
+        }
+#endif
+        bitStream.Write(static_cast<uint32_t>(extraStats));
+    }
+
+    if (replicator.settings().trackDataTypes)
+    {
+        replicator.replicatorStats.incrementPacketsSent(ReplicatorStats::PACKET_TYPE_Ping);
+        replicator.replicatorStats.samplePacketsSent(ReplicatorStats::PACKET_TYPE_Ping, bitStream.GetNumberOfBytesUsed() - byteStart);
+    }
+
+#if !defined(AYA_STUDIO)
+    if (*moreStatsCopy != moreStats)
+    {
+        Aya::Tokens::apiToken.addFlagSafe(Aya::kPingItem); // can be changed to addFlagFast later.
+    }
+
+#endif
+    return true;
+}
+
+shared_ptr<DeserializedItem> Replicator::PingItem::read(Replicator& replicator, RakNet::BitStream& inBitstream)
+{
+    shared_ptr<DeserializedPingItem> deserializedData(new DeserializedPingItem());
+
+    int start = inBitstream.GetReadOffset();
+
+    inBitstream >> deserializedData->pingBack;
+    inBitstream >> deserializedData->time;
+    inBitstream >> deserializedData->sendStats;
+    if (replicator.canUseProtocolVersion(34))
+    {
+        inBitstream >> deserializedData->extraStats;
+#ifdef AYA_SERVER
+        if (deserializedData->time & 0x20) // change things up
+        {
+            deserializedData->extraStats = ~deserializedData->extraStats;
+        }
+#endif
+    }
+
+    if (replicator.settings().trackDataTypes)
+    {
+        replicator.replicatorStats.incrementPacketsReceived(ReplicatorStats::PACKET_TYPE_Ping);
+        replicator.replicatorStats.samplePacketsReceived(ReplicatorStats::PACKET_TYPE_Ping, (inBitstream.GetReadOffset() - start) / 8);
+    }
+
+    return deserializedData;
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.PingItem.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "Item.hpp"
+#include "Replicator.hpp"
+
+#include "RakNet/RakNetTime.hpp"
+
+namespace Aya
+{
+namespace Network
+{
+
+class DeserializedPingItem : public DeserializedItem
+{
+public:
+    bool pingBack;
+    RakNet::Time time;
+    unsigned int sendStats;
+    unsigned int extraStats;
+
+    DeserializedPingItem();
+    ~DeserializedPingItem() {}
+
+    /*implement*/ void process(Replicator& replicator);
+};
+
+
+class Replicator::PingItem : public PooledItem
+{
+    RakNet::Time time;
+    unsigned int extraStats;
+
+public:
+    PingItem(Replicator* replicator, RakNet::Time time, unsigned int extraStats);
+
+    /*implement*/ virtual bool write(RakNet::BitStream& bitStream);
+    static shared_ptr<DeserializedItem> read(Replicator& replicator, RakNet::BitStream& bitStream);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.PingJob.hpp

@@ -0,0 +1,67 @@
+#pragma once
+
+#include "Replicator.hpp"
+#include "DataModel/DataModel.hpp"
+
+
+
+
+
+namespace Aya
+{
+namespace Network
+{
+
+// Regularly sends pings through the data pipe to measure total data round-trip time
+class Replicator::PingJob : public ReplicatorJob
+{
+public:
+    static const int desiredPingHz = 2;
+    static const int maxPingsPerStep = 5;
+    PingJob(Replicator& replicator)
+        : ReplicatorJob("Replicator DataPing", replicator, DataModelJob::DataIn)
+    {
+        AYAASSERT(getArbiter());
+
+        cyclicExecutive = true;
+    }
+
+private:
+    virtual Time::Interval sleepTime(const Stats& stats)
+    {
+        return computeStandardSleepTime(stats, 2);
+    }
+
+    virtual Error error(const Stats& stats)
+    {
+        return computeStandardErrorCyclicExecutiveSleeping(stats, 2);
+    }
+
+    virtual TaskScheduler::StepResult stepDataModelJob(const Stats& stats)
+    {
+        if (replicator)
+        {
+            DataModel::scoped_write_request request(replicator.get());
+            if (TaskScheduler::singleton().isCyclicExecutive() && cyclicExecutive)
+            {
+                float pingsToDo = updateStepsRequiredForCyclicExecutive(
+                    stats.timespanSinceLastStep.seconds(), (float)desiredPingHz, (float)maxPingsPerStep, (float)maxPingsPerStep);
+                for (int i = 0; i < (int)pingsToDo; i++)
+                {
+                    replicator->sendDataPing();
+                }
+            }
+            else
+            {
+                replicator->sendDataPing();
+            }
+            return TaskScheduler::Stepped;
+        }
+
+        return TaskScheduler::Done;
+    }
+};
+
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.ProcessPacketsJob.hpp

@@ -0,0 +1,157 @@
+#pragma once
+
+#include "Replicator.hpp"
+#include "Debug.hpp"
+
+#include "DataModel/DataModel.hpp"
+
+#include "Players.hpp"
+
+#include "Utility/MemoryStats.hpp"
+
+#include "Replicator.StreamJob.hpp"
+LOGGROUP(NetworkStepsMultipliers)
+LOGGROUP(MaxNetworkReadTimeInCS)
+DYNAMIC_FASTINTVARIABLE(MaxWaitTimeBeforeForcePacketProcessMS, 0)
+DYNAMIC_FASTINTVARIABLE(MaxProcessPacketsStepsPerCyclic, 5)
+DYNAMIC_FASTINTVARIABLE(MaxProcessPacketsStepsAccumulated, 15);
+DYNAMIC_FASTINTVARIABLE(MaxProcessPacketsJobScaling, 10);
+
+namespace Aya
+{
+namespace Network
+{
+
+class Replicator::ProcessPacketsJob : public ReplicatorJob
+{
+    Aya::atomic<int> isAwake;
+    DataModel* dm;
+
+public:
+    ProcessPacketsJob(Replicator& replicator)
+        : ReplicatorJob("Replicator ProcessPackets", replicator, DataModelJob::DataIn)
+    {
+        AYAASSERT(getArbiter());
+        isAwake = 1;
+        dm = DataModel::get(&replicator);
+        cyclicExecutive = true;
+        cyclicPriority = CyclicExecutiveJobPriority_Network_ProcessIncoming;
+    }
+
+    // wake and sleep are used for testing and debugging.
+    void wake()
+    {
+        if (!isAwake.compare_and_swap(1, 0))
+            Aya::TaskScheduler::singleton().reschedule(shared_from_this());
+    }
+    void sleep()
+    {
+        isAwake = false;
+    }
+
+private:
+    virtual Time::Interval sleepTime(const Stats& stats)
+    {
+        if (!isAwake)
+            return Aya::Time::Interval::max();
+
+        if (replicator)
+        {
+            return replicator->incomingPacketsCount() > 0 ? computeStandardSleepTime(stats, replicator->settings().getReceiveRate())
+                                                          : Time::Interval(0.1);
+        }
+        return Time::Interval(0.1);
+    }
+
+    virtual Error error(const Stats& stats)
+    {
+        if (!isAwake)
+            return Job::Error();
+
+        Error result;
+        result.error = 0;
+        if (replicator)
+        {
+            if (dm->getIsShuttingDown())
+                return Job::Error();
+
+            if (TaskScheduler::singleton().isCyclicExecutive() && cyclicExecutive)
+            {
+                if (replicator->incomingPacketsCount())
+                {
+                    result.error = 1.0f;
+                }
+                else
+                {
+                    result.error = 0.0f;
+                }
+            }
+            else if (replicator->settings().isQueueErrorComputed)
+            {
+                double waitTime = replicator->incomingPacketsCountHeadWaitTimeSec(stats.timeNow);
+                result.error = waitTime * replicator->settings().getReceiveRate();
+            }
+            else
+            {
+                const int count = replicator->incomingPacketsCount();
+                result.error = count * stats.timespanSinceLastStep.seconds() * replicator->settings().getReceiveRate();
+            }
+        }
+        return result;
+    }
+
+    virtual TaskScheduler::StepResult stepDataModelJob(const Stats& stats)
+    {
+        if (!isAwake)
+            return TaskScheduler::Stepped;
+
+        if (replicator)
+        {
+            FASTLOG(FLog::NetworkStepsMultipliers, "ProcessPacketsJob running");
+            DataModel::scoped_write_request request(replicator.get());
+
+            Time t;
+            if (TaskScheduler::singleton().isCyclicExecutive() && cyclicExecutive)
+            {
+                float stepLength =
+                    updateStepsRequiredForCyclicExecutive(stats.timespanSinceLastStep.seconds(), replicator->settings().getReceiveRate(),
+                        (float)DFInt::MaxProcessPacketsStepsPerCyclic, (float)DFInt::MaxProcessPacketsStepsAccumulated);
+
+                int processTimeMultiplier = std::min<int>(5, replicator->incomingPacketsCount());
+                stepLength = std::max<float>(1.0, stepLength);
+                float processTimeScale = std::min<float>((float)DFInt::MaxProcessPacketsJobScaling, stepLength * processTimeMultiplier);
+                t = Time::now<Time::Fast>() + Time::Interval((processTimeScale) * (0.5 / replicator->settings().getReceiveRate()));
+            }
+            else
+            {
+                t = Time::now<Time::Fast>() + Time::Interval(0.5 / replicator->settings().getReceiveRate());
+            }
+
+            FASTLOG2(FLog::NetworkStepsMultipliers, "ProcessPacketsJob: StepOnce, processAllPackets: %d, packets in Queue: %d",
+                (int)replicator->processAllPacketsPerStep, replicator->incomingPackets.size());
+            Time maxTime = (Time::now<Time::Fast>() + Time::Interval(0.01 * FLog::MaxNetworkReadTimeInCS)); // convert centisecond to second
+            while (!(dm->getIsShuttingDown()) && (replicator->processNextIncomingPacket()))
+            {
+                if (replicator->processAllPacketsPerStep)
+                {
+                    // break if we exceed our maximum time here
+                    if (FLog::MaxNetworkReadTimeInCS > 0)
+                        if (Time::now<Time::Fast>() > maxTime)
+                            break;
+                }
+                /// Process ALL Packets if there is anything thats been there for longer than 200ms
+                else if (Time::now<Time::Fast>() > t && !(replicator->incomingPacketsCountHeadWaitTimeSec(stats.timeNow) >
+                                                            (float)DFInt::MaxWaitTimeBeforeForcePacketProcessMS / 1000.0f))
+                    break;
+            }
+
+            // post processing
+            replicator->postProcessPacket();
+            return TaskScheduler::Stepped;
+        }
+        return TaskScheduler::Done;
+    }
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.ReferencePropertyChangedItem.cpp

@@ -0,0 +1,40 @@
+#include "Replicator.ReferencePropertyChangedItem.hpp"
+
+#include "Item.hpp"
+#include "Replicator.hpp"
+
+#include "Reflection/Property.hpp"
+
+#include "RakNet/BitStream.hpp"
+
+namespace Aya
+{
+namespace Network
+{
+
+Replicator::ReferencePropertyChangedItem::ReferencePropertyChangedItem(
+    Replicator* replicator, const shared_ptr<const Instance>& instance, const Reflection::RefPropertyDescriptor& desc)
+    : PooledItem(*replicator)
+    , instance(instance)
+    , desc(desc)
+{
+
+    Instance* refInstance = DescribedBase::fastDynamicCast<Instance>(desc.getRefValue(instance.get()));
+    if (refInstance != NULL)
+    {
+        refInstance->getGuid().extract(newValueGuid);
+    }
+    else
+    {
+        newValueGuid.scope.setNull();
+    }
+}
+
+bool Replicator::ReferencePropertyChangedItem::write(RakNet::BitStream& bitStream)
+{
+    replicator.writeChangedRefProperty(instance.get(), desc, newValueGuid, bitStream);
+    return true;
+}
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.ReferencePropertyChangedItem.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "Item.hpp"
+#include "Replicator.hpp"
+
+namespace Aya
+{
+
+class Instance;
+namespace Reflection
+{
+class RefPropertyDescriptor;
+}
+
+namespace Network
+{
+
+class Replicator::ReferencePropertyChangedItem : public PooledItem
+{
+    const shared_ptr<const Instance> instance;
+    const Reflection::RefPropertyDescriptor& desc;
+    bool newValueIsNull;
+    Guid::Data newValueGuid;
+
+public:
+    ReferencePropertyChangedItem(Replicator* replicator, const shared_ptr<const Instance>& instance, const Reflection::RefPropertyDescriptor& desc);
+
+    /*implement*/ virtual bool write(RakNet::BitStream& bitStream);
+};
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.SendDataJob.hpp

@@ -0,0 +1,169 @@
+#pragma once
+
+#include "Replicator.hpp"
+#include "Players.hpp"
+#include "Utility/RbxStringTable.hpp"
+
+#include "Utility/xxhash.hpp"
+
+#include "Utility/ObscureValue.hpp"
+
+#include "DataModel/Stats.hpp"
+
+#include "DataModel/HackDefines.hpp"
+
+
+
+
+LOGGROUP(NetworkStepsMultipliers)
+DYNAMIC_FASTINTVARIABLE(MaxDataStepsPerCyclic, 5)
+DYNAMIC_FASTINTVARIABLE(MaxDataStepsAccumulated, 15)
+DYNAMIC_FASTINTVARIABLE(MaxClusterSendStepsPerCyclic, 5)
+DYNAMIC_FASTINTVARIABLE(MaxClusterSendStepsAccumulated, 15)
+DYNAMIC_FASTINTVARIABLE(MaxDataOutJobScaling, 10)
+
+namespace Aya
+{
+namespace Network
+{
+
+class Replicator::SendDataJob : public ReplicatorJob
+{
+    const ObscureValue<float> dataSendRate;
+    const PacketPriority packetPriority;
+    unsigned int countdownToXxhashCheck;
+
+public:
+    SendDataJob(Replicator& replicator)
+        : ReplicatorJob("Replicator SendData", replicator, DataModelJob::DataOut)
+        , dataSendRate(replicator.settings().getDataSendRate())
+        , packetPriority(replicator.settings().getDataSendPriority())
+        , countdownToXxhashCheck(0)
+    {
+        cyclicExecutive = true;
+        cyclicPriority = CyclicExecutiveJobPriority_Network_ProcessOutgoing;
+    }
+
+private:
+    virtual Time::Interval sleepTime(const Stats& stats)
+    {
+        // TODO: If Replicator::pendingItems is empty then sleep
+        return computeStandardSleepTime(stats, dataSendRate);
+    }
+
+    virtual Error error(const Stats& stats);
+    virtual TaskScheduler::StepResult stepDataModelJob(const Stats& stats)
+    {
+        if (replicator)
+        {
+            try
+            {
+                if (TaskScheduler::singleton().isCyclicExecutive() && cyclicExecutive)
+                {
+                    float stepsToDo = updateStepsRequiredForCyclicExecutive(stats.timespanSinceLastStep.seconds(),
+                        replicator->settings().getDataSendRate(), (float)DFInt::MaxDataStepsPerCyclic, (float)DFInt::MaxDataStepsAccumulated);
+
+                    stepsToDo = std::max<float>(1.0, stepsToDo);
+                    stepsToDo *= std::min<int>(10, replicator->highPriorityPendingItems.size() + replicator->pendingItems.size() + 1) / 2.0;
+                    float truncatedStepsToDo = std::min<float>((float)DFInt::MaxDataOutJobScaling, stepsToDo);
+                    for (int i = 0; i < (int)truncatedStepsToDo; i++)
+                    {
+                        replicator->dataOutStep();
+                    }
+                }
+                else
+                {
+                    replicator->dataOutStep();
+                }
+            }
+            catch (Aya::base_exception& e)
+            {
+                const char* what = e.what();
+                Aya::StandardOut::singleton()->printf(Aya::MESSAGE_ERROR, "SendData: %s", what ? what : "empty error string");
+                return TaskScheduler::Done;
+            }
+            return TaskScheduler::Stepped;
+        }
+        
+        return TaskScheduler::Done;
+    }
+};
+
+class Replicator::SendClusterJob : public ReplicatorJob
+{
+    ObscureValue<float> dataSendRate;
+    PacketPriority packetPriority;
+
+public:
+    SendClusterJob(Replicator& replicator)
+        : ReplicatorJob("Replicator SendCluster", replicator, DataModelJob::DataOut)
+        , dataSendRate(replicator.settings().getDataSendRate())
+        , packetPriority(replicator.settings().getDataSendPriority())
+    {
+        cyclicExecutive = true;
+        cyclicPriority = CyclicExecutiveJobPriority_Network_ProcessOutgoing;
+    }
+
+private:
+    virtual Time::Interval sleepTime(const Stats& stats)
+    {
+        // TODO: If Replicator::pendingItems is empty then sleep
+        return computeStandardSleepTime(stats, dataSendRate);
+    }
+
+    virtual Error error(const Stats& stats);
+
+    virtual TaskScheduler::StepResult stepDataModelJob(const Stats& stats)
+    {
+        if (replicator)
+        {
+            try
+            {
+                if (TaskScheduler::singleton().isCyclicExecutive() && cyclicExecutive)
+                {
+                    double stepsRequired =
+                        updateStepsRequiredForCyclicExecutive(stats.timespanSinceLastStep.seconds(), replicator->settings().getDataSendRate(),
+                            (float)DFInt::MaxClusterSendStepsPerCyclic, (float)DFInt::MaxClusterSendStepsAccumulated);
+                    float multiplier = 1.0;
+                    if (int maxBytesSend = replicator->getAdjustedMtuSize())
+                    {
+                        // assuming there's just one cluster for now
+                        int deltas = replicator->getApproximateSizeOfPendingClusterDeltas();
+
+                        if (deltas > maxBytesSend)
+                        {
+                            multiplier *= deltas;
+                            multiplier /= maxBytesSend;
+                        }
+                    }
+                    if (replicator->getApproximateSizeOfPendingClusterDeltas() > 0.0f)
+                    {
+                        stepsRequired = std::max<float>(1.0, stepsRequired);
+                    }
+                    float sendDataSteps = std::min<float>((float)DFInt::MaxDataOutJobScaling, stepsRequired * multiplier);
+
+                    for (int i = 0; i < (int)sendDataSteps; i++)
+                    {
+                        replicator->clusterOutStep();
+                    }
+                }
+                else
+                {
+                    replicator->clusterOutStep();
+                }
+            }
+            catch (Aya::base_exception& e)
+            {
+                const char* what = e.what();
+                Aya::StandardOut::singleton()->printf(Aya::MESSAGE_ERROR, "SendCluster: %s", what ? what : "empty error string");
+                return TaskScheduler::Done;
+            }
+            return TaskScheduler::Stepped;
+        }
+        return TaskScheduler::Done;
+    }
+};
+
+
+} // namespace Network
+} // namespace Aya

engine/network/src/Replicator.StatsItem.cpp

@@ -0,0 +1,159 @@
+#include "DataModel/DataModel.hpp"
+// d9mz - member access into incomplete type 'DataModel'
+#include "Replicator.StatsItem.hpp"
+#include "Script/ScriptContext.hpp"
+#include "ClientReplicator.hpp"
+
+using namespace Aya;
+using namespace Network;
+
+struct JobInfo
+{
+    double dutyCycle;
+    double stepsPerSecond;
+    double stepTime;
+    int count;
+
+    JobInfo()
+        : dutyCycle(0)
+        , stepTime(0)
+        , stepsPerSecond(0)
+        , count(0)
+    {
+    }
+};
+
+void DeserializedStatsItem::process(Replicator& replicator)
+{
+    ClientReplicator* rep = aya_static_cast<ClientReplicator*>(&replicator);
+    rep->statsReceivedSignal(stats);
+}
+
+void writeTaskSchedulerStats(DataModel* dm, RakNet::BitStream& bitStream)
+{
+    std::vector<boost::shared_ptr<const TaskScheduler::Job>> jobs;
+    TaskScheduler::singleton().getJobsInfo(jobs);
+
+    boost::unordered_map<std::string, JobInfo> aggregatedJobInfos;
+
+    // BitStream:
+    // end flag (bool)
+    // job name (string)
+    // duty cycle (float)
+    // steps per second (float)
+    // step time (float)
+
+    for (std::vector<boost::shared_ptr<const TaskScheduler::Job>>::iterator iter = jobs.begin(); iter != jobs.end(); iter++)
+    {
+        shared_ptr<const TaskScheduler::Job> job = *iter;
+        if (job->getArbiter().get() != dm)
+            continue;
+
+        const std::string& name = job->name;
+
+        JobInfo& info = aggregatedJobInfos[name];
+        info.dutyCycle += job->averageDutyCycle();
+        info.stepsPerSecond += job->averageStepsPerSecond();
+        info.stepTime += job->averageStepTime();
+        info.count++;
+    }
+
+    // calculate replicator job averages and write to bit stream
+    for (boost::unordered_map<std::string, JobInfo>::iterator iter = aggregatedJobInfos.begin(); iter != aggregatedJobInfos.end(); iter++)
+    {
+        bitStream << false; // more items
+
+        bitStream << iter->first;
+        bitStream << (float)(iter->second.dutyCycle / iter->second.count);
+        bitStream << (float)(iter->second.stepsPerSecond / iter->second.count);
+        bitStream << (float)(iter->second.stepTime / iter->second.count);
+    }
+    bitStream << true; // done with jobs
+}
+
+void writeScriptStats(DataModel* dm, RakNet::BitStream& bitStream)
+{
+    ScriptContext* sc = dm->find<ScriptContext>();
+    std::string scriptsJson;
+    sc->setCollectScriptStats(true);
+
+    std::vector<ScriptContext::ScriptStat> scriptStats;
+    sc->getScriptStatsTyped(scriptStats);
+
+    for (std::vector<ScriptContext::ScriptStat>::iterator iter = scriptStats.begin(); iter != scriptStats.end(); iter++)
+    {
+        bitStream << false; // more stats info
+
+        bitStream << iter->name;
+        bitStream << (float)iter->activity;
+        bitStream << (int)iter->invocationCount;
+    }
+    bitStream << true; // done with stats info
+}
+
+bool Replicator::StatsItem::write(RakNet::BitStream& bitStream)
+{
+    writeItemType(bitStream, Item::ItemTypeStats);
+
+    float totalPhysicsFPS = 0;
+    float dataPacketsTotal = 0;
+    float dataSizeTotal = 0;
+    float physicsPacketsTotal = 0;
+    float physicsSizeTotal = 0;
+    float totalPing = 0;
+    float totalNewDataItems = 0;
+    float totalSentDataItems = 0;
+
+    Instances::const_iterator end = replicator.getParent()->getChildren()->end();
+    for (Instances::const_iterator iter = replicator.getParent()->getChildren()->begin(); iter != end; ++iter)
+    {
+        // accumulate stats for all replicator
+        if (Replicator* r = Instance::fastDynamicCast<Replicator>(iter->get()))
+        {
+            const ReplicatorStats& s = r->stats();
+            totalPhysicsFPS += r->stats().physicsSenderStats.physicsPacketsSent.rate();
+            dataPacketsTotal += r->stats().dataPacketsSent.rate();
+            dataSizeTotal += r->stats().dataPacketsSentSize.value();
+            physicsPacketsTotal += r->stats().physicsSenderStats.physicsPacketsSent.rate();
+            physicsSizeTotal += r->stats().physicsSenderStats.physicsPacketsSentSize.value();
+            totalPing += r->stats().dataPing.value();
+            totalNewDataItems += (float)s.dataNewItemsPerSec.getCount();
+            totalSentDataItems += (float)s.dataItemsSentPerSec.getCount();
+        }
+    }
+
+    int numChildren = replicator.getParent()->numChildren();
+
+    switch (version)
+    {
+    case 2:
+    {
+        DataModel* dm = DataModel::get(&replicator);
+        writeTaskSchedulerStats(dm, bitStream);
+        writeScriptStats(dm, bitStream);
+    }
+        // fall through
+    case 1:
+    {
+        bitStream << (totalPing / numChildren);                        // average ping
+        bitStream << (totalPhysicsFPS / numChildren);                  // average physics sender fps
+        bitStream << dataPacketsTotal * dataSizeTotal / 1000.0f;       // total data KB per sec
+        bitStream << physicsPacketsTotal * physicsSizeTotal / 1000.0f; // total physics KB per sec
+        bitStream << totalSentDataItems / totalNewDataItems;           // data throughput
+    }
+    default:
+        break;
+    }
+
+    return true;
+}
+
+shared_ptr<DeserializedItem> Replicator::StatsItem::read(Replicator& replicator, RakNet::BitStream& bitStream)
+{
+    shared_ptr<DeserializedStatsItem> deserializedData(new DeserializedStatsItem());
+
+    ClientReplicator* rep = aya_static_cast<ClientReplicator*>(&replicator);
+    deserializedData->stats = rep->readStats(bitStream);
+
+    return deserializedData;
+}