StructuredLogSettingsUtils.h#

// Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved.
//
// NVIDIA CORPORATION and its licensors retain all intellectual property
// and proprietary rights in and to this software, related documentation
// and any modifications thereto. Any use, reproduction, disclosure or
// distribution of this software and related documentation without an express
// license agreement from NVIDIA CORPORATION is strictly prohibited.
//
#pragma once

#include "IStructuredLog.h"
#include "IStructuredLogExtraFields.h"
#include "IStructuredLogSettings.h"
#include "IStructuredLogSettings2.h"
#include "IStructuredLogSettings3.h"
#include "IStructuredLogFromILog.h"
#include "IStructuredLogFromILog2.h"

#include "../../carb/extras/StringSafe.h"
#include "../../carb/settings/ISettings.h"
#include "../../carb/settings/SettingsUtils.h"
#include "../core/Omni.h"
#include "../extras/PrivacySettings.h"
#include "../extras/OmniConfig.h"
#include "../log/ILog.h"
#include "../platforminfo/IOsInfo2.h"

namespace omni
{
namespace structuredlog
{

constexpr int64_t kDefaultLogSizeLimit = 50ll * 1024ll * 1024ll;

constexpr int64_t kMinLogSizeLimit = 256ll * 1024ll;

constexpr size_t kDefaultLogRetentionCount = 3;

constexpr size_t kMinLogRetentionCount = 1;

constexpr size_t kDefaultEventQueueSize = 2 * 1024 * 1024;

constexpr size_t kMinEventQueueSize = 512 * 1024;

constexpr size_t kMaxEventQueueSize = 1024 * 1024 * 1024;

constexpr size_t kRecommendedLoggingEventQueueSize = 32 * 1024 * 1024;

constexpr IdMode kDefaultIdMode = IdMode::eFastSequential;

constexpr IdType kDefaultIdType = IdType::eUuid;

constexpr uint64_t kHeartbeatDisabled = 0;

constexpr uint64_t kDefaultHeartbeatPeriod = kHeartbeatDisabled;

constexpr bool kDefaultNeedLogHeaders = true;

constexpr bool kDefaultEmitPayloadOnlySettings = false;

constexpr bool kDefaultEmitCloudHeartbeat = false;

constexpr const char* kDefaultAnonymousUserIdMode = "random";

constexpr bool kDefaultEnableJsonStdout = false;

constexpr int64_t kDefaultQueueHighWaterMark = 80;
constexpr const char* kGlobalEnableSetting = "/structuredLog/enable";

constexpr const char* kLogDirectory = "/structuredLog/logDirectory";

constexpr const char* kDefaultLogNameSetting = "/structuredLog/defaultLogName";

constexpr const char* kPrivacyFileSetting = "/structuredLog/privacySettingsFile";

constexpr const char* kLogRetentionCountSetting = "/structuredLog/logRetentionCount";

constexpr const char* kLogSizeLimitSetting = "/structuredLog/logSizeLimit";

constexpr const char* kEventQueueSizeSetting = "/structuredLog/eventQueueSize";

constexpr const char* kEventIdModeSetting = "/structuredLog/eventIdMode";

constexpr const char* kEventIdTypeSetting = "/structuredLog/eventIdType";

constexpr const char* kEnableLogConsumerSetting = "/structuredLog/enableLogConsumer";

constexpr const char* kSchemasStateListSetting = "/structuredLog/state/schemas";

constexpr const char* kEventsStateListSetting = "/structuredLog/state/events";

constexpr const char* kSchemasStateArraySetting = "/structuredLog/schemaStates";

constexpr const char* kEventsStateArraySetting = "/structuredLog/eventStates";

constexpr const char* kHeartbeatPeriodSetting = "/structuredLog/heartbeatPeriod";

constexpr const char* kNeedLogHeadersSetting = "/structuredLog/needLogHeaders";

constexpr const char* kEmitPayloadOnlySettings = "/structuredLog/emitPayloadOnly";

constexpr const char* kEmitCloudHeartbeatSetting = "/structuredLog/emitCloudHeartbeat";

constexpr const char* kExtraFieldsSettingBranch = "/structuredLog/extraFields";

constexpr const char* kAnonymousUserIdModeSetting = "/structuredLog/anonymousUserIdMode";

constexpr const char* kEnableJsonStdoutSetting = "/structuredLog/enableJsonStdout";

constexpr const char* kQueueHighWaterMark = "/structuredLog/queueHighWaterMark";

constexpr const char* kSchemasThroughputLimitsSetting = "/structuredLog/limits/throughput";

constexpr const char* kSchemasThroughputPeriodSetting = "/structuredLog/limits/throughputPeriod";
inline bool setStructuredLogLoggingEnabled(bool enabled = true)
{
    omni::core::ObjectPtr<IStructuredLog> strucLog;
    omni::core::ObjectPtr<IStructuredLogFromILog> log;

    strucLog = omni::core::borrow(omniGetStructuredLogWithoutAcquire());

    if (strucLog.get() == nullptr)
        return false;

    log = strucLog.as<IStructuredLogFromILog>();

    if (log.get() == nullptr)
        return false;

    if (enabled)
        log->enableLogging();

    else
        log->disableLogging();

    return true;
}

inline void configureStructuredLogging(carb::settings::ISettings* settings)
{
    omni::core::ObjectPtr<IStructuredLog> strucLog;
    omni::core::ObjectPtr<IStructuredLogSettings> ts;
    omni::core::ObjectPtr<IStructuredLogSettings2> ts2;
    omni::core::ObjectPtr<IStructuredLogSettings3> ts3;
    omni::core::ObjectPtr<IStructuredLogFromILog> log;
    omni::core::ObjectPtr<IStructuredLogExtraFields> extraFields;
    carb::cpp::optional<std::string> value;
    int64_t count;
    size_t queueSize;
    IdMode idMode = kDefaultIdMode;
    IdType idType = kDefaultIdType;

    if (settings == nullptr)
        return;

    // ****** set appropriate defaults for each setting ******
    settings->setDefaultBool(kGlobalEnableSetting, false);
    settings->setDefaultString(kLogDirectory, "");
    settings->setDefaultString(kDefaultLogNameSetting, "");
    settings->setDefaultInt64(kLogRetentionCountSetting, kDefaultLogRetentionCount);
    settings->setDefaultInt64(kLogSizeLimitSetting, kDefaultLogSizeLimit / 1048576);
    settings->setDefaultInt64(kEventQueueSizeSetting, kDefaultEventQueueSize / 1024);
    settings->setDefaultString(kEventIdModeSetting, "fast-sequential");
    settings->setDefaultString(kEventIdTypeSetting, "UUID");
    settings->setDefaultBool(kEnableLogConsumerSetting, false);
    settings->setDefaultInt64(kHeartbeatPeriodSetting, kDefaultHeartbeatPeriod);
    settings->setDefaultBool(kNeedLogHeadersSetting, kDefaultNeedLogHeaders);
    settings->setDefaultBool(kEmitPayloadOnlySettings, kDefaultEmitPayloadOnlySettings);
    settings->setDefaultBool(kEmitCloudHeartbeatSetting, kDefaultEmitCloudHeartbeat);
    settings->setDefaultString(kAnonymousUserIdModeSetting, kDefaultAnonymousUserIdMode);

    if (settings->getAsBool("/app/coreapi/alpha/enabled"))
        settings->setDefaultBool(kEnableJsonStdoutSetting, true);

    else
        settings->setDefaultBool(kEnableJsonStdoutSetting, kDefaultEnableJsonStdout);

    // ****** grab the structured log settings object so the config can be set ******
    strucLog = omni::core::borrow(omniGetStructuredLogWithoutAcquire());

    if (strucLog.get() == nullptr)
        return;

    ts = strucLog.as<IStructuredLogSettings>();

    if (ts.get() == nullptr)
        return;

    // ****** retrieve the settings and make them active ******
    strucLog->setEnabled(omni::structuredlog::kBadEventId, omni::structuredlog::fEnableFlagAll,
                         settings->getAsBool(kGlobalEnableSetting));

    // set the default log name.
    value = carb::settings::getStringOpt(settings, kDefaultLogNameSetting);

    if (value && !value->empty())
        ts->setLogDefaultName(value->c_str());

    value = carb::settings::getStringOpt(settings, kLogDirectory);

    if (value && !value->empty())
    {
        ts->setLogOutputPath(value->c_str());
    }
    else
    {
        // This setting needs to be reloaded after ISerializer has been loaded, since it can read
        // omniverse.toml now in case there are overrides there.
        extras::OmniConfig config;
        ts->setLogOutputPath(config.getBaseLogsPath().c_str());
    }

    // set the log retention count.
    count = settings->getAsInt64(kLogRetentionCountSetting);
    ts->setLogRetentionCount((size_t)count);

    // set the log size limit.
    count = settings->getAsInt64(kLogSizeLimitSetting);
    ts->setLogSizeLimit(count * 1048576);

    // set the event queue size (given in KB in the setting).
    queueSize = (size_t)settings->getAsInt64(kEventQueueSizeSetting);
    ts->setEventQueueSize(queueSize * 1024);

    // set the event ID mode.
    value = carb::settings::getStringOpt(settings, kEventIdModeSetting);

    if (carb::extras::compareStringsNoCase(value->c_str(), "fast-sequential") == 0)
        idMode = IdMode::eFastSequential;

    else if (carb::extras::compareStringsNoCase(value->c_str(), "sequential") == 0)
        idMode = IdMode::eSequential;

    else if (carb::extras::compareStringsNoCase(value->c_str(), "random") == 0)
        idMode = IdMode::eRandom;

    else
        OMNI_LOG_WARN("unknown event ID mode '%s'.  Assuming 'fast-sequential'.", value->c_str());

    // set the event ID type.
    value = carb::settings::getStringOpt(settings, kEventIdTypeSetting);

    if (carb::extras::compareStringsNoCase(value->c_str(), "UUID") == 0)
        idType = IdType::eUuid;

    else if (carb::extras::compareStringsNoCase(value->c_str(), "uint64") == 0)
        idType = IdType::eUint64;

    else
        OMNI_LOG_WARN("unknown event ID type '%s'.  Assuming 'UUID'.", value->c_str());

    ts->setEventIdMode(idMode, idType);

    // load the privacy settings and set the user ID from it.
    ts->loadPrivacySettings();

    // load the enable states for each schema and event.
    ts->enableSchemasFromSettings();

    // retrieve and set the user ID if present.
    std::string uid = omni::extras::PrivacySettings::getUserIdString();

    if (!uid.empty())
        ts->setUserId(uid.c_str());

    else
    {
        carb::cpp::optional<std::string> mode = carb::settings::getStringOpt(settings, kAnonymousUserIdModeSetting);

        if (mode && carb::extras::compareStringsNoCase(mode->c_str(), "machine") == 0)
        {
            auto os2 = omni::core::createType<omni::platforminfo::IOsInfo2>();

            if (os2 != nullptr)
            {
                uint64_t id = os2->getMachineId();

                if (id != 0)
                {
                    ts->setUserId(std::to_string(id).c_str());
                }
            }
        }
    }

    // setup the structured log logger.  Note that if this is enabled on the command line
    // this will also implicitly increase the event queue's buffer size.
    log = strucLog.as<IStructuredLogFromILog>();

    if (log.get() == nullptr)
        return;

    if (settings->getAsBool(kEnableLogConsumerSetting))
    {
        log->enableLogging();
        ts->setEventQueueSize(CARB_MAX(kRecommendedLoggingEventQueueSize, queueSize * 1024));
    }

    // setup the default heartbeat event period.
    ts2 = strucLog.as<IStructuredLogSettings2>();

    if (ts2 != nullptr)
    {
        OutputFlags flags = fOutputFlagNone;

        count = settings->getAsInt64(kHeartbeatPeriodSetting);
        ts2->setHeartbeatPeriod((size_t)count);

        if (!settings->getAsBool(kNeedLogHeadersSetting))
            flags |= fOutputFlagSkipLogHeaders;

        if (settings->getAsBool(kEmitPayloadOnlySettings))
            flags |= fOutputFlagPayloadOnly;

        if (settings->getAsBool(kEmitCloudHeartbeatSetting))
            flags |= fOutputFlagEmitCloudHeartbeat;

        ts2->setOutputFlags(flags, 0);
    }

    extraFields = strucLog.as<IStructuredLogExtraFields>();

    if (extraFields != nullptr)
    {
        // walk through the extra fields branch of the settings registry and add an extra field
        // for each key/value pair in there.  In general we don't expect there to be more than
        // a couple extra fields present.
        carb::settings::ScopedRead readLock;
        if (settings->isAccessibleAs(carb::dictionary::ItemType::eDictionary, kExtraFieldsSettingBranch))
        {
            carb::dictionary::IDictionary* dictionary = carb::getCachedInterface<carb::dictionary::IDictionary>();
            const carb::dictionary::Item* branch = settings->getSettingsDictionary(kExtraFieldsSettingBranch);
            size_t childCount = dictionary->getItemChildCount(branch);

            for (size_t i = 0; i < childCount; i++)
            {
                const carb::dictionary::Item* item = dictionary->getItemChildByIndex(branch, i);

                if (item == nullptr)
                    continue;

                const char* key = dictionary->getItemName(item);
                const char* val = dictionary->getStringBuffer(item);

                if (key != nullptr && val != nullptr)
                {
                    extraFields->setValue(key, val, omni::structuredlog::fExtraFieldFlagNone);
                }
            }
        }
    }

    // settings requested that 'JSON logging' be used => enable the various settings related
    //   to setting this mode.  JSON logging mode outputs all log messages to stdout (unbuffered)
    //   formatted as JSON events.  It also silences the stdout/stderr output of the standard
    //   logger so that log messages are not doubled up in different formats on stdout.  Each
    //   log message will still be output to all other destinations (ie: debug console, log
    //   file(s), etc).
    if (log != nullptr && ts != nullptr && settings->getAsBool(kEnableJsonStdoutSetting))
    {
        auto log2 = strucLog.as<IStructuredLogFromILog2>();

        if (log2 != nullptr)
        {
            auto logging = carb::getFramework()->tryAcquireInterface<carb::logging::ILogging>();

            if (logging != nullptr)
            {
                auto standardLogger = logging->getDefaultLogger();

                if (standardLogger != nullptr)
                {
                    standardLogger->setStandardStreamOutput(false);
                }
            }

            // enable structured logging for at least the log consumer event since it is now
            // needed in order for log messages to be output.  The structured logging system
            // also needs to be globally enabled for this to take effect.
            strucLog->setEnabled(omni::structuredlog::kBadEventId, omni::structuredlog::fEnableFlagAll, true);
            strucLog->setEnabled(log->getLoggingEventId(), omni::structuredlog::fEnableFlagWholeSchema, true);
            settings->setBool(kGlobalEnableSetting, true);

            // enable the log consumer and set its destination to only stdout.
            log->enableLogging();
            log2->setLoggingEventFlags(
                omni::structuredlog::fEventFlagOutputToStdout | omni::structuredlog::fEventFlagSkipLog, 0);

            // make sure the log consumer respects the standard output stream log level.  This
            // log level further filters log messages that make it through to consumers but don't
            // then make it to the standard streams under the standard logger.
            log2->setLogLevel(
                carb::logging::stringToLevel(carb::settings::getString(settings, "/log/outputStreamLevel").c_str()));

            // increase the queue size.  If we're going to be serving all log messages through the
            // structured log queue, the queue's buffer needs to be large enough to ensure a
            // potentially large amount of concurrent message traffic can safely go through it.
            // With each message being anywhere from 200 bytes to upwards of 3KB (depending on
            // content), not that many messages will fit in the default buffer size of 2MB.
            // If a lot of message traffic suddenly occurs very quickly, the queue's buffer
            // could fill up and start either dropping messages or blocking threads until
            // space is available.
            size_t userSize = queueSize;

            userSize = CARB_MAX(userSize, omni::structuredlog::kRecommendedLoggingEventQueueSize);
            ts->setEventQueueSize(userSize);
        }
    }

    ts3 = strucLog.as<IStructuredLogSettings3>();

    if (ts3 != nullptr)
    {
        ts3->settingsLoadComplete();
    }

    strucLog.release();
}

} // namespace structuredlog
} // namespace omni