From a3750c102f19e867f3b6815cf068e902e63db7fb Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Fri, 20 Dec 2019 16:11:21 -0800 Subject: [PATCH] =?UTF-8?q?send=20shutdown=20event=20when=20dll=20is=20unl?= =?UTF-8?q?oaded=20and=20EvaluationStop,=20SessionC=E2=80=A6=20(#2704)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * send shutdown event when dll is unloaded and EvaluationStop, SessionCreationStart Events. * Add EvalutationStart Event * add comment --- onnxruntime/core/platform/telemetry.cc | 9 +++++++ onnxruntime/core/platform/telemetry.h | 6 +++++ .../core/platform/windows/telemetry.cc | 24 +++++++++++++++++++ onnxruntime/core/platform/windows/telemetry.h | 6 +++++ onnxruntime/core/session/inference_session.cc | 19 ++++++++++++--- onnxruntime/core/session/inference_session.h | 5 ++++ winml/dll/module.cpp | 1 + winml/lib/Common/inc/WinMLTelemetryHelper.h | 2 ++ winml/lib/Telemetry/WinMLTelemetryHelper.cpp | 9 +++++++ 9 files changed, 78 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/platform/telemetry.cc b/onnxruntime/core/platform/telemetry.cc index da3d21ae50..4fccbe6943 100644 --- a/onnxruntime/core/platform/telemetry.cc +++ b/onnxruntime/core/platform/telemetry.cc @@ -23,6 +23,15 @@ void Telemetry::DisableTelemetryEvents() const { void Telemetry::LogProcessInfo() const { } +void Telemetry::LogSessionCreationStart() const { +} + +void Telemetry::LogEvaluationStop() const { +} + +void Telemetry::LogEvaluationStart() const { +} + void Telemetry::LogSessionCreation(uint32_t session_id, int64_t ir_version, const std::string& model_producer_name, const std::string& model_producer_version, const std::string& model_domain, const std::unordered_map& domain_to_version_map, diff --git a/onnxruntime/core/platform/telemetry.h b/onnxruntime/core/platform/telemetry.h index f7ffd6ed7d..81e80c26fa 100644 --- a/onnxruntime/core/platform/telemetry.h +++ b/onnxruntime/core/platform/telemetry.h @@ -38,6 +38,12 @@ class Telemetry { virtual void LogProcessInfo() const; + virtual void LogSessionCreationStart() const; + + virtual void LogEvaluationStop() const; + + virtual void LogEvaluationStart() const; + virtual void LogSessionCreation(uint32_t session_id, int64_t ir_version, const std::string& model_producer_name, const std::string& model_producer_version, const std::string& model_domain, const std::unordered_map& domain_to_version_map, diff --git a/onnxruntime/core/platform/windows/telemetry.cc b/onnxruntime/core/platform/windows/telemetry.cc index b347202e8a..9952f604d8 100644 --- a/onnxruntime/core/platform/windows/telemetry.cc +++ b/onnxruntime/core/platform/windows/telemetry.cc @@ -87,6 +87,30 @@ void WindowsTelemetry::LogProcessInfo() const { process_info_logged = true; } +void WindowsTelemetry::LogSessionCreationStart() const { + TraceLoggingWrite(telemetry_provider_handle, + "SessionCreationStart", + TraceLoggingBool(true, "UTCReplace_AppSessionGuid"), + TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage), + TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES)); +} + +void WindowsTelemetry::LogEvaluationStop() const { + TraceLoggingWrite(telemetry_provider_handle, + "EvaluationStop", + TraceLoggingBool(true, "UTCReplace_AppSessionGuid"), + TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage), + TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES)); +} + +void WindowsTelemetry::LogEvaluationStart() const { + TraceLoggingWrite(telemetry_provider_handle, + "EvaluationStart", + TraceLoggingBool(true, "UTCReplace_AppSessionGuid"), + TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage), + TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES)); +} + void WindowsTelemetry::LogSessionCreation(uint32_t session_id, int64_t ir_version, const std::string& model_producer_name, const std::string& model_producer_version, const std::string& model_domain, const std::unordered_map& domain_to_version_map, diff --git a/onnxruntime/core/platform/windows/telemetry.h b/onnxruntime/core/platform/windows/telemetry.h index cf786a4aa3..ed2ae39737 100644 --- a/onnxruntime/core/platform/windows/telemetry.h +++ b/onnxruntime/core/platform/windows/telemetry.h @@ -23,6 +23,12 @@ class WindowsTelemetry : public Telemetry { void LogProcessInfo() const override; + void LogSessionCreationStart() const override; + + void LogEvaluationStop() const override; + + void LogEvaluationStart() const override; + void LogSessionCreation(uint32_t session_id, int64_t ir_version, const std::string& model_producer_name, const std::string& model_producer_version, const std::string& model_domain, const std::unordered_map& domain_to_version_map, diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index b0e4347b86..e4d6d23312 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -791,6 +791,8 @@ common::Status InferenceSession::Initialize() { try { LOGS(*session_logger_, INFO) << "Initializing session."; std::lock_guard l(session_mutex_); + const Env& env = Env::Default(); + env.GetTelemetryProvider().LogSessionCreationStart(); if (!is_model_loaded_) { LOGS(*session_logger_, ERROR) << "Model was not loaded"; return common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."); @@ -870,7 +872,6 @@ common::Status InferenceSession::Initialize() { is_inited_ = true; // and log telemetry - const Env& env = Env::Default(); bool model_use_fp16 = ModelUseFP16(model_->ToProto()); env.GetTelemetryProvider().LogSessionCreation(session_id_, model_->IrVersion(), model_->ProducerName(), model_->ProducerVersion(), model_->Domain(), model_->MainGraph().DomainToVersionMap(), model_->MainGraph().Name(), @@ -1053,13 +1054,21 @@ Status InferenceSession::Run(const RunOptions& run_options, const std::vector kDurationBetweenSendingEvaluationStart) { + env.GetTelemetryProvider().LogEvaluationStart(); + // reset counters + time_sent_last_evalutation_start_ = std::chrono::high_resolution_clock::now(); + isEvaluationStart = true; + } + ORT_RETURN_IF_ERROR_SESSIONID_(ValidateInputs(feed_names, feeds)); ORT_RETURN_IF_ERROR_SESSIONID_(ValidateOutputs(output_names, p_fetches)); @@ -1111,7 +1120,6 @@ Status InferenceSession::Run(const RunOptions& run_options, const std::vector kDurationBetweenSending) { // send the telemetry - const Env& env = Env::Default(); env.GetTelemetryProvider().LogRuntimePerf(session_id_, total_runs_since_last_, total_run_duration_since_last_); // reset counters time_sent_last_ = std::chrono::high_resolution_clock::now(); @@ -1119,6 +1127,11 @@ Status InferenceSession::Run(const RunOptions& run_options, const std::vector session_activity; diff --git a/winml/dll/module.cpp b/winml/dll/module.cpp index e4910209ad..531521edc8 100644 --- a/winml/dll/module.cpp +++ b/winml/dll/module.cpp @@ -33,6 +33,7 @@ extern "C" BOOL WINAPI DllMain(_In_ HINSTANCE hInstance, DWORD dwReason, _In_ vo wil::SetResultTelemetryFallback(&OnErrorReported); break; case DLL_PROCESS_DETACH: + telemetry_helper.LogWinMLShutDown(); // Unregister Trace Logging Provider feeding telemetry telemetry_helper.UnRegister(); diff --git a/winml/lib/Common/inc/WinMLTelemetryHelper.h b/winml/lib/Common/inc/WinMLTelemetryHelper.h index a88d564538..e2cb2ca851 100644 --- a/winml/lib/Common/inc/WinMLTelemetryHelper.h +++ b/winml/lib/Common/inc/WinMLTelemetryHelper.h @@ -79,6 +79,8 @@ class WinMLTelemetryHelper { void UnRegister() { TraceLoggingUnregister(provider_); } + + void LogWinMLShutDown(); void LogRuntimeError(HRESULT hr, std::string message, PCSTR file, PCSTR function, int line); void LogRuntimeError(HRESULT hr, PCSTR message, PCSTR file, PCSTR function, int line); void LogRegisterOperatorKernel( diff --git a/winml/lib/Telemetry/WinMLTelemetryHelper.cpp b/winml/lib/Telemetry/WinMLTelemetryHelper.cpp index 7154401d15..bdfe5ee101 100644 --- a/winml/lib/Telemetry/WinMLTelemetryHelper.cpp +++ b/winml/lib/Telemetry/WinMLTelemetryHelper.cpp @@ -16,6 +16,15 @@ WinMLTelemetryHelper::WinMLTelemetryHelper() WinMLTelemetryHelper::~WinMLTelemetryHelper() { } +void WinMLTelemetryHelper::LogWinMLShutDown() { + WinMLTraceLoggingWrite( + provider_, + "WinMLShutDown", + TraceLoggingKeyword(WINML_PROVIDER_KEYWORD_DEFAULT), + TelemetryPrivacyDataTag(PDT_ProductAndServicePerformance), + TraceLoggingString("windows.ai.machinelearning.dll is unloaded", "message"), + TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES)); +} void WinMLTelemetryHelper::LogRuntimeError(HRESULT hr, PCSTR message, PCSTR file, PCSTR function, int line) { if (!telemetry_enabled_)