add nvidia power and temperature

Cargo.lock

@@ -216,14 +216,38 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "darling"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858"
+dependencies = [
+ "darling_core 0.10.2",
+ "darling_macro 0.10.2",
+]
+
 [[package]]
 name = "darling"
 version = "0.13.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c"
 dependencies = [
- "darling_core",
+ "darling_core 0.13.4",
- "darling_macro",
+ "darling_macro 0.13.4",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim 0.9.3",
+ "syn",
 ]
 
 [[package]]
@@ -236,7 +260,18 @@ dependencies = [
  "ident_case",
  "proc-macro2",
  "quote",
- "strsim",
+ "strsim 0.10.0",
+ "syn",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72"
+dependencies = [
+ "darling_core 0.10.2",
+ "quote",
  "syn",
 ]
 
@@ -246,7 +281,7 @@ version = "0.13.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835"
 dependencies = [
- "darling_core",
+ "darling_core 0.13.4",
  "quote",
  "syn",
 ]
@@ -615,6 +650,16 @@ version = "0.2.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
 
+[[package]]
+name = "libloading"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd"
+dependencies = [
+ "cfg-if",
+ "winapi",
+]
+
 [[package]]
 name = "libmdns"
 version = "0.7.0"
@@ -745,6 +790,29 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "nvml-wrapper"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "288bd66a5a56d8c97b178412b328419b3fdec261c0cbc4628ddc49cc16db8fc6"
+dependencies = [
+ "bitflags",
+ "libloading",
+ "nvml-wrapper-sys",
+ "static_assertions",
+ "thiserror",
+ "wrapcenum-derive",
+]
+
+[[package]]
+name = "nvml-wrapper-sys"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3d606d4edf766969f16828ec047ca9aa96652a17bd353dc0613bfaca49b61d6"
+dependencies = [
+ "libloading",
+]
+
 [[package]]
 name = "object"
 version = "0.28.4"
@@ -786,6 +854,7 @@ dependencies = [
  "iai",
  "libc",
  "libmdns",
+ "nvml-wrapper",
  "once_cell",
  "regex",
  "tokio",
@@ -1008,7 +1077,7 @@ version = "1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082"
 dependencies = [
- "darling",
+ "darling 0.13.4",
  "proc-macro2",
  "quote",
  "syn",
@@ -1055,9 +1124,9 @@ checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32"
 
 [[package]]
 name = "smallvec"
-version = "1.8.0"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
+checksum = "cc88c725d61fc6c3132893370cac4a0200e3fedf5da8331c570664b1987f5ca2"
 
 [[package]]
 name = "socket2"
@@ -1069,6 +1138,18 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
+[[package]]
+name = "strsim"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c"
+
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -1363,9 +1444,9 @@ checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c"
 
 [[package]]
 name = "unicode-normalization"
-version = "0.1.19"
+version = "0.1.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9"
+checksum = "81dee68f85cab8cf68dec42158baf3a79a1cdc065a8b103025965d6ccb7f6cbd"
 dependencies = [
  "tinyvec",
 ]
@@ -1510,3 +1591,15 @@ name = "windows_x86_64_msvc"
 version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
+
+[[package]]
+name = "wrapcenum-derive"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bcc065c85ad2c3bd12aa4118bf164835712e25080c392557801a13292c60aec"
+dependencies = [
+ "darling 0.10.2",
+ "proc-macro2",
+ "quote",
+ "syn",
+]

Cargo.toml

@@ -6,27 +6,28 @@ edition = "2021"
 
 [dependencies]
 color-eyre = "0.6.1"
-warp = "0.3"
+warp = "0.3.2"
-tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
+tokio = { version = "1.19.2", features = ["macros", "rt-multi-thread"] }
-ctrlc = { version = "3", features = ["termination"] }
+ctrlc = { version = "3.2.2", features = ["termination"] }
-dotenv = "0.15"
+dotenv = "0.15.0"
-regex = { version = "1", default-features = false, features = ["std"] }
+regex = { version = "1.5.6", default-features = false, features = ["std"] }
-once_cell = "1"
+once_cell = "1.12.0"
-hostname = "0.3"
+hostname = "0.3.1"
-libc = "0.2"
+libc = "0.2.126"
-ahash = "0.7"
+ahash = "0.7.6"
 bollard = "0.13.0"
-futures-util = "0.3"
+futures-util = "0.3.21"
-libmdns = "0.7"
+libmdns = "0.7.0"
-tracing = "0.1.33"
+tracing = "0.1.35"
 tracing-subscriber = "0.3.11"
+nvml-wrapper = "0.8.0"
 
 [dev-dependencies]
-iai = "0.1"
+iai = "0.1.1"
 
 [[bench]]
 name = "iai"
 harness = false
 
 [profile.release]
-lto = true
+lto = true

src/lib.rs

@@ -1,4 +1,5 @@
 pub mod docker;
+pub mod nvidia;
 pub mod power;
 pub mod sensors;
 pub mod zfs;

src/nvidia.rs

@@ -0,0 +1,16 @@
+use nvml_wrapper::enum_wrappers::device::TemperatureSensor;
+use nvml_wrapper::Nvml;
+use once_cell::sync::Lazy;
+
+static NVIDIA: Lazy<Option<Nvml>> = Lazy::new(|| Nvml::init().ok());
+
+pub fn temperature() -> Option<f32> {
+    let device = NVIDIA.as_ref()?.device_by_index(0).ok()?;
+    let temp = device.temperature(TemperatureSensor::Gpu).ok()?;
+    Some(temp as f32)
+}
+
+pub fn power() -> Option<u64> {
+    let device = NVIDIA.as_ref()?.device_by_index(0).ok()?;
+    device.total_energy_consumption().ok()
+}

src/power.rs

@@ -2,34 +2,45 @@ use color_eyre::{Report, Result};
 use std::fmt::Write;
 use std::fs::{read_dir, read_to_string};
 use std::sync::atomic::{AtomicBool, Ordering};
+use tracing::warn;
 
 static CAN_READ: AtomicBool = AtomicBool::new(true);
 
 #[derive(Debug, Default)]
 pub struct PowerUsage {
-    total_uj: u64,
+    cpu_uj: u64,
-    packages_uj: Vec<u64>,
+    cpu_packages_uj: Vec<u64>,
+    gpu_uj: u64,
 }
 
 impl PowerUsage {
     pub fn write<W: Write>(&self, mut w: W, hostname: &str) {
         writeln!(
             &mut w,
-            "total_power{{host=\"{}\"}} {:.3}",
+            r#"total_power{{host="{}", device="cpu"}} {:.3}"#,
             hostname,
-            self.total_uj as f64 / 1_000_000.0
+            self.cpu_uj as f64 / 1_000_000.0
         )
         .ok();
-        for (i, package) in self.packages_uj.iter().enumerate() {
+        for (i, package) in self.cpu_packages_uj.iter().enumerate() {
             writeln!(
                 &mut w,
-                "package_power{{host=\"{}\", package=\"{}\"}} {:.3}",
+                r#"package_power{{host="{}", package="{}", device="cpu"}} {:.3}"#,
                 hostname,
                 i,
                 *package as f64 / 1_000_000.0
             )
             .ok();
         }
+        if self.gpu_uj > 0 {
+            writeln!(
+                &mut w,
+                r#"total_power{{host="{}", device="gpu"}} {:.3}"#,
+                hostname,
+                self.gpu_uj as f64 / 1_000_000.0
+            )
+            .ok();
+        }
     }
 }
 
@@ -59,14 +70,23 @@ pub fn power_usage() -> Result<Option<PowerUsage>> {
             let package_usage = match read_to_string(&package_path) {
                 Err(e) if e.raw_os_error() == Some(13) => {
                     CAN_READ.store(false, Ordering::Relaxed);
+                    warn!(
+                        package_path = display(package_path.display()),
+                        "can\'t read power usage"
+                    );
                     return Ok(None);
                 }
                 result => result,
             }?;
             let package_usage = package_usage.trim().parse::<u64>()?;
-            usage.total_uj += package_usage;
+            usage.cpu_uj += package_usage;
-            usage.packages_uj.push(package_usage);
+            usage.cpu_packages_uj.push(package_usage);
         }
     }
+
+    if let Some(nvidia_power) = crate::nvidia::power() {
+        usage.gpu_uj = nvidia_power;
+    }
+
     Ok(Some(usage))
 }

src/sensors.rs

@@ -108,6 +108,10 @@ pub fn temperatures() -> Result<Temperatures> {
         temps.cpu = core_total / cores_found
     }
 
+    if let Some(nvidia_temperature) = crate::nvidia::temperature() {
+        temps.gpu = nvidia_temperature;
+    }
+
     Ok(temps)
 }