veloren_server/sys/
metrics.rs

1use crate::{
2    HwStats, Tick, TickStart,
3    chunk_generator::ChunkGenerator,
4    metrics::{EcsSystemMetrics, JobMetrics, PhysicsMetrics, QueryServerMetrics, TickMetrics},
5};
6use common::{resources::TimeOfDay, slowjob::SlowJobPool, terrain::TerrainGrid};
7use common_ecs::{Job, Origin, Phase, SysMetrics, System};
8use specs::{Entities, Join, Read, ReadExpect};
9use std::{
10    sync::{Arc, Mutex},
11    time::Instant,
12};
13use veloren_query_server::server::Metrics as RawQueryServerMetrics;
14
15/// This system exports metrics
16#[derive(Default)]
17pub struct Sys;
18impl<'a> System<'a> for Sys {
19    type SystemData = (
20        Entities<'a>,
21        ReadExpect<'a, HwStats>,
22        ReadExpect<'a, Tick>,
23        ReadExpect<'a, TimeOfDay>,
24        ReadExpect<'a, TickStart>,
25        ReadExpect<'a, ChunkGenerator>,
26        Option<Read<'a, TerrainGrid>>,
27        Read<'a, SysMetrics>,
28        Read<'a, common_ecs::PhysicsMetrics>,
29        ReadExpect<'a, SlowJobPool>,
30        ReadExpect<'a, EcsSystemMetrics>,
31        ReadExpect<'a, TickMetrics>,
32        ReadExpect<'a, PhysicsMetrics>,
33        ReadExpect<'a, JobMetrics>,
34        Option<Read<'a, Arc<Mutex<RawQueryServerMetrics>>>>,
35        ReadExpect<'a, QueryServerMetrics>,
36    );
37
38    const NAME: &'static str = "metrics";
39    const ORIGIN: Origin = Origin::Server;
40    const PHASE: Phase = Phase::Apply;
41
42    fn run(
43        _job: &mut Job<Self>,
44        (
45            entities,
46            hw_stats,
47            tick,
48            time_of_day,
49            tick_start,
50            chunk_generator,
51            terrain,
52            sys_metrics,
53            phys_metrics,
54            slowjobpool,
55            export_ecs,
56            export_tick,
57            export_physics,
58            export_jobs,
59            raw_query_server,
60            export_query_server,
61        ): Self::SystemData,
62    ) {
63        const NANOSEC_PER_SEC: f64 = std::time::Duration::from_secs(1).as_nanos() as f64;
64
65        let start = Instant::now();
66
67        let mut state = sys_metrics.stats.lock().unwrap();
68        //this system hasn't run yet
69        state.remove(Self::NAME);
70
71        for (name, stat) in common_ecs::gen_stats(
72            &state,
73            tick_start.0,
74            hw_stats.rayon_threads,
75            hw_stats.hardware_threads,
76        ) {
77            export_ecs
78                .system_start_time
79                .with_label_values(&[&name])
80                .set(stat.start_ns() as i64);
81            export_ecs
82                .system_thread_avg
83                .with_label_values(&[&name])
84                .set(stat.avg_threads() as f64);
85            let len = stat.length_ns();
86            export_ecs
87                .system_length_time
88                .with_label_values(&[&name])
89                .set(len as i64);
90            export_ecs
91                .system_length_count
92                .with_label_values(&[&name])
93                .inc_by(len);
94            export_ecs
95                .system_length_hist
96                .with_label_values(&[&name])
97                .observe(len as f64 / NANOSEC_PER_SEC);
98        }
99
100        // Report other info
101        export_tick.time_of_day.set(time_of_day.0);
102        if tick.0.rem_euclid(100) == 0 {
103            if let Some(terrain) = terrain.as_ref() {
104                let mut chonk_cnt = 0;
105                let mut group_cnt = 0;
106                let chunk_cnt = terrain.iter().fold(0, |a, (_, c)| {
107                    chonk_cnt += 1;
108                    group_cnt += c.sub_chunk_groups();
109                    a + c.sub_chunks_len()
110                });
111                export_tick.chonks_count.set(chonk_cnt as i64);
112                export_tick.chunks_count.set(chunk_cnt as i64);
113                export_tick.chunk_groups_count.set(group_cnt as i64);
114            }
115
116            let entity_count = entities.join().count();
117            export_tick.entity_count.set(entity_count as i64);
118        }
119        common_base::plot!("entity count", entities.join().count() as f64);
120        common_base::plot!(
121            "pending chunks",
122            chunk_generator.pending_chunks().count() as f64
123        );
124        if let Some(terrain) = terrain.as_ref() {
125            common_base::plot!("chunk count", terrain.iter().count() as f64);
126        }
127
128        //detailed physics metrics
129        export_physics
130            .entity_entity_collision_checks_count
131            .inc_by(phys_metrics.entity_entity_collision_checks);
132        export_physics
133            .entity_entity_collisions_count
134            .inc_by(phys_metrics.entity_entity_collisions);
135
136        //detailed job metrics
137        for (name, jobs) in slowjobpool.take_metrics() {
138            let queried = export_jobs.job_queried_hst.with_label_values(&[&name]);
139            let executed = export_jobs.job_execution_hst.with_label_values(&[&name]);
140            for job in jobs {
141                queried.observe(
142                    job.execution_start
143                        .duration_since(job.queue_created)
144                        .as_secs_f64(),
145                );
146                executed.observe(
147                    job.execution_end
148                        .duration_since(job.execution_start)
149                        .as_secs_f64(),
150                );
151            }
152        }
153
154        // export self time as best as possible
155        export_ecs
156            .system_start_time
157            .with_label_values(&["metrics"])
158            .set(start.duration_since(tick_start.0).as_nanos() as i64);
159        export_ecs
160            .system_thread_avg
161            .with_label_values(&["metrics"])
162            .set(1.0);
163        let len = start.elapsed().as_nanos() as u64;
164        export_ecs
165            .system_length_time
166            .with_label_values(&["metrics"])
167            .set(len as i64);
168        export_ecs
169            .system_length_count
170            .with_label_values(&["metrics"])
171            .inc_by(len);
172        export_ecs
173            .system_length_hist
174            .with_label_values(&["metrics"])
175            .observe(len as f64 / NANOSEC_PER_SEC);
176
177        if let Some(Ok(metrics)) = raw_query_server
178            .as_ref()
179            // Hold the lock for the shortest time possible
180            .map(|m| m.lock().map(|mut metrics| metrics.reset()))
181        {
182            export_query_server.apply(metrics);
183        }
184    }
185}
veloren_server/sys/metrics.rs

veloren_server/sys/
metrics.rs
