415 lines · python
1# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.2# See https://llvm.org/LICENSE.txt for license information.3# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception4"""Tests for metrics.py"""5 6from dataclasses import dataclass7import requests8import unittest9import unittest.mock10 11import metrics12 13 14class TestMetrics(unittest.TestCase):15 def test_upload_gauge_metric(self):16 """Test that we can upload a gauge metric correctly.17 18 Also verify that we pass around parameters like API keys and user IDs19 correctly to the HTTP POST request.20 """21 test_metrics = [metrics.GaugeMetric("gauge_test", 5, 1000)]22 return_value = requests.Response()23 return_value.status_code = 20424 with unittest.mock.patch(25 "requests.post", return_value=return_value26 ) as post_mock:27 metrics.upload_metrics(test_metrics, "test_userid", "test_api_key")28 self.assertSequenceEqual(post_mock.call_args.args, [metrics.GRAFANA_URL])29 self.assertEqual(30 post_mock.call_args.kwargs["data"], "gauge_test value=5 1000"31 )32 self.assertEqual(33 post_mock.call_args.kwargs["auth"], ("test_userid", "test_api_key")34 )35 36 def test_upload_job_metric(self):37 """Test that we can upload a job metric correctly."""38 test_metrics = [39 metrics.JobMetrics(40 "test_job", 5, 10, 1, 1000, 1000, 1000, 7, "test_workflow"41 )42 ]43 return_value = requests.Response()44 return_value.status_code = 20445 with unittest.mock.patch(46 "requests.post", return_value=return_value47 ) as post_mock:48 metrics.upload_metrics(test_metrics, "test_userid", "test_aoi_key")49 self.assertEqual(50 post_mock.call_args.kwargs["data"],51 "test_job queue_time=5,run_time=10,status=1 1000",52 )53 54 def test_upload_aggregate_metric(self):55 """Test that we can upload an aggregate metric correctly."""56 test_metrics = [57 metrics.AggregateMetric("stage1_aggregate", 211, 1124, 1, 1200, 9)58 ]59 return_value = requests.Response()60 return_value.status_code = 20461 with unittest.mock.patch(62 "requests.post", return_value=return_value63 ) as post_mock:64 metrics.upload_metrics(test_metrics, "test_userid", "test_aoi_key")65 self.assertEqual(66 post_mock.call_args.kwargs["data"],67 "stage1_aggregate queue_time=211,run_time=1124,status=1 1200",68 )69 70 def test_upload_unknown_metric(self):71 """Test we report an error if we encounter an unknown metric type."""72 73 @dataclass74 class FakeMetric:75 fake_data: str76 77 test_metrics = [FakeMetric("test")]78 79 with self.assertRaises(ValueError):80 metrics.upload_metrics(test_metrics, "test_userid", "test_api_key")81 82 def test_bad_response_code(self):83 """Test that we gracefully handle HTTP response errors."""84 test_metrics = [metrics.GaugeMetric("gauge_test", 5, 1000)]85 return_value = requests.Response()86 return_value.status_code = 40387 # Just assert that we continue running here and do not raise anything.88 with unittest.mock.patch("requests.post", return_value=return_value) as _:89 metrics.upload_metrics(test_metrics, "test_userid", "test_api_key")90 91 def test_create_and_append_aggregate_metric_1_stage(self):92 """Test the creation of a single AggregateMetric"""93 test_metrics = [94 metrics.JobMetrics(95 "libcxx_stage1_test1",96 8,97 388,98 1,99 created_at_ns=1755697953000000000,100 started_at_ns=1755697961000000000,101 completed_at_ns=1755698349000000000,102 workflow_id=3,103 workflow_name="Build and Test libc++",104 ),105 metrics.JobMetrics(106 "libcxx_stage1_test2",107 107,108 357,109 1,110 created_at_ns=1755697953000000000,111 started_at_ns=1755698060000000000,112 completed_at_ns=1755698417000000000,113 workflow_id=3,114 workflow_name="Build and Test libc++",115 ),116 metrics.JobMetrics(117 "libcxx_stage1_test3",118 8,119 824,120 1,121 created_at_ns=1755697953000000000,122 started_at_ns=1755697961000000000,123 completed_at_ns=1755698785000000000,124 workflow_id=3,125 workflow_name="Build and Test libc++",126 ),127 ]128 metrics.create_and_append_libcxx_aggregates(test_metrics)129 self.assertEqual(len(test_metrics), 4)130 self.assertTrue(isinstance(test_metrics[-1], metrics.AggregateMetric))131 aggregate = test_metrics[-1]132 self.assertEqual(133 aggregate.aggregate_name, "github_libcxx_premerge_checks_stage1_aggregate"134 )135 self.assertEqual(aggregate.aggregate_queue_time, 107)136 self.assertEqual(aggregate.aggregate_run_time, 824)137 self.assertEqual(aggregate.aggregate_status, 1)138 self.assertEqual(aggregate.completed_at_ns, 1755698785000000000)139 self.assertEqual(aggregate.workflow_id, 3)140 141 def test_create_and_append_aggregate_metric_multiple_workflow_ids(self):142 """Test creation of AggregateMetric for same stage with diff workflow ids."""143 test_metrics = [144 metrics.JobMetrics(145 "libcxx_stage1_test1",146 8,147 388,148 1,149 created_at_ns=1755697953000000000,150 started_at_ns=1755697961000000000,151 completed_at_ns=1755698349000000000,152 workflow_id=3,153 workflow_name="Build and Test libc++",154 ),155 metrics.JobMetrics(156 "libcxx_stage1_test2",157 107,158 357,159 0,160 created_at_ns=1755697953000000000,161 started_at_ns=1755698060000000000,162 completed_at_ns=1755698417000000000,163 workflow_id=3,164 workflow_name="Build and Test libc++",165 ),166 metrics.JobMetrics(167 "libcxx_stage1_test3",168 8,169 824,170 1,171 created_at_ns=1755697953000000000,172 started_at_ns=1755697961000000000,173 completed_at_ns=1755698785000000000,174 workflow_id=25,175 workflow_name="Build and Test libc++",176 ),177 ]178 metrics.create_and_append_libcxx_aggregates(test_metrics)179 self.assertEqual(len(test_metrics), 5)180 self.assertTrue(isinstance(test_metrics[3], metrics.AggregateMetric))181 self.assertTrue(isinstance(test_metrics[4], metrics.AggregateMetric))182 aggregate = test_metrics[3]183 self.assertEqual(184 aggregate.aggregate_name, "github_libcxx_premerge_checks_stage1_aggregate"185 )186 self.assertEqual(aggregate.aggregate_queue_time, 107)187 self.assertEqual(aggregate.aggregate_run_time, 456)188 self.assertEqual(aggregate.aggregate_status, 0)189 self.assertEqual(aggregate.completed_at_ns, 1755698417000000000)190 self.assertEqual(aggregate.workflow_id, 3)191 192 aggregate = test_metrics[4]193 self.assertEqual(194 aggregate.aggregate_name, "github_libcxx_premerge_checks_stage1_aggregate"195 )196 self.assertEqual(aggregate.aggregate_queue_time, 8)197 self.assertEqual(aggregate.aggregate_run_time, 824)198 self.assertEqual(aggregate.aggregate_status, 1)199 self.assertEqual(aggregate.completed_at_ns, 1755698785000000000)200 self.assertEqual(aggregate.workflow_id, 25)201 202 def test_create_and_append_aggregate_metric_3_stages(self):203 """Test the creation of AggregateMetric for each of 3 stages."""204 test_metrics = [205 metrics.JobMetrics(206 "libcxx_stage1_test1",207 124,208 1454,209 1,210 created_at_ns=1755696929000000000,211 started_at_ns=1755697053000000000,212 completed_at_ns=1755698507000000000,213 workflow_id=17,214 workflow_name="Build and Test libc++",215 ),216 metrics.JobMetrics(217 "libcxx_stage1_test2",218 129,219 827,220 1,221 created_at_ns=1755696929000000000,222 started_at_ns=1755697058000000000,223 completed_at_ns=1755697885000000000,224 workflow_id=17,225 workflow_name="Build and Test libc++",226 ),227 metrics.JobMetrics(228 "libcxx_stage2_test1",229 6,230 580,231 1,232 created_at_ns=1755698507000000000,233 started_at_ns=1755698513000000000,234 completed_at_ns=1755699093000000000,235 workflow_id=17,236 workflow_name="Build and Test libc++",237 ),238 metrics.JobMetrics(239 "libcxx_stage2_test2",240 7,241 473,242 1,243 created_at_ns=1755698507000000000,244 started_at_ns=1755698514000000000,245 completed_at_ns=1755698987000000000,246 workflow_id=17,247 workflow_name="Build and Test libc++",248 ),249 metrics.JobMetrics(250 "libcxx_stage2_test3",251 7,252 820,253 1,254 created_at_ns=1755698507000000000,255 started_at_ns=1755698514000000000,256 completed_at_ns=1755699334000000000,257 workflow_id=17,258 workflow_name="Build and Test libc++",259 ),260 metrics.JobMetrics(261 "libcxx_stage3_test1",262 7,263 919,264 1,265 created_at_ns=1755709005000000000,266 started_at_ns=1755709012000000000,267 completed_at_ns=1755709931000000000,268 workflow_id=17,269 workflow_name="Build and Test libc++",270 ),271 metrics.JobMetrics(272 "libcxx_stage3_test2",273 141,274 834,275 1,276 created_at_ns=1755709005000000000,277 started_at_ns=1755709146000000000,278 completed_at_ns=1755709980000000000,279 workflow_id=17,280 workflow_name="Build and Test libc++",281 ),282 metrics.JobMetrics(283 "libcxx_stage3_test3",284 131,285 370,286 1,287 created_at_ns=1755709005000000000,288 started_at_ns=1755709136000000000,289 completed_at_ns=1755709514000000000,290 workflow_id=17,291 workflow_name="Build and Test libc++",292 ),293 ]294 metrics.create_and_append_libcxx_aggregates(test_metrics)295 self.assertEqual(len(test_metrics), 11)296 self.assertTrue(isinstance(test_metrics[8], metrics.AggregateMetric))297 self.assertTrue(isinstance(test_metrics[9], metrics.AggregateMetric))298 self.assertTrue(isinstance(test_metrics[10], metrics.AggregateMetric))299 aggregate = test_metrics[8]300 self.assertEqual(301 aggregate.aggregate_name, "github_libcxx_premerge_checks_stage1_aggregate"302 )303 self.assertEqual(aggregate.aggregate_queue_time, 129)304 self.assertEqual(aggregate.aggregate_run_time, 1454)305 self.assertEqual(aggregate.aggregate_status, 1)306 self.assertEqual(aggregate.completed_at_ns, 1755698507000000000)307 self.assertEqual(aggregate.workflow_id, 17)308 309 aggregate = test_metrics[9]310 self.assertEqual(311 aggregate.aggregate_name, "github_libcxx_premerge_checks_stage2_aggregate"312 )313 self.assertEqual(aggregate.aggregate_queue_time, 7)314 self.assertEqual(aggregate.aggregate_run_time, 821)315 self.assertEqual(aggregate.aggregate_status, 1)316 self.assertEqual(aggregate.completed_at_ns, 1755699334000000000)317 self.assertEqual(aggregate.workflow_id, 17)318 319 aggregate = test_metrics[10]320 self.assertEqual(321 aggregate.aggregate_name, "github_libcxx_premerge_checks_stage3_aggregate"322 )323 self.assertEqual(aggregate.aggregate_queue_time, 141)324 self.assertEqual(aggregate.aggregate_run_time, 968)325 self.assertEqual(aggregate.aggregate_status, 1)326 self.assertEqual(aggregate.completed_at_ns, 1755709980000000000)327 self.assertEqual(aggregate.workflow_id, 17)328 329 def test_create_and_append_aggregate_metric_mixed_job_types(self):330 """Test the creation of AggregateMetric with non-lib++ jobs thrown in."""331 test_metrics = [332 metrics.JobMetrics(333 "ci_test1", 5, 10, 1, 1000, 1200, 1400, 5, "premerge_test"334 ),335 metrics.JobMetrics(336 "libcxx_stage1_test1",337 8,338 388,339 1,340 created_at_ns=1755697953000000000,341 started_at_ns=1755697961000000000,342 completed_at_ns=1755698349000000000,343 workflow_id=3,344 workflow_name="Build and Test libc++",345 ),346 metrics.JobMetrics(347 "ci_test2", 3, 20, 1, 2000, 2200, 2400, 37, "premerge_test"348 ),349 metrics.JobMetrics(350 "libcxx_stage1_test2",351 107,352 357,353 0,354 created_at_ns=1755697953000000000,355 started_at_ns=1755698060000000000,356 completed_at_ns=1755698417000000000,357 workflow_id=3,358 workflow_name="Build and Test libc++",359 ),360 metrics.JobMetrics(361 "ci_test3", 7, 35, 1, 3000, 3200, 3400, 85, "premerge_test"362 ),363 ]364 metrics.create_and_append_libcxx_aggregates(test_metrics)365 self.assertEqual(len(test_metrics), 6)366 self.assertTrue(isinstance(test_metrics[5], metrics.AggregateMetric))367 aggregate = test_metrics[5]368 self.assertEqual(369 aggregate.aggregate_name, "github_libcxx_premerge_checks_stage1_aggregate"370 )371 self.assertEqual(aggregate.aggregate_queue_time, 107)372 self.assertEqual(aggregate.aggregate_run_time, 456)373 self.assertEqual(aggregate.aggregate_status, 0)374 self.assertEqual(aggregate.completed_at_ns, 1755698417000000000)375 self.assertEqual(aggregate.workflow_id, 3)376 377 def test_create_and_append_aggregate_metric_no_libcxx_jobs(self):378 """Test the creation of AggregateMetric with no libc++ jobs.379 380 In this case, no AggregateMetric should be created, but no381 errors or complaints should be raised.382 """383 test_metrics = [384 metrics.JobMetrics(385 "ci_test1", 5, 10, 1, 1000, 1200, 1400, 5, "premerge_test"386 ),387 metrics.JobMetrics(388 "ci_test2", 3, 20, 1, 2000, 2200, 2400, 37, "premerge_test"389 ),390 metrics.JobMetrics(391 "ci_test3", 7, 35, 1, 3000, 3200, 3400, 85, "premerge_test"392 ),393 ]394 metrics.create_and_append_libcxx_aggregates(test_metrics)395 self.assertEqual(len(test_metrics), 3)396 397 def test_clean_up_libcxx_job_name(self):398 """Test that we correctly update (or not) libcxx job names."""399 stage1_name = "stage1 (test1, C++-test2, my-c++-test-25)"400 stage2_name = "stage2 (generic-cxx26, clang-21, clang++21)"401 stage3_name = "stage3 (generic-cxx26, libcxx-next-runners, junk)"402 bad_name = "this is a bad name"403 out_name1 = metrics.clean_up_libcxx_job_name(stage1_name)404 self.assertEqual(out_name1, "stage1_test1__Cxx_test2__my_cxx_test_25")405 out_name2 = metrics.clean_up_libcxx_job_name(stage2_name)406 self.assertEqual(out_name2, "stage2_generic_cxx26__clang_21__clangxx21")407 out_name3 = metrics.clean_up_libcxx_job_name(stage3_name)408 self.assertEqual(out_name3, "stage3_generic_cxx26__libcxx_next_runners__junk")409 out_name4 = metrics.clean_up_libcxx_job_name(bad_name)410 self.assertEqual(out_name4, bad_name)411 412 413if __name__ == "__main__":414 unittest.main()415