Skip to content

Fix tracing search filter by service

Daniele Rossetti requested to merge rossetd/tracing-fix-service-filtering into master

What does this MR do and why?

Fix a bug where searching traces by service does not work. This was caused by the filter name to be wrong, and hence filtered out.

Closes gitlab-org/opstrace/opstrace#2513 (closed)

Screenshots or screen recordings

NA

How to set up and validate locally

  • Enable :observability_tracing FF

Apply this patch to load mocks ( pbpaste | git apply --allow-empty )

diff --git a/app/assets/javascripts/observability/client.js b/app/assets/javascripts/observability/client.js
index e91964232aa9..ffb04c9e8c0a 100644
--- a/app/assets/javascripts/observability/client.js
+++ b/app/assets/javascripts/observability/client.js
@@ -1,20 +1,29 @@
 import * as Sentry from '~/sentry/sentry_browser_wrapper';
 import axios from '~/lib/utils/axios_utils';
 import { logError } from '~/lib/logger';
+import mockData from './mock_traces.json';
 
 function reportErrorAndThrow(e) {
   logError(e);
   Sentry.captureException(e);
   throw e;
 }
+
+function mockReturnDataWithDelay(data) {
+  return new Promise((resolve) => {
+    setTimeout(() => resolve(data), 500);
+  });
+}
+
 // Provisioning API spec: https://gitlab.com/gitlab-org/opstrace/opstrace/-/blob/main/provisioning-api/pkg/provisioningapi/routes.go#L59
 async function enableObservability(provisioningUrl) {
   try {
     // Note: axios.put(url, undefined, {withCredentials: true}) does not send cookies properly, so need to use the API below for the correct behaviour
-    return await axios(provisioningUrl, {
-      method: 'put',
-      withCredentials: true,
-    });
+    // return await axios(provisioningUrl, {
+    //   method: 'put',
+    //   withCredentials: true,
+    // });
+    return mockReturnDataWithDelay();
   } catch (e) {
     return reportErrorAndThrow(e);
   }
@@ -23,19 +32,20 @@ async function enableObservability(provisioningUrl) {
 // Provisioning API spec: https://gitlab.com/gitlab-org/opstrace/opstrace/-/blob/main/provisioning-api/pkg/provisioningapi/routes.go#L37
 async function isObservabilityEnabled(provisioningUrl) {
   try {
-    const { data } = await axios.get(provisioningUrl, { withCredentials: true });
+    // const { data } = await axios.get(provisioningUrl, { withCredentials: true });
+    const data = { status: 'ready' };
     if (data && data.status) {
       // we currently ignore the 'status' payload and just check if the request was successful
       // We might improve this as part of https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/2315
-      return true;
+      return mockReturnDataWithDelay(true);
     }
+    return false;
   } catch (e) {
     if (e.response.status === 404) {
       return false;
     }
     return reportErrorAndThrow(e);
   }
-  return reportErrorAndThrow(new Error('Failed to check provisioning')); // eslint-disable-line @gitlab/require-i18n-strings
 }
 
 async function fetchTrace(tracingUrl, traceId) {
@@ -44,18 +54,19 @@ async function fetchTrace(tracingUrl, traceId) {
       throw new Error('traceId is required.');
     }
 
-    const { data } = await axios.get(tracingUrl, {
-      withCredentials: true,
-      params: {
-        trace_id: traceId,
-      },
-    });
+    // const { data } = await axios.get(tracingUrl, {
+    //   withCredentials: true,
+    //   params: {
+    //     trace_id: traceId,
+    //   },
+    // });
+    const data = { traces: [mockData.traces.find((t) => t.trace_id === traceId)] };
 
     if (!Array.isArray(data.traces) || data.traces.length === 0) {
       throw new Error('traces are missing/invalid in the response'); // eslint-disable-line @gitlab/require-i18n-strings
     }
 
-    return data.traces[0];
+    return mockReturnDataWithDelay(data.traces[0]);
   } catch (e) {
     return reportErrorAndThrow(e);
   }
@@ -200,15 +211,18 @@ async function fetchTraces(tracingUrl, { filters = {}, pageToken, pageSize } = {
     params.append('page_size', pageSize);
   }
 
+  console.log(`fetching traces with params: ${params.toString()}`);
+
   try {
-    const { data } = await axios.get(tracingUrl, {
-      withCredentials: true,
-      params,
-    });
+    // const { data } = await axios.get(tracingUrl, {
+    //   withCredentials: true,
+    //   params,
+    // });
+    const data = mockData;
     if (!Array.isArray(data.traces)) {
       throw new Error('traces are missing/invalid in the response'); // eslint-disable-line @gitlab/require-i18n-strings
     }
-    return data;
+    return mockReturnDataWithDelay(data);
   } catch (e) {
     return reportErrorAndThrow(e);
   }
@@ -216,15 +230,20 @@ async function fetchTraces(tracingUrl, { filters = {}, pageToken, pageSize } = {
 
 async function fetchServices(servicesUrl) {
   try {
-    const { data } = await axios.get(servicesUrl, {
-      withCredentials: true,
-    });
+    // const { data } = await axios.get(servicesUrl, {
+    //   withCredentials: true,
+    // });
+
+    const uniqueServices = new Set(
+      mockData.traces.map((t) => t.spans.map((s) => s.service_name)).flat(),
+    );
+    const data = { services: Array.from(uniqueServices).map((s) => ({ name: s })) };
 
     if (!Array.isArray(data.services)) {
       throw new Error('failed to fetch services. invalid response'); // eslint-disable-line @gitlab/require-i18n-strings
     }
 
-    return data.services;
+    return mockReturnDataWithDelay(data.services);
   } catch (e) {
     return reportErrorAndThrow(e);
   }
@@ -239,15 +258,24 @@ async function fetchOperations(operationsUrl, serviceName) {
       throw new Error('fetchOperations() - operationsUrl must contain $SERVICE_NAME$');
     }
     const url = operationsUrl.replace('$SERVICE_NAME$', serviceName);
-    const { data } = await axios.get(url, {
-      withCredentials: true,
-    });
+    // const { data } = await axios.get(url, {
+    //   withCredentials: true,
+    // });
+
+    console.log('fetching operations suggestions for', url); // eslint-disable-line @gitlab/require-i18n-strings
+    const uniqOps = new Set(
+      mockData.traces
+        .map((t) => t.spans.filter((s) => s.service_name === serviceName))
+        .flat()
+        .map((s) => s.operation),
+    );
+    const data = { operations: Array.from(uniqOps).map((s) => ({ name: s })) };
 
     if (!Array.isArray(data.operations)) {
       throw new Error('failed to fetch operations. invalid response'); // eslint-disable-line @gitlab/require-i18n-strings
     }
 
-    return data.operations;
+    return mockReturnDataWithDelay(data.operations);
   } catch (e) {
     return reportErrorAndThrow(e);
   }
diff --git a/app/assets/javascripts/observability/components/observability_container.vue b/app/assets/javascripts/observability/components/observability_container.vue
index b89c2624f81c..f6cbf7ee771f 100644
--- a/app/assets/javascripts/observability/components/observability_container.vue
+++ b/app/assets/javascripts/observability/components/observability_container.vue
@@ -27,12 +27,12 @@ export default {
 
     // TODO: Improve local GDK dev experience with tracing https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/2308
     // Uncomment the lines below to to test this locally
-    // setTimeout(() => {
-    //   this.messageHandler({
-    //     data: { type: 'AUTH_COMPLETION', status: 'success' },
-    //     origin: new URL(this.oauthUrl).origin,
-    //   });
-    // }, 2000);
+    setTimeout(() => {
+      this.messageHandler({
+        data: { type: 'AUTH_COMPLETION', status: 'success' },
+        origin: new URL(this.apiConfig.oauthUrl).origin,
+      });
+    }, 2000);
   },
   destroyed() {
     window.removeEventListener('message', this.messageHandler);
diff --git a/app/assets/javascripts/observability/mock_traces.json b/app/assets/javascripts/observability/mock_traces.json
new file mode 100644
index 000000000000..47cfca3cf064
--- /dev/null
+++ b/app/assets/javascripts/observability/mock_traces.json
@@ -0,0 +1,1497 @@
+{
+  "project_id": 1,
+  "traces": [
+    {
+      "timestamp": "2023-10-06T11:55:59.598163997Z",
+      "trace_id": "ccaffd4e-a91a-d804-0e0f-f922c4c95ed7",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 2127527881,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:59.598163997Z",
+          "span_id": "B37C17DBFF15E41D",
+          "trace_id": "ccaffd4e-a91a-d804-0e0f-f922c4c95ed7",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 2127527881,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        }
+      ],
+      "totalSpans": 1
+    },
+    {
+      "timestamp": "2023-10-06T11:55:59.475236942Z",
+      "trace_id": "82c50614-2aa6-8e33-62b5-a2097d47feef",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 2248422103,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:59.475236942Z",
+          "span_id": "0938FA5DA440885E",
+          "trace_id": "82c50614-2aa6-8e33-62b5-a2097d47feef",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 2248422103,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        }
+      ],
+      "totalSpans": 1
+    },
+    {
+      "timestamp": "2023-10-06T11:55:56.751654326Z",
+      "trace_id": "5ffc70f5-2ddd-917c-a5b3-ed4b08aaf1f2",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 5002628251,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:56.751654326Z",
+          "span_id": "17A162ECC1C0909D",
+          "trace_id": "5ffc70f5-2ddd-917c-a5b3-ed4b08aaf1f2",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5002628251,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        }
+      ],
+      "totalSpans": 1
+    },
+    {
+      "timestamp": "2023-10-06T11:55:56.718823805Z",
+      "trace_id": "2771e54e-7029-3886-9bb1-25a159dda380",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 396589042,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:56.718823805Z",
+          "span_id": "FF943BA1E5266248",
+          "trace_id": "2771e54e-7029-3886-9bb1-25a159dda380",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 396589042,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:56.719722Z",
+          "span_id": "2589A0DAAC06088D",
+          "trace_id": "2771e54e-7029-3886-9bb1-25a159dda380",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 394466000,
+          "parent_span_id": "FF943BA1E5266248",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:56.542930163Z",
+      "trace_id": "93dff16a-3ec6-d09b-a154-75a928636a86",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 5002656067,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:56.542930163Z",
+          "span_id": "747271934A59C0E3",
+          "trace_id": "93dff16a-3ec6-d09b-a154-75a928636a86",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5002656067,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        }
+      ],
+      "totalSpans": 1
+    },
+    {
+      "timestamp": "2023-10-06T11:55:55.648453589Z",
+      "trace_id": "0d3cf1f7-d611-76e2-d776-9265adb1af9b",
+      "service_name": "loadgenerator",
+      "operation": "POST",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 5000814327,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:55.648453589Z",
+          "span_id": "2C4CEB0BE86B2DA0",
+          "trace_id": "0d3cf1f7-d611-76e2-d776-9265adb1af9b",
+          "service_name": "loadgenerator",
+          "operation": "POST",
+          "duration_nano": 5000814327,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        }
+      ],
+      "totalSpans": 1
+    },
+    {
+      "timestamp": "2023-10-06T11:55:55.279674996Z",
+      "trace_id": "5766ddab-7eee-cdae-3586-9f939e789d6a",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 1703361037,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:55.279674996Z",
+          "span_id": "733124CF29C09031",
+          "trace_id": "5766ddab-7eee-cdae-3586-9f939e789d6a",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 1703361037,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:55.280899Z",
+          "span_id": "EE3A66448BAD6797",
+          "trace_id": "5766ddab-7eee-cdae-3586-9f939e789d6a",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 1700994000,
+          "parent_span_id": "733124CF29C09031",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:54.155602447Z",
+      "trace_id": "512ee0a8-e7ce-f535-9fb7-d24e74970bbd",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 5014426685,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:54.155602447Z",
+          "span_id": "1C0EF2059EDB15EC",
+          "trace_id": "512ee0a8-e7ce-f535-9fb7-d24e74970bbd",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5014426685,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        }
+      ],
+      "totalSpans": 1
+    },
+    {
+      "timestamp": "2023-10-06T11:55:53.613428903Z",
+      "trace_id": "167dc5d1-fab3-2536-e02a-54aae6ad0e51",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 5000100610,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:53.613428903Z",
+          "span_id": "6643B634846E8790",
+          "trace_id": "167dc5d1-fab3-2536-e02a-54aae6ad0e51",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5000100610,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:53.614291Z",
+          "span_id": "0E90919A51493DAB",
+          "trace_id": "167dc5d1-fab3-2536-e02a-54aae6ad0e51",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 4998281000,
+          "parent_span_id": "6643B634846E8790",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:53.305205604Z",
+      "trace_id": "bee441ce-8417-fc94-c13c-6764c37827b0",
+      "service_name": "frontend-proxy",
+      "operation": "ingress",
+      "statusCode": "STATUS_CODE_UNSET",
+      "duration_nano": 493620000,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:53.306589Z",
+          "span_id": "7A95C3E94E619E8B",
+          "trace_id": "bee441ce-8417-fc94-c13c-6764c37827b0",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 493620000,
+          "parent_span_id": "DAE1E37B9930821D",
+          "statusCode": "STATUS_CODE_UNSET"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:53.305205604Z",
+          "span_id": "DAE1E37B9930821D",
+          "trace_id": "bee441ce-8417-fc94-c13c-6764c37827b0",
+          "service_name": "loadgenerator",
+          "operation": "POST",
+          "duration_nano": 496549491,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:52.113342692Z",
+      "trace_id": "0efe9727-0e5a-f20a-3686-e2a095d4494e",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 5002894331,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:52.113342692Z",
+          "span_id": "6BDB4DA828A2F7D3",
+          "trace_id": "0efe9727-0e5a-f20a-3686-e2a095d4494e",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5002894331,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:52.114429Z",
+          "span_id": "59E4FF2E4707BFEA",
+          "trace_id": "0efe9727-0e5a-f20a-3686-e2a095d4494e",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 4999835000,
+          "parent_span_id": "6BDB4DA828A2F7D3",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:51.981354665Z",
+      "trace_id": "3f917d5b-81c1-8319-3aaf-74c7c11bca89",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 1323083652,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:51.981354665Z",
+          "span_id": "3D69B158C2AFDFBA",
+          "trace_id": "3f917d5b-81c1-8319-3aaf-74c7c11bca89",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 1323083652,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:51.982215Z",
+          "span_id": "3588DA6D14910718",
+          "trace_id": "3f917d5b-81c1-8319-3aaf-74c7c11bca89",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 1321045000,
+          "parent_span_id": "3D69B158C2AFDFBA",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:50.643316698Z",
+      "trace_id": "70f3d706-c3cf-57ce-e40d-e8f96f2a9001",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 5004455156,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:50.643316698Z",
+          "span_id": "612F7863A775A173",
+          "trace_id": "70f3d706-c3cf-57ce-e40d-e8f96f2a9001",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5004455156,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:50.644165Z",
+          "span_id": "9DF351C79B0676F1",
+          "trace_id": "70f3d706-c3cf-57ce-e40d-e8f96f2a9001",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 5002605000,
+          "parent_span_id": "612F7863A775A173",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:50.212288079Z",
+      "trace_id": "fef41346-a584-daca-bda3-81e6ae743772",
+      "service_name": "featureflagservice",
+      "operation": "/featureflags",
+      "statusCode": "STATUS_CODE_UNSET",
+      "duration_nano": 2420246,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:50.212288079Z",
+          "span_id": "44D4038CC63D0871",
+          "trace_id": "fef41346-a584-daca-bda3-81e6ae743772",
+          "service_name": "featureflagservice",
+          "operation": "/featureflags",
+          "duration_nano": 2420246,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_UNSET"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:50.212626103Z",
+          "span_id": "6163630D76176FF1",
+          "trace_id": "fef41346-a584-daca-bda3-81e6ae743772",
+          "service_name": "featureflagservice",
+          "operation": "featureflagservice.repo.query:featureflags",
+          "duration_nano": 1370183,
+          "parent_span_id": "44D4038CC63D0871",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:48.799130403Z",
+      "trace_id": "e0ce9945-effc-92c9-a92f-cb252f8db3dd",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 4508060447,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:48.799130403Z",
+          "span_id": "9B0B4EDE75DBD151",
+          "trace_id": "e0ce9945-effc-92c9-a92f-cb252f8db3dd",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 4508060447,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:48.800147Z",
+          "span_id": "B4C86FDF2B249A75",
+          "trace_id": "e0ce9945-effc-92c9-a92f-cb252f8db3dd",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 4503193000,
+          "parent_span_id": "9B0B4EDE75DBD151",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:48.649676084Z",
+      "trace_id": "7e29e2a0-4e88-d554-a828-b7106756a16a",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 960467039,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:48.649676084Z",
+          "span_id": "9CFA437044CD601A",
+          "trace_id": "7e29e2a0-4e88-d554-a828-b7106756a16a",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 960467039,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:48.650744Z",
+          "span_id": "5E0C9408692A8DFA",
+          "trace_id": "7e29e2a0-4e88-d554-a828-b7106756a16a",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 958297000,
+          "parent_span_id": "9CFA437044CD601A",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:48.303476356Z",
+      "trace_id": "62ca5bc0-aa3f-6b7c-1fcf-c54e008bd6ca",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 5003432217,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:48.303476356Z",
+          "span_id": "A95584D78DA209EC",
+          "trace_id": "62ca5bc0-aa3f-6b7c-1fcf-c54e008bd6ca",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5003432217,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:48.30431Z",
+          "span_id": "F9DEE9CE2F553196",
+          "trace_id": "62ca5bc0-aa3f-6b7c-1fcf-c54e008bd6ca",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 4999083000,
+          "parent_span_id": "A95584D78DA209EC",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:44.603635139Z",
+      "trace_id": "ba620100-214c-6002-70e2-317f06613698",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 1480294366,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:44.603635139Z",
+          "span_id": "FF13AB1ED9FC0229",
+          "trace_id": "ba620100-214c-6002-70e2-317f06613698",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 1480294366,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:44.605025Z",
+          "span_id": "46D63CE6A105E626",
+          "trace_id": "ba620100-214c-6002-70e2-317f06613698",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 1477836000,
+          "parent_span_id": "FF13AB1ED9FC0229",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:43.465288869Z",
+      "trace_id": "206eec00-b21c-66a0-c480-c65026e1ae53",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 5003399346,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:43.465288869Z",
+          "span_id": "6E5544458A0864D5",
+          "trace_id": "206eec00-b21c-66a0-c480-c65026e1ae53",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5003399346,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:43.466157Z",
+          "span_id": "8374413A26AA15F1",
+          "trace_id": "206eec00-b21c-66a0-c480-c65026e1ae53",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 5001446000,
+          "parent_span_id": "6E5544458A0864D5",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:41.876908072Z",
+      "trace_id": "e6510ccf-b966-546d-dfd8-a0f14ce7f5ad",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 5005142605,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:41.876908072Z",
+          "span_id": "CB0AAA0DD240543D",
+          "trace_id": "e6510ccf-b966-546d-dfd8-a0f14ce7f5ad",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5005142605,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:41.87814Z",
+          "span_id": "1E886EC50D17DE09",
+          "trace_id": "e6510ccf-b966-546d-dfd8-a0f14ce7f5ad",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 5002625000,
+          "parent_span_id": "CB0AAA0DD240543D",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:41.690765809Z",
+      "trace_id": "c2cea2df-30fa-2df3-fd45-30fa653d1b23",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 5006927073,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:41.690765809Z",
+          "span_id": "6372BBCBE324D4DF",
+          "trace_id": "c2cea2df-30fa-2df3-fd45-30fa653d1b23",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5006927073,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:41.691638Z",
+          "span_id": "7E264C16B5932D29",
+          "trace_id": "c2cea2df-30fa-2df3-fd45-30fa653d1b23",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 5004971000,
+          "parent_span_id": "6372BBCBE324D4DF",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:41.141410499Z",
+      "trace_id": "8557d8d9-ea31-9f57-b076-5580c2ecd542",
+      "service_name": "loadgenerator",
+      "operation": "POST",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 4998737278,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:41.141410499Z",
+          "span_id": "1014F6FED940B6DC",
+          "trace_id": "8557d8d9-ea31-9f57-b076-5580c2ecd542",
+          "service_name": "loadgenerator",
+          "operation": "POST",
+          "duration_nano": 4998737278,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:41.142188Z",
+          "span_id": "7C949588AF7D788D",
+          "trace_id": "8557d8d9-ea31-9f57-b076-5580c2ecd542",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 4996805000,
+          "parent_span_id": "1014F6FED940B6DC",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:41.080812502Z",
+      "trace_id": "40dd5e45-b283-6b5b-f7af-0cd5cb92dc9c",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 5003851281,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:41.080812502Z",
+          "span_id": "06E4A468012D8639",
+          "trace_id": "40dd5e45-b283-6b5b-f7af-0cd5cb92dc9c",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5003851281,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:41.082036Z",
+          "span_id": "BE51364F03C5D820",
+          "trace_id": "40dd5e45-b283-6b5b-f7af-0cd5cb92dc9c",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 5000885000,
+          "parent_span_id": "06E4A468012D8639",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:40.212726902Z",
+      "trace_id": "ae4bd107-3820-e593-a5c6-5898d4c239c2",
+      "service_name": "featureflagservice",
+      "operation": "/featureflags",
+      "statusCode": "STATUS_CODE_UNSET",
+      "duration_nano": 2511410,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:40.212726902Z",
+          "span_id": "C92EECC4F9575B00",
+          "trace_id": "ae4bd107-3820-e593-a5c6-5898d4c239c2",
+          "service_name": "featureflagservice",
+          "operation": "/featureflags",
+          "duration_nano": 2511410,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_UNSET"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:40.213119592Z",
+          "span_id": "A553A8DE107E8CB8",
+          "trace_id": "ae4bd107-3820-e593-a5c6-5898d4c239c2",
+          "service_name": "featureflagservice",
+          "operation": "featureflagservice.repo.query:featureflags",
+          "duration_nano": 1229795,
+          "parent_span_id": "C92EECC4F9575B00",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:40.046488271Z",
+      "trace_id": "c53d7dc8-a60f-0d7c-1b60-0879d2c67fc5",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_OK",
+      "duration_nano": 2000376195,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:40.046488271Z",
+          "span_id": "C444F1FEC2BA8819",
+          "trace_id": "c53d7dc8-a60f-0d7c-1b60-0879d2c67fc5",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 2000376195,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:40.047866Z",
+          "span_id": "55C6FC54E3D5D334",
+          "trace_id": "c53d7dc8-a60f-0d7c-1b60-0879d2c67fc5",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 1997469000,
+          "parent_span_id": "C444F1FEC2BA8819",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:39.201064082Z",
+      "trace_id": "6cd194e8-d869-0f8e-e256-d70b44d2819c",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 2848833737,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:39.201064082Z",
+          "span_id": "2474C12ADEB41E4F",
+          "trace_id": "6cd194e8-d869-0f8e-e256-d70b44d2819c",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 2848833737,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:39.210427Z",
+          "span_id": "7C82CE36FB37D71C",
+          "trace_id": "6cd194e8-d869-0f8e-e256-d70b44d2819c",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 2834964000,
+          "parent_span_id": "2474C12ADEB41E4F",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:37.044547199Z",
+      "trace_id": "b8256ede-d3c5-f260-85b1-45f8fce9e609",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 5004998888,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:37.044547199Z",
+          "span_id": "CC9F510C6A33D5DE",
+          "trace_id": "b8256ede-d3c5-f260-85b1-45f8fce9e609",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5004998888,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:37.045884Z",
+          "span_id": "8BA28174DC4F2800",
+          "trace_id": "b8256ede-d3c5-f260-85b1-45f8fce9e609",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 4999544000,
+          "parent_span_id": "CC9F510C6A33D5DE",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:37.04353471Z",
+      "trace_id": "f1d86bf2-a7cb-3114-00f5-619653306fd2",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 5005572363,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:37.04353471Z",
+          "span_id": "1CAE17E35616EF02",
+          "trace_id": "f1d86bf2-a7cb-3114-00f5-619653306fd2",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5005572363,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:37.045351Z",
+          "span_id": "6D1A0A914A5ACAF2",
+          "trace_id": "f1d86bf2-a7cb-3114-00f5-619653306fd2",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 5000457000,
+          "parent_span_id": "1CAE17E35616EF02",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:36.138448115Z",
+      "trace_id": "c852fe52-7845-6c20-c9f4-dcd06e35a0d3",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 5002279465,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:36.138448115Z",
+          "span_id": "013D44FBAFECAF3E",
+          "trace_id": "c852fe52-7845-6c20-c9f4-dcd06e35a0d3",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5002279465,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:36.139697Z",
+          "span_id": "F5214C79F52FA1D3",
+          "trace_id": "c852fe52-7845-6c20-c9f4-dcd06e35a0d3",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 4999948000,
+          "parent_span_id": "013D44FBAFECAF3E",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:35.063009962Z",
+      "trace_id": "55ab8cd4-b2fb-e05b-1813-8b92e99715b4",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 2462137886,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:35.063009962Z",
+          "span_id": "E1B4C72A38690973",
+          "trace_id": "55ab8cd4-b2fb-e05b-1813-8b92e99715b4",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 2462137886,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:35.064395Z",
+          "span_id": "A52918F4C0929CF5",
+          "trace_id": "55ab8cd4-b2fb-e05b-1813-8b92e99715b4",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 2459363000,
+          "parent_span_id": "E1B4C72A38690973",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:34.402711283Z",
+      "trace_id": "a6f5cdfb-48cb-319d-20b0-80d1bc71dfe7",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 5005182495,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:34.402711283Z",
+          "span_id": "8FDDAFD8C2F77937",
+          "trace_id": "a6f5cdfb-48cb-319d-20b0-80d1bc71dfe7",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5005182495,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:34.403988Z",
+          "span_id": "2B88D36102DEFAA9",
+          "trace_id": "a6f5cdfb-48cb-319d-20b0-80d1bc71dfe7",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 5000542000,
+          "parent_span_id": "8FDDAFD8C2F77937",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:33.582152671Z",
+      "trace_id": "f14ed0ae-82a2-3821-4a95-988c56506c0a",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 1696858107,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:33.582152671Z",
+          "span_id": "AFF8492F14E36FFA",
+          "trace_id": "f14ed0ae-82a2-3821-4a95-988c56506c0a",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 1696858107,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:33.583699Z",
+          "span_id": "F4F41B58483F800D",
+          "trace_id": "f14ed0ae-82a2-3821-4a95-988c56506c0a",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 1693457000,
+          "parent_span_id": "AFF8492F14E36FFA",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:33.33485647Z",
+      "trace_id": "07402476-3308-4282-7202-d7ac61c374a8",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 5003319301,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:33.33485647Z",
+          "span_id": "B468FAAC4680C389",
+          "trace_id": "07402476-3308-4282-7202-d7ac61c374a8",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5003319301,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:33.335802Z",
+          "span_id": "CD99B62403B9056D",
+          "trace_id": "07402476-3308-4282-7202-d7ac61c374a8",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 5001314000,
+          "parent_span_id": "B468FAAC4680C389",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:32.522281159Z",
+      "trace_id": "c9e824bf-ee76-2547-ef6a-5b58c5ae359a",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 360049289,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:32.522281159Z",
+          "span_id": "C533EC62FBCF3BC7",
+          "trace_id": "c9e824bf-ee76-2547-ef6a-5b58c5ae359a",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 360049289,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:32.523484Z",
+          "span_id": "FE66B73C5DA0E3EB",
+          "trace_id": "c9e824bf-ee76-2547-ef6a-5b58c5ae359a",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 356974000,
+          "parent_span_id": "C533EC62FBCF3BC7",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:31.91562832Z",
+      "trace_id": "78d2a960-e458-f71e-6c25-5f51a6ca1cc9",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 966326342,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:31.91562832Z",
+          "span_id": "ECF63D15580C0433",
+          "trace_id": "78d2a960-e458-f71e-6c25-5f51a6ca1cc9",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 966326342,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:31.916657Z",
+          "span_id": "DBC990DA6678F173",
+          "trace_id": "78d2a960-e458-f71e-6c25-5f51a6ca1cc9",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 963854000,
+          "parent_span_id": "ECF63D15580C0433",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:30.395858139Z",
+      "trace_id": "4f384bd2-f5ac-2a0c-6fdc-39bc3c7e02de",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 5004048334,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:30.395858139Z",
+          "span_id": "9AA29E279BDA0933",
+          "trace_id": "4f384bd2-f5ac-2a0c-6fdc-39bc3c7e02de",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5004048334,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:30.397063Z",
+          "span_id": "0517F682FD889ADA",
+          "trace_id": "4f384bd2-f5ac-2a0c-6fdc-39bc3c7e02de",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 5001512000,
+          "parent_span_id": "9AA29E279BDA0933",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:30.277148086Z",
+      "trace_id": "7f56682e-e0db-779f-7466-c94312aab3f9",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 1534282843,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:30.277148086Z",
+          "span_id": "CDDA802ECF692034",
+          "trace_id": "7f56682e-e0db-779f-7466-c94312aab3f9",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 1534282843,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:30.278435Z",
+          "span_id": "73454E596A9A590A",
+          "trace_id": "7f56682e-e0db-779f-7466-c94312aab3f9",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 1529627000,
+          "parent_span_id": "CDDA802ECF692034",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:30.212587976Z",
+      "trace_id": "793a243b-5584-7f2c-2fbe-017e8cd30d17",
+      "service_name": "featureflagservice",
+      "operation": "/featureflags",
+      "statusCode": "STATUS_CODE_UNSET",
+      "duration_nano": 2107878,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:30.212587976Z",
+          "span_id": "898FD4D39776E8C9",
+          "trace_id": "793a243b-5584-7f2c-2fbe-017e8cd30d17",
+          "service_name": "featureflagservice",
+          "operation": "/featureflags",
+          "duration_nano": 2107878,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_UNSET"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:30.212975003Z",
+          "span_id": "170E03EA6E912603",
+          "trace_id": "793a243b-5584-7f2c-2fbe-017e8cd30d17",
+          "service_name": "featureflagservice",
+          "operation": "featureflagservice.repo.query:featureflags",
+          "duration_nano": 1069751,
+          "parent_span_id": "898FD4D39776E8C9",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:28.934794988Z",
+      "trace_id": "6d24a1b0-3e82-431a-0b55-4a0c84395edd",
+      "service_name": "frontend-proxy",
+      "operation": "ingress",
+      "statusCode": "STATUS_CODE_UNSET",
+      "duration_nano": 4999844000,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:28.935829Z",
+          "span_id": "F5698F0154052EDE",
+          "trace_id": "6d24a1b0-3e82-431a-0b55-4a0c84395edd",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 4999844000,
+          "parent_span_id": "AC5B5A6B877CC28E",
+          "statusCode": "STATUS_CODE_UNSET"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:28.934794988Z",
+          "span_id": "AC5B5A6B877CC28E",
+          "trace_id": "6d24a1b0-3e82-431a-0b55-4a0c84395edd",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5002384628,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:27.880531908Z",
+      "trace_id": "e8db847b-d5b8-08ff-b1f8-e2d8edf539c9",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 2975295439,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:27.880531908Z",
+          "span_id": "8C0FFEF3FB76814D",
+          "trace_id": "e8db847b-d5b8-08ff-b1f8-e2d8edf539c9",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 2975295439,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:27.881633Z",
+          "span_id": "1A6E7E04D25BA6E1",
+          "trace_id": "e8db847b-d5b8-08ff-b1f8-e2d8edf539c9",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 2971951000,
+          "parent_span_id": "8C0FFEF3FB76814D",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:27.633202774Z",
+      "trace_id": "05b75faa-e4a4-77cb-988a-8dd1865b7a43",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 4177888028,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:27.633202774Z",
+          "span_id": "1F1C84D589ED847E",
+          "trace_id": "05b75faa-e4a4-77cb-988a-8dd1865b7a43",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 4177888028,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:27.634279Z",
+          "span_id": "119F25365FBA8469",
+          "trace_id": "05b75faa-e4a4-77cb-988a-8dd1865b7a43",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 4173865000,
+          "parent_span_id": "1F1C84D589ED847E",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:26.806916235Z",
+      "trace_id": "ffb38eb4-f31f-242a-c4b0-7d8fe6cf80e2",
+      "service_name": "loadgenerator",
+      "operation": "POST",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 5003612673,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:26.806916235Z",
+          "span_id": "646E465225A0239A",
+          "trace_id": "ffb38eb4-f31f-242a-c4b0-7d8fe6cf80e2",
+          "service_name": "loadgenerator",
+          "operation": "POST",
+          "duration_nano": 5003612673,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:26.807814Z",
+          "span_id": "65C72D731F9D16B2",
+          "trace_id": "ffb38eb4-f31f-242a-c4b0-7d8fe6cf80e2",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 5000389000,
+          "parent_span_id": "646E465225A0239A",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:26.451461426Z",
+      "trace_id": "967a9424-3b63-74e8-139b-ce5a91e8d931",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 354667116,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:26.451461426Z",
+          "span_id": "B3DB071411AE52F1",
+          "trace_id": "967a9424-3b63-74e8-139b-ce5a91e8d931",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 354667116,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:26.452419Z",
+          "span_id": "15D755DD09F55533",
+          "trace_id": "967a9424-3b63-74e8-139b-ce5a91e8d931",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 352693000,
+          "parent_span_id": "B3DB071411AE52F1",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:25.851421608Z",
+      "trace_id": "aa447bd7-a854-0914-39f3-3e6b4eae200d",
+      "service_name": "loadgenerator",
+      "operation": "POST",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 5003869067,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:25.851421608Z",
+          "span_id": "8E179C425955A841",
+          "trace_id": "aa447bd7-a854-0914-39f3-3e6b4eae200d",
+          "service_name": "loadgenerator",
+          "operation": "POST",
+          "duration_nano": 5003869067,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:25.852466Z",
+          "span_id": "8618BE9F3CBBF905",
+          "trace_id": "aa447bd7-a854-0914-39f3-3e6b4eae200d",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 5001183000,
+          "parent_span_id": "8E179C425955A841",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:25.312155662Z",
+      "trace_id": "0614bd44-0e43-d66c-146d-1d299e5fcd82",
+      "service_name": "loadgenerator",
+      "operation": "POST",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 538595527,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:25.312155662Z",
+          "span_id": "AA6ACF1D3F8E54EC",
+          "trace_id": "0614bd44-0e43-d66c-146d-1d299e5fcd82",
+          "service_name": "loadgenerator",
+          "operation": "POST",
+          "duration_nano": 538595527,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:25.31465Z",
+          "span_id": "68F9E651179E3EEA",
+          "trace_id": "0614bd44-0e43-d66c-146d-1d299e5fcd82",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 534790000,
+          "parent_span_id": "AA6ACF1D3F8E54EC",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:23.696615328Z",
+      "trace_id": "7121e26a-485c-b049-ad6b-51019a56f829",
+      "service_name": "frontend-proxy",
+      "operation": "ingress",
+      "statusCode": "STATUS_CODE_UNSET",
+      "duration_nano": 5000380000,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:23.697912Z",
+          "span_id": "C295CA98D0119566",
+          "trace_id": "7121e26a-485c-b049-ad6b-51019a56f829",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 5000380000,
+          "parent_span_id": "8A4EF2E93FCFFAC9",
+          "statusCode": "STATUS_CODE_UNSET"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:23.696615328Z",
+          "span_id": "8A4EF2E93FCFFAC9",
+          "trace_id": "7121e26a-485c-b049-ad6b-51019a56f829",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5003136653,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:23.2483005Z",
+      "trace_id": "de01ef59-6cff-058c-bf1d-0128f8b3d0be",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 5000646894,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:23.2483005Z",
+          "span_id": "FC7815D4D1991E30",
+          "trace_id": "de01ef59-6cff-058c-bf1d-0128f8b3d0be",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 5000646894,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:23.249229Z",
+          "span_id": "D154D142A462CD2D",
+          "trace_id": "de01ef59-6cff-058c-bf1d-0128f8b3d0be",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 4998661000,
+          "parent_span_id": "FC7815D4D1991E30",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:22.502517932Z",
+      "trace_id": "b42db0ac-e364-36a0-56d1-23d3ab3e9fa4",
+      "service_name": "loadgenerator",
+      "operation": "POST",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 5005224271,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:22.502517932Z",
+          "span_id": "3B9A9C3DDAA69C04",
+          "trace_id": "b42db0ac-e364-36a0-56d1-23d3ab3e9fa4",
+          "service_name": "loadgenerator",
+          "operation": "POST",
+          "duration_nano": 5005224271,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:22.503613Z",
+          "span_id": "C1DBAEA069A4EFD2",
+          "trace_id": "b42db0ac-e364-36a0-56d1-23d3ab3e9fa4",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 5002788000,
+          "parent_span_id": "3B9A9C3DDAA69C04",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:22.307362405Z",
+      "trace_id": "070f61c7-5c98-6421-152b-9102f0531a7f",
+      "service_name": "loadgenerator",
+      "operation": "POST",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 3006936837,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:22.307362405Z",
+          "span_id": "F5CC6AB8847A71E9",
+          "trace_id": "070f61c7-5c98-6421-152b-9102f0531a7f",
+          "service_name": "loadgenerator",
+          "operation": "POST",
+          "duration_nano": 3006936837,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:22.308517Z",
+          "span_id": "552A72023A433574",
+          "trace_id": "070f61c7-5c98-6421-152b-9102f0531a7f",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 3002823000,
+          "parent_span_id": "F5CC6AB8847A71E9",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    },
+    {
+      "timestamp": "2023-10-06T11:55:21.805009204Z",
+      "trace_id": "12a263fc-4e09-6a7f-909f-4045830922a8",
+      "service_name": "loadgenerator",
+      "operation": "GET",
+      "statusCode": "STATUS_CODE_ERROR",
+      "duration_nano": 502716963,
+      "spans": [
+        {
+          "timestamp": "2023-10-06T11:55:21.805009204Z",
+          "span_id": "E5ACBF6213EEE7AF",
+          "trace_id": "12a263fc-4e09-6a7f-909f-4045830922a8",
+          "service_name": "loadgenerator",
+          "operation": "GET",
+          "duration_nano": 502716963,
+          "parent_span_id": "",
+          "statusCode": "STATUS_CODE_ERROR"
+        },
+        {
+          "timestamp": "2023-10-06T11:55:21.8059Z",
+          "span_id": "BC31EB878CEEF3BB",
+          "trace_id": "12a263fc-4e09-6a7f-909f-4045830922a8",
+          "service_name": "frontend-proxy",
+          "operation": "ingress",
+          "duration_nano": 498294000,
+          "parent_span_id": "E5ACBF6213EEE7AF",
+          "statusCode": "STATUS_CODE_UNSET"
+        }
+      ],
+      "totalSpans": 2
+    }
+  ],
+  "total_traces": 50,
+  "next_page_token": "eyJzdGFydF90aW1lc3RhbXAiOiIyMDIzLTEwLTA2IDExOjU1OjIxLjgwNTAwOTIwNCJ9"
+}
  • Check the console and confirm that the service_name is appended to the query

MR acceptance checklist

This checklist encourages us to confirm any changes have been analyzed to reduce risks in quality, performance, reliability, security, and maintainability.

Edited by Daniele Rossetti

Merge request reports

Loading