feat: add service discovery, OTel instrumentation, and k6 load tests (#1)

2026-06-02 15:42:52 +00:00
parent 3f54964f18
commit d932ec3e5e
6 changed files with 322 additions and 0 deletions
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -139,3 +139,161 @@ spec:

 # ─── Per-service Components (from Watcher discovery) ─────────────────

+
+---
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: demo-db
+  description: "demo-db service — part of petclinic"
+  labels:
+    backstage.io/environment: "dev"
+    app.kubernetes.io/managed-by: "backstage"
+  tags:
+    - deployment
+    - argocd
+    
+    - opentelemetry
+    
+    - load-testing
+    - k6
+    
+    - chaos-engineering
+    - chaos-mesh
+    
+    
+  annotations:
+    argocd/app-name: "petclinic"
+    argocd/app-namespace: "argocd"
+    argocd/instance-name: "argocd"
+    backstage.io/source-location: "url:https://gitea.kyndemo.live/validate/petclinic/src/branch/main"
+    backstage.io/kubernetes-namespace: "demo-apps"
+    backstage.io/kubernetes-label-selector: "app=demo-db"
+    backstage.io/techdocs-ref: dir:.
+    gitea.kyndemo.live/repo-slug: "validate/petclinic"
+    sonarqube.org/project-key: petclinic
+    
+    grafana/grafana-instance: "default"
+    grafana/alert-label-selector: "app=demo-db"
+    grafana/dashboard-selector: "uid == 'otel-app-observability-v2'"
+    grafana.com/alert-label-selector: "app=demo-db"
+    grafana.com/dashboard-url: "https://grafana.kyndemo.live/d/otel-app-observability-v2/opentelemetry-application-observability?orgId=1&var-service=demo-db"
+    
+    k6/enabled: "true"
+    k6/test-configmap: "k6-test-petclinic"
+    k6/test-namespace: "demo-apps"
+    k6/target-service: "demo-db"
+    
+    chaos-mesh/enabled: "true"
+    
+  links:
+    - url: https://petclinic.kyndemo.live
+      title: Live Application
+      icon: web
+    - url: https://gitea.kyndemo.live/validate/petclinic
+      title: Repository
+      icon: github
+    - url: https://argocd.kyndemo.live/applications/petclinic
+      title: ArgoCD App
+      icon: dashboard
+    
+    - url: https://grafana.kyndemo.live/d/otel-app-observability-v2/opentelemetry-application-observability?orgId=1&var-service=demo-db
+      title: Grafana Dashboard
+      icon: dashboard
+    
+spec:
+  type: service
+  owner: "platform-engineering"
+  lifecycle: experimental
+
+  system: petclinic
+  dependsOn:
+    - component:default/argocd-service
+    
+    - resource:default/veterinary-platform
+    
+    
+    - resource:default/otel-collector
+    
+    - resource:default/k6-operator
+
+---
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: petclinic
+  description: "petclinic service — part of petclinic"
+  labels:
+    backstage.io/environment: "dev"
+    app.kubernetes.io/managed-by: "backstage"
+  tags:
+    - deployment
+    - argocd
+    
+    - opentelemetry
+    
+    - load-testing
+    - k6
+    
+    - chaos-engineering
+    - chaos-mesh
+    
+    
+    - java
+    
+  annotations:
+    argocd/app-name: "petclinic"
+    argocd/app-namespace: "argocd"
+    argocd/instance-name: "argocd"
+    backstage.io/source-location: "url:https://gitea.kyndemo.live/validate/petclinic/src/branch/main"
+    backstage.io/kubernetes-namespace: "demo-apps"
+    backstage.io/kubernetes-label-selector: "app=petclinic"
+    backstage.io/techdocs-ref: dir:.
+    gitea.kyndemo.live/repo-slug: "validate/petclinic"
+    sonarqube.org/project-key: petclinic
+    
+    grafana/grafana-instance: "default"
+    grafana/alert-label-selector: "app=petclinic"
+    grafana/dashboard-selector: "uid == 'otel-app-observability-v2'"
+    grafana.com/alert-label-selector: "app=petclinic"
+    grafana.com/dashboard-url: "https://grafana.kyndemo.live/d/otel-app-observability-v2/opentelemetry-application-observability?orgId=1&var-service=petclinic"
+    
+    k6/enabled: "true"
+    k6/test-configmap: "k6-test-petclinic"
+    k6/test-namespace: "demo-apps"
+    k6/target-service: "petclinic"
+    
+    chaos-mesh/enabled: "true"
+    
+  links:
+    - url: https://petclinic.kyndemo.live
+      title: Live Application
+      icon: web
+    - url: https://gitea.kyndemo.live/validate/petclinic
+      title: Repository
+      icon: github
+    - url: https://argocd.kyndemo.live/applications/petclinic
+      title: ArgoCD App
+      icon: dashboard
+    
+    - url: https://grafana.kyndemo.live/d/otel-app-observability-v2/opentelemetry-application-observability?orgId=1&var-service=petclinic
+      title: Grafana Dashboard
+      icon: dashboard
+    
+spec:
+  type: service
+  owner: "platform-engineering"
+  lifecycle: experimental
+
+  system: petclinic
+  dependsOn:
+    - component:default/argocd-service
+    
+    - resource:default/veterinary-platform
+    
+    
+    - resource:default/otel-collector
+    
+    - resource:default/k6-operator
+
+
--- a/k6/load-test.js
+++ b/k6/load-test.js
@@ -0,0 +1,36 @@
+// FALLBACK k6 load-test script.
+// This static skeleton is only used when The Watcher agent fails to generate
+// a bespoke k6 script tailored to the application's detected HTTP endpoints.
+// When generation succeeds, the agent produces a custom script that replaces
+// this file in the scaffolded output repository.
+import http from 'k6/http';
+import { check, sleep } from 'k6';
+
+export const options = {
+  scenarios: {
+    load_test: {
+      executor: 'ramping-vus',
+      startVUs: 0,
+      stages: [
+        { duration: '${{ values.k6_ramp_up | default("10s") }}', target: ${{ values.k6_virtual_users | default(10) }} },
+        { duration: '${{ values.k6_duration | default("30s") }}', target: ${{ values.k6_virtual_users | default(10) }} },
+        { duration: '5s', target: 0 },
+      ],
+    },
+  },
+  thresholds: {
+    http_req_duration: ['p(95)<500'],
+    http_req_failed: ['rate<0.01'],
+  },
+};
+
+const BASE_URL = `http://${{ values.frontend_service_name | default("frontend") }}.${{ values.destination_namespace }}.svc.cluster.local:${{ values.frontend_service_port | default(80) }}`;
+
+export default function () {
+  const res = http.get(`${BASE_URL}${{ values.k6_target_path | default("/") }}`);
+  check(res, {
+    'status is 200': (r) => r.status === 200,
+    'response time < 500ms': (r) => r.timings.duration < 500,
+  });
+  sleep(0.5);
+}
--- a/k6/testrun.yaml
+++ b/k6/testrun.yaml
@@ -0,0 +1,36 @@
+# FALLBACK k6 TestRun CRD — reference template for load testing ${{ values.component_id }}.
+# This static skeleton is only used when The Watcher agent fails to generate
+# a bespoke k6 script. When generation succeeds, the agent produces a custom
+# TestRun CRD that replaces this file in the scaffolded output repository.
+#
+# TestRun CRDs are committed to the repo as a reference. They are created
+# dynamically from Backstage (not auto-synced by ArgoCD) because they are
+# ephemeral one-shot resources.
+apiVersion: k6.io/v1alpha1
+kind: TestRun
+metadata:
+  name: k6-${{ values.component_id }}
+  namespace: ${{ values.destination_namespace }}
+  labels:
+    app: ${{ values.component_id }}
+    backstage.io/component: ${{ values.component_id }}
+    app.kubernetes.io/managed-by: backstage
+    app.kubernetes.io/component: load-testing
+spec:
+  parallelism: 1
+  script:
+    configMap:
+      name: k6-test-${{ values.component_id }}
+      file: load-test.js
+  runner:
+    image: grafana/k6:latest
+    envFrom:
+      - configMapRef:
+          name: k6-test-${{ values.component_id }}
+    env:
+      - name: K6_OTEL_SERVICE_NAME
+        value: k6-${{ values.component_id }}
+      - name: TEST_VUS
+        value: "10"
+      - name: TEST_DURATION
+        value: "30s"
--- a/overlays/deploy/k6-configmap.yaml
+++ b/overlays/deploy/k6-configmap.yaml
@@ -0,0 +1,59 @@
+# FALLBACK ConfigMap.
+# This static skeleton is only used when The Watcher agent fails to generate
+# a bespoke k6 script. When generation succeeds, the agent produces a custom
+# ConfigMap containing the tailored load-test.js that replaces this file in
+# the scaffolded output repository.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: k6-test-${{ values.component_id }}
+  namespace: ${{ values.destination_namespace }}
+  labels:
+    app: ${{ values.component_id }}
+    app.kubernetes.io/managed-by: backstage
+    app.kubernetes.io/component: load-testing
+data:
+  K6_OUT: "opentelemetry"
+  K6_OTEL_GRPC_EXPORTER_INSECURE: "true"
+  K6_OTEL_GRPC_EXPORTER_ENDPOINT: "otel-collector.monitoring.svc.cluster.local:4317"
+  K6_OTEL_METRIC_PREFIX: "k6_"
+  K6_OTEL_FLUSH_INTERVAL: "1000"
+  K6_OTEL_EXPORT_INTERVAL: "5000"
+  K6_OTEL_SERVICE_NAME: "k6-${{ values.component_id }}"
+  load-test.js: |
+    import http from 'k6/http';
+    import { check, sleep } from 'k6';
+
+    const vus = parseInt(__ENV.TEST_VUS || '10');
+    const duration = __ENV.TEST_DURATION || '30s';
+    const targetUrl = __ENV.TARGET_URL || 'http://localhost';
+
+    export const options = {
+      scenarios: {
+        load_test: {
+          executor: 'ramping-vus',
+          startVUs: 0,
+          stages: [
+            { duration: '10s', target: vus },
+            { duration: duration, target: vus },
+            { duration: '5s', target: 0 },
+          ],
+        },
+      },
+      thresholds: {
+        http_req_duration: ['p(95)<500'],
+        http_req_failed: ['rate<0.075'],
+      },
+    };
+
+    // Treat 2xx and 3xx responses as expected (redirects won't inflate http_req_failed)
+    http.setResponseCallback(http.expectedStatuses({ min: 200, max: 399 }));
+
+    export default function () {
+      const res = http.get(targetUrl);
+      check(res, {
+        'status is 200': (r) => r.status === 200,
+        'response time < 500ms': (r) => r.timings.duration < 500,
+      });
+      sleep(0.5);
+    }
--- a/overlays/otel/kustomization.yaml
+++ b/overlays/otel/kustomization.yaml
@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../deploy
+patches:
+- target:
+    kind: Deployment
+    name: petclinic
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: petclinic\nspec:\n\
+    \  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-java:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ workload\n        env:\n        - name: OTEL_SERVICE_NAME\n          value:\
+    \ petclinic\n        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=petclinic\n"
--- a/overlays/otel/patches/otel-patch.yaml
+++ b/overlays/otel/patches/otel-patch.yaml
@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: petclinic
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-java: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: workload
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: petclinic
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=petclinic