feat: add service discovery, OTel instrumentation, and k6 load tests (#1)

k6/load-test.js

@@ -0,0 +1,36 @@
+// FALLBACK k6 load-test script.
+// This static skeleton is only used when The Watcher agent fails to generate
+// a bespoke k6 script tailored to the application's detected HTTP endpoints.
+// When generation succeeds, the agent produces a custom script that replaces
+// this file in the scaffolded output repository.
+import http from 'k6/http';
+import { check, sleep } from 'k6';
+
+export const options = {
+  scenarios: {
+    load_test: {
+      executor: 'ramping-vus',
+      startVUs: 0,
+      stages: [
+        { duration: '${{ values.k6_ramp_up | default("10s") }}', target: ${{ values.k6_virtual_users | default(10) }} },
+        { duration: '${{ values.k6_duration | default("30s") }}', target: ${{ values.k6_virtual_users | default(10) }} },
+        { duration: '5s', target: 0 },
+      ],
+    },
+  },
+  thresholds: {
+    http_req_duration: ['p(95)<500'],
+    http_req_failed: ['rate<0.01'],
+  },
+};
+
+const BASE_URL = `http://${{ values.frontend_service_name | default("frontend") }}.${{ values.destination_namespace }}.svc.cluster.local:${{ values.frontend_service_port | default(80) }}`;
+
+export default function () {
+  const res = http.get(`${BASE_URL}${{ values.k6_target_path | default("/") }}`);
+  check(res, {
+    'status is 200': (r) => r.status === 200,
+    'response time < 500ms': (r) => r.timings.duration < 500,
+  });
+  sleep(0.5);
+}

k6/testrun.yaml

@@ -0,0 +1,36 @@
+# FALLBACK k6 TestRun CRD — reference template for load testing ${{ values.component_id }}.
+# This static skeleton is only used when The Watcher agent fails to generate
+# a bespoke k6 script. When generation succeeds, the agent produces a custom
+# TestRun CRD that replaces this file in the scaffolded output repository.
+#
+# TestRun CRDs are committed to the repo as a reference. They are created
+# dynamically from Backstage (not auto-synced by ArgoCD) because they are
+# ephemeral one-shot resources.
+apiVersion: k6.io/v1alpha1
+kind: TestRun
+metadata:
+  name: k6-${{ values.component_id }}
+  namespace: ${{ values.destination_namespace }}
+  labels:
+    app: ${{ values.component_id }}
+    backstage.io/component: ${{ values.component_id }}
+    app.kubernetes.io/managed-by: backstage
+    app.kubernetes.io/component: load-testing
+spec:
+  parallelism: 1
+  script:
+    configMap:
+      name: k6-test-${{ values.component_id }}
+      file: load-test.js
+  runner:
+    image: grafana/k6:latest
+    envFrom:
+      - configMapRef:
+          name: k6-test-${{ values.component_id }}
+    env:
+      - name: K6_OTEL_SERVICE_NAME
+        value: k6-${{ values.component_id }}
+      - name: TEST_VUS
+        value: "10"
+      - name: TEST_DURATION
+        value: "30s"

overlays/deploy/k6-configmap.yaml

@@ -0,0 +1,59 @@
+# FALLBACK ConfigMap.
+# This static skeleton is only used when The Watcher agent fails to generate
+# a bespoke k6 script. When generation succeeds, the agent produces a custom
+# ConfigMap containing the tailored load-test.js that replaces this file in
+# the scaffolded output repository.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: k6-test-${{ values.component_id }}
+  namespace: ${{ values.destination_namespace }}
+  labels:
+    app: ${{ values.component_id }}
+    app.kubernetes.io/managed-by: backstage
+    app.kubernetes.io/component: load-testing
+data:
+  K6_OUT: "opentelemetry"
+  K6_OTEL_GRPC_EXPORTER_INSECURE: "true"
+  K6_OTEL_GRPC_EXPORTER_ENDPOINT: "otel-collector.monitoring.svc.cluster.local:4317"
+  K6_OTEL_METRIC_PREFIX: "k6_"
+  K6_OTEL_FLUSH_INTERVAL: "1000"
+  K6_OTEL_EXPORT_INTERVAL: "5000"
+  K6_OTEL_SERVICE_NAME: "k6-${{ values.component_id }}"
+  load-test.js: |
+    import http from 'k6/http';
+    import { check, sleep } from 'k6';
+
+    const vus = parseInt(__ENV.TEST_VUS || '10');
+    const duration = __ENV.TEST_DURATION || '30s';
+    const targetUrl = __ENV.TARGET_URL || 'http://localhost';
+
+    export const options = {
+      scenarios: {
+        load_test: {
+          executor: 'ramping-vus',
+          startVUs: 0,
+          stages: [
+            { duration: '10s', target: vus },
+            { duration: duration, target: vus },
+            { duration: '5s', target: 0 },
+          ],
+        },
+      },
+      thresholds: {
+        http_req_duration: ['p(95)<500'],
+        http_req_failed: ['rate<0.075'],
+      },
+    };
+
+    // Treat 2xx and 3xx responses as expected (redirects won't inflate http_req_failed)
+    http.setResponseCallback(http.expectedStatuses({ min: 200, max: 399 }));
+
+    export default function () {
+      const res = http.get(targetUrl);
+      check(res, {
+        'status is 200': (r) => r.status === 200,
+        'response time < 500ms': (r) => r.timings.duration < 500,
+      });
+      sleep(0.5);
+    }

overlays/otel/kustomization.yaml

@@ -0,0 +1,68 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../deploy
+patches:
+- target:
+    kind: Deployment
+    name: redis-cart
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: redis-cart\nspec:\n\
+    \  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-python:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ redis\n        env:\n        - name: OTEL_SERVICE_NAME\n          value: redis-cart\n\
+    \        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=security-scan-test\n"
+- target:
+    kind: Deployment
+    name: adservice
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: adservice\nspec:\n\
+    \  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-java:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ server\n        env:\n        - name: OTEL_SERVICE_NAME\n          value: adservice\n\
+    \        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=security-scan-test\n"
+- target:
+    kind: Deployment
+    name: currencyservice
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: currencyservice\n\
+    spec:\n  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-nodejs:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ server\n        env:\n        - name: OTEL_SERVICE_NAME\n          value: currencyservice\n\
+    \        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=security-scan-test\n"
+- target:
+    kind: Deployment
+    name: emailservice
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: emailservice\n\
+    spec:\n  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-python:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ server\n        env:\n        - name: OTEL_SERVICE_NAME\n          value: emailservice\n\
+    \        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=security-scan-test\n"
+- target:
+    kind: Deployment
+    name: loadgenerator
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: loadgenerator\n\
+    spec:\n  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-python:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ main\n        env:\n        - name: OTEL_SERVICE_NAME\n          value: loadgenerator\n\
+    \        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=security-scan-test\n"
+- target:
+    kind: Deployment
+    name: paymentservice
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: paymentservice\n\
+    spec:\n  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-nodejs:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ server\n        env:\n        - name: OTEL_SERVICE_NAME\n          value: paymentservice\n\
+    \        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=security-scan-test\n"
+- target:
+    kind: Deployment
+    name: recommendationservice
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: recommendationservice\n\
+    spec:\n  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-python:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ server\n        env:\n        - name: OTEL_SERVICE_NAME\n          value: recommendationservice\n\
+    \        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=security-scan-test\n"

overlays/otel/patches/otel-patch.yaml

@@ -0,0 +1,139 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis-cart
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-python: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: redis
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: redis-cart
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=security-scan-test
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: adservice
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-java: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: server
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: adservice
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=security-scan-test
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: currencyservice
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-nodejs: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: server
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: currencyservice
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=security-scan-test
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: emailservice
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-python: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: server
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: emailservice
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=security-scan-test
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: loadgenerator
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-python: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: main
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: loadgenerator
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=security-scan-test
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: paymentservice
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-nodejs: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: server
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: paymentservice
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=security-scan-test
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: recommendationservice
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-python: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: server
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: recommendationservice
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=security-scan-test