feat: add service discovery, OTel instrumentation, and k6 load tests (#1)

overlays/deploy/k6-configmap.yaml

@@ -0,0 +1,59 @@
+# FALLBACK ConfigMap.
+# This static skeleton is only used when The Watcher agent fails to generate
+# a bespoke k6 script. When generation succeeds, the agent produces a custom
+# ConfigMap containing the tailored load-test.js that replaces this file in
+# the scaffolded output repository.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: k6-test-${{ values.component_id }}
+  namespace: ${{ values.destination_namespace }}
+  labels:
+    app: ${{ values.component_id }}
+    app.kubernetes.io/managed-by: backstage
+    app.kubernetes.io/component: load-testing
+data:
+  K6_OUT: "opentelemetry"
+  K6_OTEL_GRPC_EXPORTER_INSECURE: "true"
+  K6_OTEL_GRPC_EXPORTER_ENDPOINT: "otel-collector.monitoring.svc.cluster.local:4317"
+  K6_OTEL_METRIC_PREFIX: "k6_"
+  K6_OTEL_FLUSH_INTERVAL: "1000"
+  K6_OTEL_EXPORT_INTERVAL: "5000"
+  K6_OTEL_SERVICE_NAME: "k6-${{ values.component_id }}"
+  load-test.js: |
+    import http from 'k6/http';
+    import { check, sleep } from 'k6';
+
+    const vus = parseInt(__ENV.TEST_VUS || '10');
+    const duration = __ENV.TEST_DURATION || '30s';
+    const targetUrl = __ENV.TARGET_URL || 'http://localhost';
+
+    export const options = {
+      scenarios: {
+        load_test: {
+          executor: 'ramping-vus',
+          startVUs: 0,
+          stages: [
+            { duration: '10s', target: vus },
+            { duration: duration, target: vus },
+            { duration: '5s', target: 0 },
+          ],
+        },
+      },
+      thresholds: {
+        http_req_duration: ['p(95)<500'],
+        http_req_failed: ['rate<0.075'],
+      },
+    };
+
+    // Treat 2xx and 3xx responses as expected (redirects won't inflate http_req_failed)
+    http.setResponseCallback(http.expectedStatuses({ min: 200, max: 399 }));
+
+    export default function () {
+      const res = http.get(targetUrl);
+      check(res, {
+        'status is 200': (r) => r.status === 200,
+        'response time < 500ms': (r) => r.timings.duration < 500,
+      });
+      sleep(0.5);
+    }

overlays/otel/kustomization.yaml

@@ -0,0 +1,119 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../deploy
+patches:
+- target:
+    kind: Deployment
+    name: ad
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: ad\n  namespace:\
+    \ default\nspec:\n  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-java:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ ad\n        env:\n        - name: OTEL_SERVICE_NAME\n          value: ad\n \
+    \       - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=lint-enforcement-v2\n"
+- target:
+    kind: Deployment
+    name: email
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: email\n  namespace:\
+    \ default\nspec:\n  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-ruby:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ email\n        env:\n        - name: OTEL_SERVICE_NAME\n          value: email\n\
+    \        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=lint-enforcement-v2\n"
+- target:
+    kind: Deployment
+    name: fraud-detection
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: fraud-detection\n\
+    \  namespace: default\nspec:\n  template:\n    metadata:\n      annotations:\n\
+    \        instrumentation.opentelemetry.io/inject-java: monitoring/otel-instrumentation\n\
+    \    spec:\n      containers:\n      - name: fraud-detection\n        env:\n \
+    \       - name: OTEL_SERVICE_NAME\n          value: fraud-detection\n        -\
+    \ name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=lint-enforcement-v2\n"
+- target:
+    kind: Deployment
+    name: frontend
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: frontend\n  namespace:\
+    \ default\nspec:\n  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-nodejs:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ frontend\n        env:\n        - name: OTEL_SERVICE_NAME\n          value:\
+    \ frontend\n        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=lint-enforcement-v2\n"
+- target:
+    kind: Deployment
+    name: kafka
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: kafka\n  namespace:\
+    \ default\nspec:\n  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-python:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ kafka\n        env:\n        - name: OTEL_SERVICE_NAME\n          value: kafka\n\
+    \        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=lint-enforcement-v2\n"
+- target:
+    kind: Deployment
+    name: llm
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: llm\n  namespace:\
+    \ default\nspec:\n  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-python:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ llm\n        env:\n        - name: OTEL_SERVICE_NAME\n          value: llm\n\
+    \        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=lint-enforcement-v2\n"
+- target:
+    kind: Deployment
+    name: load-generator
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: load-generator\n\
+    \  namespace: default\nspec:\n  template:\n    metadata:\n      annotations:\n\
+    \        instrumentation.opentelemetry.io/inject-python: monitoring/otel-instrumentation\n\
+    \    spec:\n      containers:\n      - name: load-generator\n        env:\n  \
+    \      - name: OTEL_SERVICE_NAME\n          value: load-generator\n        - name:\
+    \ OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=lint-enforcement-v2\n"
+- target:
+    kind: Deployment
+    name: payment
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: payment\n  namespace:\
+    \ default\nspec:\n  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-nodejs:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ payment\n        env:\n        - name: OTEL_SERVICE_NAME\n          value: payment\n\
+    \        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=lint-enforcement-v2\n"
+- target:
+    kind: Deployment
+    name: product-reviews
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: product-reviews\n\
+    \  namespace: default\nspec:\n  template:\n    metadata:\n      annotations:\n\
+    \        instrumentation.opentelemetry.io/inject-python: monitoring/otel-instrumentation\n\
+    \    spec:\n      containers:\n      - name: product-reviews\n        env:\n \
+    \       - name: OTEL_SERVICE_NAME\n          value: product-reviews\n        -\
+    \ name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=lint-enforcement-v2\n"
+- target:
+    kind: Deployment
+    name: recommendation
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: recommendation\n\
+    \  namespace: default\nspec:\n  template:\n    metadata:\n      annotations:\n\
+    \        instrumentation.opentelemetry.io/inject-python: monitoring/otel-instrumentation\n\
+    \    spec:\n      containers:\n      - name: recommendation\n        env:\n  \
+    \      - name: OTEL_SERVICE_NAME\n          value: recommendation\n        - name:\
+    \ OTEL_EXPORTER_OTLP_ENDPOINT\n          value: http://otel-collector.monitoring.svc.cluster.local:4318\n\
+    \        - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=lint-enforcement-v2\n"
+- target:
+    kind: Deployment
+    name: opensearch
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: opensearch\n \
+    \ namespace: default\nspec:\n  template:\n    metadata:\n      annotations:\n\
+    \        instrumentation.opentelemetry.io/inject-java: monitoring/otel-instrumentation\n\
+    \    spec:\n      containers:\n      - name: opensearch\n        env:\n      \
+    \  - name: OTEL_SERVICE_NAME\n          value: opensearch\n        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n\
+    \          value: http://otel-collector.monitoring.svc.cluster.local:4318\n  \
+    \      - name: OTEL_RESOURCE_ATTRIBUTES\n          value: app=lint-enforcement-v2\n"
+- target:
+    kind: Deployment
+    name: grafana
+  patch: "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: grafana\n  namespace:\
+    \ default\nspec:\n  template:\n    metadata:\n      annotations:\n        instrumentation.opentelemetry.io/inject-python:\
+    \ monitoring/otel-instrumentation\n    spec:\n      containers:\n      - name:\
+    \ grafana-sc-alerts\n        env:\n        - name: OTEL_SERVICE_NAME\n       \
+    \   value: grafana\n        - name: OTEL_EXPORTER_OTLP_ENDPOINT\n          value:\
+    \ http://otel-collector.monitoring.svc.cluster.local:4318\n        - name: OTEL_RESOURCE_ATTRIBUTES\n\
+    \          value: app=lint-enforcement-v2\n"

overlays/otel/patches/otel-patch.yaml

@@ -0,0 +1,251 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ad
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-java: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: ad
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: ad
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=lint-enforcement-v2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: email
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-ruby: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: email
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: email
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=lint-enforcement-v2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: fraud-detection
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-java: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: fraud-detection
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: fraud-detection
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=lint-enforcement-v2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: frontend
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-nodejs: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: frontend
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: frontend
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=lint-enforcement-v2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kafka
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-python: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: kafka
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: kafka
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=lint-enforcement-v2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: llm
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-python: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: llm
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: llm
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=lint-enforcement-v2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: load-generator
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-python: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: load-generator
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: load-generator
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=lint-enforcement-v2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: payment
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-nodejs: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: payment
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: payment
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=lint-enforcement-v2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: product-reviews
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-python: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: product-reviews
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: product-reviews
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=lint-enforcement-v2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: recommendation
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-python: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: recommendation
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: recommendation
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=lint-enforcement-v2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: opensearch
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-java: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: opensearch
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: opensearch
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=lint-enforcement-v2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        instrumentation.opentelemetry.io/inject-python: monitoring/otel-instrumentation
+    spec:
+      containers:
+      - name: grafana-sc-alerts
+        env:
+        - name: OTEL_SERVICE_NAME
+          value: grafana
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: http://otel-collector.monitoring.svc.cluster.local:4318
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: app=lint-enforcement-v2