admin/k8s-infra-cluster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

setup k3s cluster

Browse Source
This commit is contained in:
wendi gunawan 2026-03-25 11:46:39 +01:00
commit 4119e44d83
363 changed files with 46488 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
.gitignore vendored Normal file
View File

3
ansible-k3s/ansible.cfg Normal file
View File

@ -0,0 +1,3 @@
[defaults]
host_key_checking = False
inventory = hosts.ini

37
ansible-k3s/hosts.ini Normal file
View File

@ -0,0 +1,37 @@
; [all:vars]
; ansible_user=ubuntu-s # using same user for all VMs
; # If you use the SSH key method above:
; # ansible_ssh_private_key_file=~/.ssh/id_rsa
; # using mounted private key from windows folder structure from does not work
; #ansible_ssh_private_key_file=/mnt/c/Users/wendg2/.ssh/id_ed25519
; ansible_ssh_private_key_file=~/.ssh/id_ed25519
; [master]
; 192.168.3.91
; [workers]
; 192.168.3.93
; 192.168.3.92
; 192.168.3.94
[master]
192.168.3.91 ansible_user=master-1
# different user name
[dbworkers]
192.168.3.92 ansible_user=master-2
192.168.3.93 ansible_user=master-3
192.168.3.94 ansible_user=master-4
[workers]
192.168.3.95 ansible_user=master-5
192.168.3.96 ansible_user=master-6
# This group combines both for easy targeting
[k8s_nodes:children]
master
workers
# [k8s_nodes:vars]
# # Ensure Ansible uses the correct Python on the remote nodes
# ansible_python_interpreter=/usr/bin/python3
# # If you use the same SSH key for all:
# ansible_ssh_private_key_file=~/.ssh/id_rsa

61
ansible-k3s/k3s-remote-control.yaml Normal file
View File

@ -0,0 +1,61 @@
---
# - name: Setup Local Laptop Tools
# hosts: localhost
# connection: local
# become: yes
# tasks:
# - name: Set ANSIBLE_CONFIG locally
# become: yes
# ansible.builtin.lineinfile:
# path: "~/.bashrc"
# line: 'export ANSIBLE_CONFIG="/mnt/f/Invixel/k8s-infra-cluster/ansible.cfg"'
# state: present
- name: Setup Laptop to Control K3s Cluster
hosts: master
become: yes
vars:
# Replace this with your Master's actual LAN or Tailscale IP
master_public_ip: "192.168.3.91"
local_kube_path: "~/.kube/config"
tasks:
- name: Ensure K3s is running with TLS SAN for the Public IP
ansible.builtin.shell: |
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--tls-san {{ master_public_ip }}" sh -
register: k3s_install
- name: Wait for k3s.yaml to be fully generated
ansible.builtin.wait_for:
path: /etc/rancher/k3s/k3s.yaml
search_regex: "certificate-authority-data" # Ensures the file isn't empty
timeout: 30
# - name: Read k3s.yaml as plain text
# ansible.builtin.command: cat /etc/rancher/k3s/k3s.yaml
# register: k3s_config_raw
# changed_when: false
# - name: Prepare and save config to local laptop
# delegate_to: localhost
# become: no
# ansible.builtin.copy:
# # We use .stdout here because that's where the 'cat' text is stored
# content: "{{ k3s_config_raw.stdout | replace('127.0.0.1', master_public_ip) }}"
# dest: "{{ local_kube_path }}"
# mode: '0600'
- name: Read k3s.yaml from master
ansible.builtin.slurp:
src: /etc/rancher/k3s/k3s.yaml
register: k3s_config_encoded
- name: Prepare and save config to local laptop
delegate_to: localhost
become: no
ansible.builtin.copy:
content: "{{ k3s_config_encoded.content | b64decode | replace('127.0.0.1', master_public_ip) }}"
dest: "{{ local_kube_path }}"
mode: '0600'
# ansible-playbook -i hosts.ini k3s-remote-control.yml

32
ansible-prepare-k3s.yml Normal file
View File

@ -0,0 +1,32 @@
---
- name: Prepare Nodes for Longhorn
hosts: k8s_nodes
become: yes
tasks:
- name: Install prerequisites
apt:
name: [open-iscsi, nfs-common, util-linux]
state: present
update_cache: yes
- name: Enable and start iscsid
systemd:
name: iscsid
enabled: yes
state: started
# - name: Deploy Infra and Router via Kustomize
# become: false
# shell: "kustomize build --enable-helm . | kubectl apply --server-side --force-conflicts -f -"
# args:
# chdir: "/mnt/f/Invixel/k8s-infra-cluster"
# - name: Install Cosmo Router via Helm
# become: false
# shell: "helm upgrade --install cosmo-router ./cosmo-router -n apps --create-namespace"
# args:
# chdir: "/mnt/f/Invixel/k8s-infra-cluster"
# ansible-playbook -i hosts.ini ansible-prepare-k3s.yml
# ansible k8s_nodes -i hosts.ini -m ping

3
ansible.cfg Normal file
View File

@ -0,0 +1,3 @@
[defaults]
host_key_checking = False
inventory = hosts.ini

BIN
argocd/argocd-manifest.yaml Normal file
View File

Binary file not shown.

72
argocd/kustomization.yaml Normal file
View File

@ -0,0 +1,72 @@
# /argocd/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: argocd # Forces everything in this folder into the 'db' namespace
resources:
- https://raw.githubusercontent.com/argoproj/argo-cd/v3.3.4/manifests/ha/install.yaml
#- https://raw.githubusercontent.com/argoproj/argo-cd/v3.4.0-rc2/manifests/ha/install.yaml
#- https://raw.githubusercontent.com/argoproj/argo-cd/v2.10.4/manifests/install.yaml
#- https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
#- argocd-manifest.yaml # Your StatefulSet/Service file generated via helm template
patches:
- target:
kind: Deployment
patch: |-
- op: add
path: /spec/template/spec/nodeSelector
value:
node-role.kubernetes.io/nworker: "true"
- op: add
path: /spec/template/spec/tolerations
value:
- key: "node-role.kubernetes.io/nworker"
operator: "Exists"
effect: "NoSchedule"
# - op: add
# path: /spec/template/spec/tolerations
# value:
# node-role.kubernetes.io/dbworker: "true"
# # - key: "node-role.kubernetes.io/dbworker"
# # operator: "Exists"
# # effect: "NoSchedule"
# This section allows you to modify the behavior of ArgoCD
# (e.g., making the UI accessible via NodePort or Ingress)
# patches:
# - target:
# kind: Service
# name: argocd-server
# patch: |-
# - op: replace
# path: /spec/type
# value: NodePort # Change to LoadBalancer if your cloud supports it
# using Kustomize way
# helmCharts:
# - name: yugabyte
# repo: https://charts.yugabyte.com
# version: 2.18.0 # Specify the version you want
# releaseName: yugabytedb
# namespace: db
# # This replaces the "--set" flags you used in the command line
# valuesInline:
# storage:
# master:
# storageClass: longhorn
# tserver:
# storageClass: longhorn
# replicas:
# master: 4
# tserver: 3
# enableLoadBalancer: false
# gflags:
# master:
# max_clock_skew_usec: 2000000
# time_source: system
# tserver:
# max_clock_skew_usec: 2000000
# time_source: system
# start_pgsql_proxy: true

23
cosmo-router/.helmignore Normal file
View File

@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

24
cosmo-router/Chart.yaml Normal file
View File

@ -0,0 +1,24 @@
apiVersion: v2
name: cosmo-router
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "1.16.0"

26
cosmo-router/config.yaml Normal file
View File

@ -0,0 +1,26 @@
version: "1"
dev_mode: true
execution_config:
file:
path: "/etc/cosmo/router.json" # Inside the container
watch: true
#override_graph_config_path: "/etc/cosmo/router.json"
# This is the standard way to enable Subscription protocols in Cosmo
# traffic_management:
# protocols:
# graphql_sse:
# enabled: true
# graphql_ws:
# enabled: true
# edfs:
# enabled: true
# EDFS:
# enabled: true
events:
providers:
nats:
- id: "my-nats"
url: "nats://nats-cluster.infra.svc.cluster.local:4222"

62
cosmo-router/graph.yaml Normal file
View File

@ -0,0 +1,62 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/wundergraph/cosmo/main/router/pkg/config/config.schema.json
version: "1"
subgraphs:
- name: users
routing_url: http://subgraph-python:8000/graphql
schema:
file: ./subgraphs/users/users-schema.graphql
# introspection:
# url: http://localhost:8000/graphql
- name: nats_events
routing_url: http://nats-virtual # The router handles this via EDFS
schema:
file: ./subgraphs/users/schema.graphqls
# Use the "custom_schema" key instead of "custom_queries"
# to ensure the base Mutation type is initialized correctly.
# custom_schema: |
# schema {
# query: Query
# mutation: Mutation
# }
# type Mutation {
# createUser(id: ID!, username: String!, email: String!): edfs__PublishResult!
# @edfs__natsPublish(subject: "user.created", providerId: "my-nats")
# }
# Use the 'override_config' key to force the Mutation into the final schema
# Use custom_queries to 'extend' the Mutation type provided by your Python app
# custom_queries: |
# extend type Mutation {
# createUser(id: ID!, username: String!, email: String!): edfs__PublishResult!
# @edfs__natsPublish(subject: "user.created", providerId: "my-nats")
# }
# This 'extends' the Mutation type you just created in app.py
# custom_queries: |
# extend type Mutation {
# createUser(id: ID!, username: String!, email: String!): edfs__PublishResult!
# @edfs__natsPublish(subject: "user.created", providerId: "my-nats")
# }
# This "unlocks" the Mutation type in the router
# custom_schema: |
# type Mutation
# This tells the COMPOSER (wgc) how to build the router.json
# edfs:
# nats:
# - id: "my-nats"
# url: "nats://nats:4222"
# publish:
# - name: "createUser"
# subject: "user.created"
# providerId: "my-nats"
# arguments:
# - name: "id"
# type: "ID!"
# - name: "username"
# type: "String!"
# - name: "email"
# type: "String!"

58
cosmo-router/kustomization.yaml Normal file
View File

@ -0,0 +1,58 @@
# apiVersion: kustomize.config.k8s.io/v1beta1
# kind: Kustomization
# namespace: apps
# # THIS IS THE MISSING LINK
# # It tells Kustomize: "Include the YAML coming from the pipe"
# resources:
# - ./base-helm.yaml
# # 1. Automatic Linkerd Injection for all pods in this folder
# commonAnnotations:
# linkerd.io/inject: enabled
# # 2. Manage your Graph and Config as Kustomize-managed secrets/configs
# # This ensures that if router.json changes, the pods perform a rolling restart.
# configMapGenerator:
# - name: router-configs
# files:
# - config.yaml
# - router.json
# # 3. Fix the ServiceAccount conflict if the Helm chart tries to create it
# patches:
# - target:
# kind: ServiceAccount
# name: cosmo-router
# patch: |-
# apiVersion: v1
# kind: ServiceAccount
# metadata:
# name: cosmo-router
# annotations:
# kustomize.config.k8s.io/behavior: merge
# - target:
# kind: Deployment
# name: cosmo-router
# patch: |-
# apiVersion: apps/v1
# kind: Deployment
# metadata:
# name: cosmo-router
# spec:
# template:
# spec:
# nodeSelector:
# kubernetes.io/hostname: nworker
# containers:
# - name: cosmo-router
# volumeMounts:
# - name: config-volume
# # This overrides the Helm mount to use the Kustomize ConfigMap
# mountPath: /etc/cosmo
# volumes:
# - name: config-volume
# configMap:
# name: router-configs

1
cosmo-router/router.json Normal file
View File

File diff suppressed because one or more lines are too long

49
cosmo-router/subgraphs/users/schema.graphqls Normal file
View File

@ -0,0 +1,49 @@
# 1. Required EDFS Directives
directive @edfs__natsPublish(subject: String!, providerId: String! = "default") on FIELD_DEFINITION
directive @edfs__natsRequest(subject: String!, providerId: String! = "default") on FIELD_DEFINITION
directive @edfs__natsSubscribe(subjects: [String!]!, providerId: String! = "default") on FIELD_DEFINITION
# 2. Result type ONLY for Mutations
type edfs__PublishResult {
success: Boolean!
}
scalar JSONB
#input CreateRequestLogInput {
# content_hash: String!
# payload: JSONB!
#}
# 3. Mutation (Follows Rule #1: Must return edfs__PublishResult)
type Mutation {
createUser(id: ID!, username: String!, email: String!): edfs__PublishResult!
@edfs__natsPublish(subject: "user.created", providerId: "my-nats")
createRequestLog( content: String!, producer_timestamp: String!, source: String!): edfs__PublishResult!
@edfs__natsPublish(
subject: "input_request_logs",
providerId: "my-nats"
)
}
# 4a. Reference the Entity from your Python subgraph
#type User @key(fields: "id", resolvable: false) {
# id: ID! @external
#}
# 4b. Faster then 4a without communicating with python subgraph container
type User @key(fields: "id username email", resolvable: false) {
id: ID! @external
username: String! @external
email: String! @external
}
type Query {
# Root fields MUST have a directive.
_natsHealthCheck: User!
@edfs__natsRequest(subject: "health.check", providerId: "my-nats")
}
type Subscription {
# FIX: Changed directive name to @edfs__natsSubscribe and used 'subjects' (plural/list)
userCreatedStream: User!
@edfs__natsSubscribe(subjects: ["user.created"], providerId: "my-nats")
}

29
cosmo-router/subgraphs/users/users-schema.graphql Normal file
View File

@ -0,0 +1,29 @@
schema @link(url: "https://specs.apollo.dev/federation/v2.11", import: ["@key"]) {
query: Query
mutation: Mutation
}
type Mutation {
ping: String!
}
type Query {
_entities(representations: [_Any!]!): [_Entity]!
_service: _Service!
me: User!
allUsers: [User!]!
}
type User @key(fields: "id") {
id: ID!
username: String!
email: String!
}
scalar _Any
union _Entity = User
type _Service {
sdl: String!
}

0
cosmo-router/templates/NOTES.txt Normal file
View File

62
cosmo-router/templates/_helpers.tpl Normal file
View File

@ -0,0 +1,62 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "cosmo-router.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "cosmo-router.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "cosmo-router.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "cosmo-router.labels" -}}
helm.sh/chart: {{ include "cosmo-router.chart" . }}
{{ include "cosmo-router.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "cosmo-router.selectorLabels" -}}
app.kubernetes.io/name: {{ include "cosmo-router.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "cosmo-router.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "cosmo-router.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}

11
cosmo-router/templates/configmap.yaml Normal file
View File

@ -0,0 +1,11 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "cosmo-router.fullname" . }}-config
labels:
{{- include "cosmo-router.labels" . | nindent 4 }}
data:
config.yaml: |
{{ .Files.Get "config.yaml" | indent 4 }}
router.json: |
{{ .Files.Get "router.json" | indent 4 }}

54
cosmo-router/templates/deployment.yaml Normal file
View File

@ -0,0 +1,54 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "cosmo-router.fullname" . }}
labels:
{{- include "cosmo-router.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
{{- include "cosmo-router.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
# Linkerd sidecar injection
linkerd.io/inject: enabled
# This "checksum" forces a restart if your config files change
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
labels:
{{- include "cosmo-router.selectorLabels" . | nindent 8 }}
spec:
containers:
- name: {{ .Chart.Name }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
{{- range $key, $val := .Values.env }}
- name: {{ $key }}
value: {{ $val | quote }}
{{- end }}
- name: CONFIG_PATH
value: "/etc/cosmo/config.yaml"
ports:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
# Production health checks
livenessProbe:
httpGet:
path: /health
port: http
readinessProbe:
httpGet:
path: /health
port: http
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- name: config-volume
mountPath: /etc/cosmo
volumes:
- name: config-volume
configMap:
name: {{ include "cosmo-router.fullname" . }}-config

32
cosmo-router/templates/hpa.yaml Normal file
View File

@ -0,0 +1,32 @@
{{- if .Values.autoscaling.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "cosmo-router.fullname" . }}
labels:
{{- include "cosmo-router.labels" . | nindent 4 }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "cosmo-router.fullname" . }}
minReplicas: {{ .Values.autoscaling.minReplicas }}
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
metrics:
{{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}

39
cosmo-router/templates/ingress.yaml Normal file
View File

@ -0,0 +1,39 @@
{{- if .Values.ingress.enabled -}}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "cosmo-router.fullname" . }}
labels:
{{- include "cosmo-router.labels" . | nindent 4 }}
annotations:
kubernetes.io/ingress.class: {{ .Values.ingress.className | default "traefik" | quote }}
{{- with .Values.ingress.annotations }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if .Values.ingress.tls }}
tls:
{{- range .Values.ingress.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
pathType: {{ .pathType | default "Prefix" }}
backend:
service:
name: {{ include "cosmo-router.fullname" $ }}
port:
number: {{ $.Values.service.port }}
{{- end }}
{{- end }}
{{- end }}

15
cosmo-router/templates/service.yaml Normal file
View File

@ -0,0 +1,15 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "cosmo-router.fullname" . }}
labels:
{{- include "cosmo-router.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "cosmo-router.selectorLabels" . | nindent 4 }}

77
cosmo-router/values.yaml Normal file
View File

@ -0,0 +1,77 @@
replicaCount: 2
image:
repository: ghcr.io/wundergraph/cosmo/router
pullPolicy: IfNotPresent
tag: "latest"
service:
type: ClusterIP
port: 3002
# Environment variables for the container
env:
DEV_MODE: "true"
LISTEN_ADDR: "0.0.0.0:3002"
# Production resource limits (important for k3s stability)
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 100m
memory: 128Mi
# Ingress Configuration
ingress:
enabled: true
className: "traefik"
hosts:
- host: cosmo.local
paths:
- path: /
pathType: Prefix
# Autoscaling
autoscaling:
enabled: true
minReplicas: 2
maxReplicas: 5
targetCPUUtilizationPercentage: 80
serviceAccount:
# Specifies whether a service account should be created
create: true
# Automatically mount a ServiceAccount token into the pod
automount: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
# image:
# repository: ghcr.io/wundergraph/cosmo/router
# tag: latest
# pullPolicy: IfNotPresent
# service:
# type: ClusterIP
# port: 3002
# env:
# DEV_MODE: "true"
# LISTEN_ADDR: "0.0.0.0:3002"
# ingressClassName: traefik
# # # Map your local volumes to ConfigMaps or Secrets in K8s
# # configFiles:
# # router.json: |
# # { "example": "config" }
# # config.yaml: |
# # # Your cosmo config here
podAnnotations:
linkerd.io/inject: disabled
# pathType: ImplementationSpecific

26
db/kustomization.yaml Normal file
View File

@ -0,0 +1,26 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
# This pulls the Operator installation directly from the web
- https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-1.22.1.yaml
- postgres-cluster.yaml
# This patch ensures the Postgres Cluster pods go to dbworkers
# patches:
# - target:
# kind: Cluster
# group: postgresql.cnpg.io
# patch: |-
# apiVersion: postgresql.cnpg.io/v1
# kind: Cluster
# metadata:
# name: postgres-ha
# spec:
# nodeSelector:
# node-role.kubernetes.io/dbworker: "true"
# - op: add
# path: /spec/nodeSelector
# value:
# node-role.kubernetes.io/dbworker: "true"

40
db/postgres-cluster.yaml Normal file
View File

@ -0,0 +1,40 @@
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: postgres-ha
namespace: db
spec:
instances: 3 # The operator will manage 1 Primary and 2 Replicas
imageName: ghcr.io/cloudnative-pg/postgresql:16
enableSuperuserAccess: true # <--- ADD THIS LINE
# ADD THIS SECTION HERE
affinity:
nodeSelector:
node-role.kubernetes.io/dbworker: "true"
tolerations:
- key: "node-role.kubernetes.io/dbworker"
operator: "Exists"
effect: "NoSchedule"
storage:
size: 4Gi
storageClass: longhorn
# The operator handles resource management across all 3 nodes
resources:
requests:
cpu: "250m"
memory: "512Mi"
limits:
cpu: "1"
memory: "1Gi"
# It even handles its own self-signed certificates for TLS
bootstrap:
initdb:
database: ecommerce
owner: invixel_admin
# storage:
# size: 10Gi
# storageClass: longhorn

64
deploy-k3s.yml Normal file
View File

@ -0,0 +1,64 @@
---
- name: Deploy Cluster Applications
hosts: master # Run this ONLY on the master (or localhost)
become: no
vars:
project_root: "/mnt/f/Invixel/k8s-infra-cluster"
namespaces: ["infra", "stream", "apps"]
win_user: "wendg2"
tasks:
- name: Ensure WSL .kube directory exists
delegate_to: localhost
become: no
ansible.builtin.file:
path: "~/.kube"
state: directory
mode: '0700'
- name: Link Windows Kubeconfig to WSL (Fixes Helm/Kubectl)
delegate_to: localhost
become: no
ansible.builtin.file:
src: "/mnt/c/Users/{{ win_user }}/.kube/config"
dest: "~/.kube/config"
state: link
# - name: Deploy Infra via Kustomize
# ansible.builtin.shell: "kustomize build --enable-helm . | kubectl apply --server-side --force-conflicts --insecure-skip-tls-verify -f -"
# args:
# chdir: "{{ project_root }}"
# delegate_to: localhost # <--- CRITICAL: Run on your laptop
# become: no # Your laptop doesn't need sudo for this
- name: Install Cosmo Router via Helm
# Use upgrade --install to prevent "already exists" errors
ansible.builtin.shell: "helm upgrade --install cosmo-router ./cosmo-router -n apps"
args:
chdir: "{{ project_root }}"
delegate_to: localhost # <--- CRITICAL: Run on your laptop
become: no # Your laptop doesn't need sudo for this
- name: Apply Linkerd CRDs
ansible.builtin.shell: "kustomize build --enable-helm . | kubectl apply --server-side --force-conflicts --insecure-skip-tls-verify -f -"
args:
chdir: "{{ project_root }}/linkerd"
delegate_to: localhost # <--- ADD THIS, Ansible use the Python environment on our laptop
become: no # Laptop doesn't need sudo
- name: Annotate Namespaces for Linkerd Injection
ansible.builtin.shell: |
kubectl annotate namespace {{ item }} linkerd.io/inject=enabled --overwrite --insecure-skip-tls-verify
loop: "{{ namespaces }}"
delegate_to: localhost # <--- ADD THIS, Ansible use the Python environment on our laptop
become: no # Laptop doesn't need sudo
register: annot_result
until: annot_result.rc == 0
retries: 5 # Try 5 times
delay: 10 # Wait 10 seconds between tries
# export ANSIBLE_CONFIG=./ansible.cfg
# ansible-playbook -i hosts.ini deploy-k3s.yml
# ansible k8s_nodes -i hosts.ini -m ping

38
hosts.ini Normal file
View File

@ -0,0 +1,38 @@
; [all:vars]
; ansible_user=ubuntu-s # using same user for all VMs
; # If you use the SSH key method above:
; # ansible_ssh_private_key_file=~/.ssh/id_rsa
; # using mounted private key from windows folder structure from does not work
; #ansible_ssh_private_key_file=/mnt/c/Users/wendg2/.ssh/id_ed25519
; ansible_ssh_private_key_file=~/.ssh/id_ed25519
; [master]
; 192.168.3.91
; [workers]
; 192.168.3.93
; 192.168.3.92
; 192.168.3.94
[master]
192.168.3.91 ansible_user=master-1
# different user name
[dbworkers]
192.168.3.92 ansible_user=master-2
192.168.3.93 ansible_user=master-3
192.168.3.94 ansible_user=master-4
[workers]
192.168.3.95 ansible_user=master-5
192.168.3.96 ansible_user=master-6
# This group combines both for easy targeting
[k8s_nodes:children]
master
workers
# [k8s_nodes:vars]
# # Ensure Ansible uses the correct Python on the remote nodes
# ansible_python_interpreter=/usr/bin/python3
# # If you use the same SSH key for all:
# ansible_ssh_private_key_file=~/.ssh/id_rsa

122
infra/benthos/benthos-deployment.yaml Normal file
View File

@ -0,0 +1,122 @@
apiVersion: v1
kind: Secret
metadata:
name: postgres-ha-app
namespace: infra # Matches your benthos namespace
type: Opaque
stringData:
# This must be the base64 encoded version of your password
# PowerShell: [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("your_password"))
password: 0lkzPxlwj6JVOXwwoLYROZJsONJoPK3MtrqkxnH3iaXUs0gFg0WL78RxyDdB86Sk
---
apiVersion: v1
kind: Secret
metadata:
name: postgres-ha-superuser
namespace: infra # Matches your benthos namespace
type: Opaque
stringData:
# This must be the base64 encoded version of your password
# PowerShell: [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("your_password"))
password: CuUwr9dPXJibVFygh7oGastvIJb4syMZXKljsf0dbEl91TwuYLqvEW35hN98ytKe
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: benthos
#namespace: infra
spec:
replicas: 3
selector:
matchLabels:
app: benthos
template:
metadata:
labels:
app: benthos
spec:
automountServiceAccountToken: false # Fixes SonarQube RBAC warning
# initContainers: # InitContainer that "pings" the database port before benthos start. FQDN: yb-tservers.db.svc.cluster.local
# - name: wait-for-yugabyte
# image: busybox:latest
# command: ['sh', '-c', 'until nc -vz yb-tservers.db.svc.cluster.local 5433; do echo waiting for yugabyte; sleep 2; done;']
initContainers:
- name: wait-for-postgres
image: busybox:latest
# We now ping the -rw service on port 5432
command: ['sh', '-c', 'until nc -vz postgres-ha-rw.db.svc.cluster.local 5432; do echo waiting for postgres; sleep 2; done;']
containers:
- name: benthos
image: jeffail/benthos
# Commands to look for your streams and templates
args: ["--chilled", "-r", "/configs/resources/resources.yaml", "-t", "/configs/templates/*.yaml", "streams", "/configs/streams/*.yaml"]
# - "-r"
# - "/configs/templates/*.yaml"
# - "streams"
# - "/configs/streams/*.yaml"
ports:
- name: dashboard
containerPort: 4195
env:
- name: ECOM_PASS # This is the name Benthos will see
valueFrom:
secretKeyRef:
name: postgres-ha-app # This must match your Secret metadata.name
key: password
resources: # Fixes CPU request warning
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "512Mi"
volumeMounts:
- name: resources-vol
mountPath: /configs/resources
- name: streams-vol
mountPath: /configs/streams
- name: templates-vol
mountPath: /configs/templates
volumes:
- name: resources-vol
configMap:
name: benthos-resources
- name: streams-vol
configMap:
name: benthos-streams # Matches the name in kustomization.yaml
- name: templates-vol
configMap:
name: benthos-templates
---
apiVersion: v1
kind: Service
metadata:
name: benthos-ui
#namespace: infra
spec:
selector:
app: benthos # This MUST match the labels in your deployment
ports:
- protocol: TCP
port: 4195
targetPort: 4195
type: ClusterIP
# benthos:
# image: jeffail/benthos
# container_name: benthos
# volumes:
# - ./benthos-configs:/configs # A folder containing your pass1.yaml and pass2.yaml
# - ./benthos-configs/streams:/configs/streams # A folder containing your pass1.yaml and pass2.yaml
# - ./benthos-configs/templates:/configs/templates # Maps a local folder to the cache
# # 'streams' mode watches the folder and runs everything inside it
# command: ["--chilled", "-r", "/configs/resources.yaml", "-t", "/configs/templates/*.yaml", "streams", "/configs/streams/*.yaml"]
# depends_on:
# - nats
# - yb-master
# ports:
# - "4195:4195" # Dashboard/Metrics port
# networks:
# - nats-network

20
infra/benthos/benthos-ingress.yaml Normal file
View File

@ -0,0 +1,20 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: benthos-ingress
namespace: infra
annotations:
traefik.ingress.kubernetes.io/router.entrypoints: web
spec:
rules:
- host: benthos.local # You will add this to your Windows 'hosts' file
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: benthos-ui
port:
number: 4195
# kubectl apply -f infra/benthos/benthos-ingress.yaml

60
infra/benthos/kustomization.yaml Normal file
View File

@ -0,0 +1,60 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: infra # <-- Add this here instead insidde of benthos-deployment.yaml
# 1. List your static YAML files here
resources:
#- nats-cluster.yaml
- benthos-deployment.yaml
#- benthos-service.yaml
# 2. List your config sources here
# This tells K8s: "Take every file in these folders and make ConfigMaps"
configMapGenerator:
- name: benthos-streams
files:
- streams/ingest_request_log_id_serial.yaml
- streams/ingest_transaction_log_id_serial.yaml
#- streams/poll_request_log_id_serial.yaml
#- streams/poll_transaction_id_serial.yaml
- name: benthos-templates
files:
- templates/ingest_template_id_serial.yaml
- templates/poll_template_id_serial.yaml
- name: benthos-resources
files:
- resources/resources.yaml
patches:
- target:
kind: Deployment
name: benthos # Ensure this matches the name inside benthos-deployment.yaml
patch: |-
- op: add
path: /spec/template/spec/nodeSelector
value:
node-role.kubernetes.io/nworker: "true"
# - target:
# kind: StatefulSet
# name: nats-cluster # Ensure this matches the 'metadata.name' in nats-cluster.yaml
# patch: |-
# - op: add
# path: /spec/template/spec/nodeSelector
# value:
# node-role.kubernetes.io/nworker: "true"
# helmCharts:
# - name: nats
# repo: https://nats-io.github.io/k8s/helm/charts/
# releaseName: nats-cluster
# namespace: infra
# valuesInline:
# nats:
# jetstream:
# enabled: true
# cluster:
# enabled: true
# replicas: 3

16
infra/benthos/resources/resources.yaml Normal file
View File

@ -0,0 +1,16 @@
# /configs/resources.yaml
cache_resources:
- label: state_cache
file:
directory: /configs/cache_data
# TIER 1: The "Bloom-style" local gate (Resets on restart, ultra-fast)
- label: local_bloom_gate
memory:
default_ttl: 1h
- label: dedupe_cache
redis:
url: "redis://redis-nats:6379"
default_ttl: "24h" # <--- Changed from 'expiration' to 'default_ttl'
- label: yugabyte_checkpoints
redis:
url: "redis://redis-nats:6379"

44
infra/benthos/streams/ingest_request_log_id_serial.yaml Normal file
View File

@ -0,0 +1,44 @@
# streams/request_logs.yaml
# http:
# address: 0.0.0.0:4195
# enabled: true
# debug_endpoints: true
input:
nats_jetstream:
#urls: [ "nats-cluster:4222" ]
urls: [ "nats://nats-cluster.infra.svc.cluster.local:4222" ]
stream: "request_stream"
subject: "input_request_logs"
manage_stream: true # <--- Add this
output:
yugabyte_insert:
dsn: "postgres://invixel_admin:${ECOM_PASS}@postgres-ha-rw.db.svc.cluster.local:5432/ecommerce?sslmode=disable"
#dsn: "postgres://yugabyte@yb-tservers.db.svc.cluster.local:5433/ecommerce?sslmode=disable"
table_name: "public.request_logs"
# 1. Define the columns for THIS specific table
columns: ["content_hash", "payload"]
# 2. Define how to map data to those columns
value_mapping: |
root = [
# content().hash("sha256").encode("hex"),
this.content.hash("sha256").encode("hex"),
this.string()
]
# CREATE SEQUENCE log_seq CACHE 10000;
# CREATE TABLE request_logs (
# id BIGINT DEFAULT nextval('log_seq'), -- BIGSERIAL
# content_hash TEXT UNIQUE,
# payload JSONB, -- Stores the full original message
# PRIMARY KEY (id)
# );
# CREATE TABLE request_logs (
# id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
# content_hash TEXT UNIQUE,
# payload JSONB
# );

131
infra/benthos/streams/ingest_transaction_log_id_serial.yaml Normal file
View File

@ -0,0 +1,131 @@
# streams/transaction.yaml
input:
nats_jetstream:
#urls: [ "nats-cluster:4222" ]
urls: [ "nats://nats-cluster.infra.svc.cluster.local:4222" ]
stream: "request_stream"
subject: "input_transaction_logs"
manage_stream: true # <--- Add this
# NEW: Global Pipeline for the input
pipeline:
processors:
- mapping: |
# Set metadata globally so all outputs and templates can see it
root = this
meta conflict_col = this.data.order.id.or("unknown")
# with thhe following assignment only streams can see the value but template not
meta transaction_id = this.transaction_id.or("unknown")
meta transaction_status = this.transaction_status
root.timestamp = this.timestamp
meta order_id = this.data.order.id.or("unknown")
# root.flat_status = this.data.order.status.lowercase()
# root.order_customer_email = this.data.order.customer.email
# root.order_customer_loyalty_tier = this.data.order.customer.loyalty_tier
# root.metadata_ip_address = this.data.metadata.ip_address
# root.metadata_region = this.data.metadata.region
# root.metadata_user_agent = this.data.metadata.user_agent
root.data.order.items = this.data.order.items
# meta metadata_items = this.data.items.map_each(item -> item.sku)
- log:
level: INFO
message: "New transaction: status ${! meta(\"transaction_status\") } - order ID: ${! meta(\"order_id\") } timestamp: ${! json(\"timestamp\")} timestamp2a ${! this.timestamp }"
# output:
# broker:
# pattern: fan_out
# outputs:
# # --- Branch 1: Archive the raw JSON ---
# - yugabyte_insert:
# dsn: "postgres://yugabyte@yugabytedb-3:5433/ecommerce?sslmode=disable"
# table_name: "public.transaction"
# #conflict_columns: "transaction_id"
# #columns: ["transaction_id", "orders", "order_status"]
# # if use this deactivate conflict_columns
# columns: ["content_hash", "orders", "status","created_at"]
# value_mapping: |
# root = [
# # if use meta define it first above
# # meta("transaction_id"),
# # meta("order_id"),
# # meta("order_status")
# this.transaction_id,
# this.data.order.id,
# this.transaction_status,
# this.timestamp
# ]
# # --- Branch 2: Extract nested GraphQL data into Facts ---
# - yugabyte_insert:
# dsn: "postgres://yugabyte@yugabytedb-3:5433/ecommerce?sslmode=disable"
# table_name: "public.order"
# # conflict_columns: "content_hash"
# columns: ["content_hash", "customer_email", "customer_loyalty_tier","status", "ip_address","region"]
# # if use this deactivate conflict_columns
# #columns: ["content_hash", "customer_email", "status"]
# value_mapping: |
# root = [
# # if use meta define it first above
# # meta("order_id"),
# # meta("order_customer_email"),
# # meta("order_status")
# this.data.order.id,
# this.data.order.customer.email,
# this.data.order.customer.loyalty_tier,
# this.data.order.status,
# this.data.order.metadata.ip_address,
# this.data.order.metadata.region
# ]
# - yugabyte_insert:
# dsn: "postgres://yugabyte@yugabytedb-3:5433/ecommerce?sslmode=disable"
# table_name: "public.items"
# columns: ["content_hash", "items"]
# # No unarchive needed!
# processors:
# - mapping: |
# root = this
# # 1. Update dedupe key for this specific table
# meta conflict_col = meta("order_id") + "-item"
# # 2. Stringify the items array for the JSONB column
# # root.items_blob = this.data.order.items.format_json()
# value_mapping: |
# root = [
# meta("order_id"),
# this.data.order.items.string()
# ]
# - kafka:
# addresses: [ "redpanda:9092" ]
# topic: "transactions_redpanda"
# # By not adding a 'processors' block here,
# # it sends the current state of 'root' from the global pipeline.
# # To ensure it is the ABSOLUTE original, we can force a mapping:
# key : ${! meta("transaction_id")}
# processors:
# - mapping: |
# root = content()
# 1. example below for using content_hash
# 2. change it accordingly to your defined conflict_columns
# 3. IMPORTANT: conflict_columns is to be defined as UNIQUE in sql
# CREATE TABLE public.transaction (
# id BIGSERIAL,
# content_hash TEXT UNIQUE,
# orders TEXT,
# order_status TEXT,
# timestamp TIMESTAMPTZ,
# PRIMARY KEY (id)
# );
# CREATE TABLE public.order (
# id TEXT PRIMARY KEY, -- Must be TEXT for "ORD-5521"
# content_hash TEXT UNIQUE,
# customer_email TEXT,
# amount DECIMAL
# );

28
infra/benthos/streams/poll_request_log_id_serial.yaml Normal file
View File

@ -0,0 +1,28 @@
logger:
level: DEBUG # Force Benthos to tell us everything it's doing
format: logfmt
# This tells Benthos: Use the 'sql_poller' template as the input
input:
sql_poller:
dsn: "postgres://invixel_admin:${ECOM_PASS}@postgres-ha-rw.db.svc.cluster.local:5432/ecommerce?sslmode=disable"
#stream: "fact_stream"
#subject: "output_request_logs"
#dsn: "postgres://yugabyte@yugabytedb-2:5433/ecommerce?sslmode=disable"
table_name: "public.request_logs"
id_column: "id" # Or 'id' if you use the hashed string
pipeline:
processors:
- log:
level: INFO
message: "Polled a message: ${! content() }"
# The output sends the polled data back to NATS
output:
nats_jetstream:
#urls: [ "nats:4222" ]
urls: [ "nats://nats-cluster.infra.svc.cluster.local:4222" ]
subject: "output_request_logs"
stream: "fact_stream"
#manage_stream: true # This is the key setting

28
infra/benthos/streams/poll_transaction_id_serial.yaml Normal file
View File

@ -0,0 +1,28 @@
# This tells Benthos: Use the 'sql_poller' template as the input
input:
sql_poller:
dsn: "postgres://invixel_admin:${ECOM_PASS}@postgres-ha-rw.db.svc.cluster.local:5432/ecommerce?sslmode=allow"
#dsn: "postgres://yugabyte@yugabytedb-2:5433/ecommerce?sslmode=disable"
table_name: "transaction"
id_column: "id" # Or 'id' if you use the hashed string
# pipeline:
# processors:
# - mapping: |
# root = this
# # Set metadata globally so all outputs and templates can see it
# root.transaction_id = this.transaction_id.or("unknown")
# - log:
# level: INFO
# message: "2b.New transaction: status - order ID: ${! meta(\"order_id\") } timestamp: ${! json(\"timestamp\")} timestamp3 ${! timestamp }"
# The output sends the polled data back to NATS
# output:
# nats_jetstream:
# urls: [ "nats:4222" ]
# subject: "output_transaction"
# stream: "fact_stream"
# manage_stream: true # This is the key setting

26
infra/benthos/streams/reserve/ingest_request_log_id_hash_received_at.yaml Normal file
View File

@ -0,0 +1,26 @@
# streams/request_logs.yaml
input:
nats_jetstream:
urls: [ "nats:4222" ]
subject: "input_request_logs"
output:
yugabyte_insert:
dsn: "postgres://yugabyte@yugabytedb-3:5433/yugabyte?sslmode=disable"
table: "public.request_logs"
# 1. Define the columns for THIS specific table
columns: ["id", "payload", "received_at"]
# 2. Define how to map data to those columns
value_mapping: |
root = [
content().hash("sha256").encode("hex"),
this.string(),
now()
]
# CREATE TABLE request_logs (
# id TEXT PRIMARY KEY, -- This will store our SHA-256 hash
# payload JSONB, -- Stores the full original message
# received_at TIMESTAMPTZ -- Timestamp for auditing
# );

14
infra/benthos/streams/reserve/poll_request_log_received_at.yaml Normal file
View File

@ -0,0 +1,14 @@
# This tells Benthos: Use the 'sql_poller' template as the input
input:
sql_poller:
dsn: "postgres://yugabyte@yugabytedb-2:5433/yugabyte?sslmode=disable"
table_name: "request_logs"
id_column: "received_at" # Or 'id' if you use the hashed string
# The output sends the polled data back to NATS
# output:
# nats_jetstream:
# urls: [ "nats:4222" ]
# subject: "public.request_logs"
# stream: "fact_stream"
# manage_stream: true # This is the key setting

13
infra/benthos/streams/upsert_user.yaml Normal file
View File

@ -0,0 +1,13 @@
input:
nats_jetstream:
urls: ["nats://nats:4222"]
subject: "user.updates"
stream: "request_stream"
pipeline:
processors:
- mapping: |
root.id = this.id
root.username = this.username
root.email = this.email
root.updated_at = now()

76
infra/benthos/templates/ingest_template_id_serial.yaml Normal file
View File

@ -0,0 +1,76 @@
# templates/sql_ingest.yaml
name: yugabyte_insert
type: output
fields:
- name: table_name
type: string
- name: dsn
type: string
- name: columns
type: unknown
- name: value_mapping
type: string
- name: conflict_columns # NEW: Make it flexible!
type: string
default: "content_hash"
mapping: |
# root.processors = [
# # LAYER 2: Local Bloom-style Gate (RAM)
# {
# "dedupe": {
# "cache": "local_bloom_gate",
# # "key": "${! content().hash(\"sha256\").encode(\"hex\") }",
# # without concatenating with this.table name inserting multiple tables not possible
# "key": "${! meta(\"conflict_col\") }-" + this.table_name ,
# "drop_on_err": false
# }
# },
# # VISIBILITY: Log what survived the Bloom Filter
# {
# "log": {
# "level": "INFO",
# "message": "Passed Bloom Filter: transaction ${! meta(\"conflict_col\") }"
# }
# },
# # LAYER 3 Redis Global Gate (Network)
# {
# "dedupe": {
# "cache": "dedupe_cache",
# # "key": "${! content().hash(\"sha256\").encode(\"hex\") }",
# "key": "${! meta(\"conflict_col\") }-" + this.table_name ,
# "drop_on_err": false
# }
# },
# # NEW: Add Global Logging/Error Handling for ALL inserts using this template
# {
# "catch": [
# {
# "log": {
# "level": "ERROR",
# "message": "SQL Error on " + this.table_name + ": ${! error() }"
# }
# }
# ]
# },
# # B. Success Logging
# {
# "log": {
# "level": "INFO",
# "message": "Attempting insert for " + this.table_name + " ID: ${! json(\"data.order.id\") }"
# }
# }
# ]
root.sql_insert = {
"driver": "postgres",
"dsn": this.dsn,
"table": this.table_name,
"columns": this.columns,
# FIX: We use 'this.value_mapping' directly so the stream's logic is used
"args_mapping": this.value_mapping,
"suffix": "ON CONFLICT ( "+ this.conflict_columns + ") DO NOTHING"
#"suffix": "ON CONFLICT (content_hash) DO NOTHING"
}

59
infra/benthos/templates/poll_template_id_serial.yaml Normal file
View File

@ -0,0 +1,59 @@
# templates/poll_template.yaml
name: sql_poller
type: input
# using BIGSERIAL
fields:
- name: table_name
type: string
- name: dsn
type: string
- name: id_column
type: string
default: "id"
mapping: |
# 1. The base input is a generator that ticks every 2s
root.generate.interval = "2s"
root.generate.mapping = "root = {}"
# 2. We attach the processing logic directly to this input
root.processors = [
{
"cache": {
"resource": "yugabyte_checkpoints",
"operator": "get",
"key": "last_id_" + this.table_name
}
},
{ "catch": [ { "mapping": "root = \"0\"" } ] }, # Default to 0 if cache missing
{ "mapping": "meta last_id = content().string()" },
# 3. Perform the Query
{
"sql_raw": {
"driver": "postgres",
"dsn": this.dsn,
"query": "SELECT * FROM " + this.table_name + " WHERE " + this.id_column + " > $1 ORDER BY " + this.id_column + " ASC LIMIT 500;",
"args_mapping": "root = [ meta(\"last_id\").number() ]"
}
},
# 4. Filter empty results
{
"mapping": "root = if this.type() != \"array\" || this.length() == 0 { deleted() } else { this }"
},
# 5. Update the Cache with the new max ID
{
"branch": {
"processors": [
{ "mapping": "root = this.index(-1)." + this.id_column + ".string()" },
{ "cache": { "resource": "yugabyte_checkpoints", "operator": "set", "key": "last_id_" + this.table_name, "value": "${! content() }" }}
]
}
},
# 6. Flatten the array into individual messages
{ "unarchive": { "format": "json_array" } }
]

28
infra/benthos/templates/reserve/ingest_template_id_hash_received_at.yaml Normal file
View File

@ -0,0 +1,28 @@
# templates/ingest_template.yaml
name: yugabyte_insert
type: output
fields:
- name: dsn
type: string
- name: table
type: string
- name: columns
type: unknown
description: "List of column names for this table"
- name: value_mapping
type: string
description: "Bloblang mapping that generates the array of values matching the columns"
mapping: |
root.sql_insert.driver = "postgres"
root.sql_insert.dsn = this.dsn
root.sql_insert.table = this.table
root.sql_insert.columns = this.columns
# We inject the specific mapping logic passed from the config
root.sql_insert.args_mapping = this.value_mapping
# Standardize your conflict handling here
root.sql_insert.suffix = "ON CONFLICT (id) DO NOTHING"

136
infra/benthos/templates/reserve/poll_template_id_hash_received_at.yaml Normal file
View File

@ -0,0 +1,136 @@
# templates/poll_template.yaml
name: sql_poller
type: input
# using received_at
fields:
- name: table_name
type: string
- name: dsn
type: string
- name: id_column
type: string
default: "received_at"
mapping: |
root.generate.interval = "2s"
root.generate.mapping = "root = {}"
root.processors = [
# 1. Fetch from cache. If it fails, we catch it inside the mapping.
{
"branch": {
"processors": [
{ "try": [ { "cache": { "resource": "yugabyte_checkpoints", "operator": "get", "key": "last_ts_" + this.table_name } } ] }
],
"catch": [ { "mapping": "root = \"1970-01-01T00:00:00Z\"" } ],
"result_map": "meta last_ts = content().string()"
}
},
# 2. Hard-set a default if metadata is STILL null/empty
{
"mapping": """
meta last_ts = if meta("last_ts") == null || meta("last_ts") == "" {
"1970-01-01T00:00:00Z"
} else {
meta("last_ts")
}
"""
},
# DEBUG: Log the timestamp being used
{ "log": { "level": "INFO", "message": "Polling " + this.table_name + " with TS: ${! meta(\"last_ts\") }" } },
# 3. Query Yugabyte
{
"sql_raw": {
"driver": "postgres",
"dsn": this.dsn,
"query": "SELECT * FROM " + this.table_name + " WHERE " + this.id_column + " >= ($1::timestamptz - INTERVAL '5 seconds') ORDER BY " + this.id_column + " ASC LIMIT 500;",
"args_mapping": "root = [ meta(\"last_ts\") ]"
}
},
{ "catch": [ { "log": { "level": "ERROR", "message": "SQL Error: ${! error() }" } } ] },
# 3. Check if results exist
{
"mapping": "root = if this.type() != \"array\" || this.length() == 0 { deleted() } else { this }"
},
# 4. Save the NEW checkpoint
{
"branch": {
"processors": [
{ "mapping": "root = this.index(-1)." + this.id_column + ".string()" },
{ "log": { "level": "INFO", "message": "Saving new checkpoint for " + this.table_name + ": ${! content() }" } },
{
"cache": {
"resource": "yugabyte_checkpoints",
"operator": "set",
"key": "last_ts_" + this.table_name,
"value": "${! content() }"
}
}
]
}
},
# 5. Fan out
{ "unarchive": { "format": "json_array" } }
]
# # using BIGSERIAL
# fields:
# - name: table_name
# type: string
# - name: dsn
# type: string
# - name: id_column
# type: string
# default: "id"
# mapping: |
# # 1. The base input is a generator that ticks every 2s
# root.generate.interval = "2s"
# root.generate.mapping = "root = {}"
# # 2. We attach the processing logic directly to this input
# root.processors = [
# {
# "cache": {
# "resource": "state_cache",
# "operator": "get",
# "key": "last_id_" + this.table_name
# }
# },
# { "catch": [ { "mapping": "root = \"0\"" } ] }, # Default to 0 if cache missing
# { "mapping": "meta last_id = content().string()" },
# # 3. Perform the Query
# {
# "sql_raw": {
# "driver": "postgres",
# "dsn": this.dsn,
# "query": "SELECT * FROM " + this.table_name + " WHERE " + this.id_column + " > $1 ORDER BY " + this.id_column + " ASC LIMIT 500;",
# "args_mapping": "root = [ meta(\"last_id\") ]"
# }
# },
# # 4. Filter empty results
# {
# "mapping": "root = if this.type() != \"array\" || this.length() == 0 { deleted() } else { this }"
# },
# # 5. Update the Cache with the new max ID
# {
# "branch": {
# "processors": [
# { "mapping": "root = this.index(-1)." + this.id_column + ".string()" },
# { "cache": { "resource": "state_cache", "operator": "set", "key": "last_id_" + this.table_name, "value": "${! content() }" }}
# ]
# }
# },
# # 6. Flatten the array into individual messages
# { "unarchive": { "format": "json_array" } }
# ]

26
infra/charts/nats/.helmignore Normal file
View File

@ -0,0 +1,26 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
# template tests
/test

16
infra/charts/nats/Chart.yaml Normal file
View File

@ -0,0 +1,16 @@
apiVersion: v2
appVersion: 2.12.5
description: A Helm chart for the NATS.io High Speed Cloud Native Distributed Communications
Technology.
home: http://github.com/nats-io/k8s
icon: https://nats.io/img/nats-icon-color.png
keywords:
- nats
- messaging
- cncf
maintainers:
- email: info@nats.io
name: The NATS Authors
url: https://github.com/nats-io
name: nats
version: 2.12.5

354
infra/charts/nats/README.md Normal file
View File

@ -0,0 +1,354 @@
# NATS Server
---
[NATS](https://nats.io) is a simple, secure and performant communications system for digital systems, services and devices.
NATS is part of the Cloud Native Computing Foundation ([CNCF](https://cncf.io)).
NATS has over [30 client language implementations](https://nats.io/download/), and its server can run on-premise, in the cloud, at the edge, and even on a Raspberry Pi.
NATS can secure and simplify design and operation of modern distributed systems.
```shell
helm repo add nats https://nats-io.github.io/k8s/helm/charts/
helm upgrade --install nats nats/nats
```
## Upgrade Nodes
- **Upgrading from 0.x**: The `values.yaml` schema changed significantly from 0.x to 1.x. Read [UPGRADING.md](UPGRADING.md) for instructions on upgrading a 0.x release to 1.x.
## Values
There are a handful of explicitly defined options which are documented with comments in the [values.yaml](values.yaml) file.
Everything in the NATS Config or Kubernetes Resources can be overridden by `merge` and `patch`, which is supported for the following values:
| key | type | enabled by default |
|----------------------------------|-----------------------------------------------------------------------------------------------------------------------------|-----------------------------------------|
| `config` | [NATS Config](https://docs.nats.io/running-a-nats-service/configuration) | yes |
| `config.cluster` | [NATS Cluster](https://docs.nats.io/running-a-nats-service/configuration/clustering/cluster_config) | no |
| `config.cluster.tls` | [NATS TLS](https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls) | no |
| `config.jetstream` | [NATS JetStream](https://docs.nats.io/running-a-nats-service/configuration#jetstream) | no |
| `config.jetstream.fileStore.pvc` | [k8s PVC](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#persistentvolumeclaim-v1-core) | yes, when `config.jetstream` is enabled |
| `config.nats.tls` | [NATS TLS](https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls) | no |
| `config.leafnodes` | [NATS LeafNodes](https://docs.nats.io/running-a-nats-service/configuration/leafnodes/leafnode_conf) | no |
| `config.leafnodes.tls` | [NATS TLS](https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls) | no |
| `config.websocket` | [NATS WebSocket](https://docs.nats.io/running-a-nats-service/configuration/websocket/websocket_conf) | no |
| `config.websocket.tls` | [NATS TLS](https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls) | no |
| `config.websocket.ingress` | [k8s Ingress](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#ingress-v1-networking-k8s-io) | no |
| `config.mqtt` | [NATS MQTT](https://docs.nats.io/running-a-nats-service/configuration/mqtt/mqtt_config) | no |
| `config.mqtt.tls` | [NATS TLS](https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls) | no |
| `config.gateway` | [NATS Gateway](https://docs.nats.io/running-a-nats-service/configuration/gateways/gateway#gateway-configuration-block) | no |
| `config.gateway.tls` | [NATS TLS](https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls) | no |
| `config.resolver` | [NATS Resolver](https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/jwt/resolver) | no |
| `config.resolver.pvc` | [k8s PVC](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#persistentvolumeclaim-v1-core) | yes, when `config.resolver` is enabled |
| `container` | nats [k8s Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core) | yes |
| `reloader` | config reloader [k8s Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core) | yes |
| `promExporter` | prometheus exporter [k8s Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core) | no |
| `promExporter.podMonitor` | [prometheus PodMonitor](https://prometheus-operator.dev/docs/api-reference/api/#monitoring.coreos.com/v1.PodMonitor) | no |
| `service` | [k8s Service](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#service-v1-core) | yes |
| `statefulSet` | [k8s StatefulSet](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#statefulset-v1-apps) | yes |
| `podTemplate` | [k8s PodTemplate](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#pod-v1-core) | yes |
| `headlessService` | [k8s Service](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#service-v1-core) | yes |
| `configMap` | [k8s ConfigMap](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#configmap-v1-core) | yes |
| `natsBox.contexts.default` | [NATS Context](https://docs.nats.io/using-nats/nats-tools/nats_cli#nats-contexts) | yes |
| `natsBox.contexts.[name]` | [NATS Context](https://docs.nats.io/using-nats/nats-tools/nats_cli#nats-contexts) | no |
| `natsBox.container` | nats-box [k8s Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core) | yes |
| `natsBox.deployment` | [k8s Deployment](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#deployment-v1-apps) | yes |
| `natsBox.podTemplate` | [k8s PodTemplate](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#pod-v1-core) | yes |
| `natsBox.contextsSecret` | [k8s Secret](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secret-v1-core) | yes |
| `natsBox.contentsSecret` | [k8s Secret](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secret-v1-core) | yes |
### Merge
Merging is performed using the Helm [`merge` function](https://helm.sh/docs/chart_template_guide/function_list/#merge-mustmerge). Example - add NATS accounts and container resources:
```yaml
config:
merge:
accounts:
A:
users:
- {user: a, password: a}
B:
users:
- {user: b, password: b}
natsBox:
contexts:
a:
merge: {user: a, password: a}
b:
merge: {user: b, password: b}
defaultContextName: a
```
## Patch
Patching is performed using [JSON Patch](https://jsonpatch.com/). Example - add additional route to end of route list:
```yaml
config:
cluster:
enabled: true
patch:
- op: add
path: /routes/-
value: nats://demo.nats.io:6222
```
## Common Configurations
### JetStream Cluster on 3 separate hosts
```yaml
config:
cluster:
enabled: true
replicas: 3
jetstream:
enabled: true
fileStore:
pvc:
size: 10Gi
podTemplate:
topologySpreadConstraints:
kubernetes.io/hostname:
maxSkew: 1
whenUnsatisfiable: DoNotSchedule
```
### NATS Container Resources
We recommend setting both **requests and limits** - for both **CPU and memory** - **to the same value** for the following reasons:
* It ensures your NATS pod has [predictable performance](https://www.datadoghq.com/blog/kubernetes-cpu-requests-limits/#predictability:~:text=If%20containers%20are,available%20capacity%20decreases.).
* The Go runtime [automatically sets](https://go.dev/doc/go1.25#container-aware-gomaxprocs) [GOMAXPROCS](https://pkg.go.dev/runtime#GOMAXPROCS) to the number of CPU cores defined in the `limits` section. If `limits` are not set, GOMAXPROCS defaults to the node's physical core count, which can lead to [poor performance](https://github.com/golang/go/issues/33803).
* The pod will be assigned to the ["Guaranteed" QoS class](https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/#guaranteed), making it less likely to be evicted when node resources are constrained.
* When deciding how much CPU time to dedicate to Garbage Collection, the Go Runtime assumes that it has access to `GOMAXPROCS*N` seconds of CPU time in `N` second of wall time. [It can cause issues, if this assumption is not true.](https://github.com/golang/go/issues/59715)
Deviate from this recommendation only if you fully understand the implications of your settings.
```yaml
container:
env:
# Different from k8s units, suffix must be B, KiB, MiB, GiB, or TiB
# Should be ~80% of memory limit
GOMEMLIMIT: 6GiB
merge:
# Recommended minimum: at least 2 CPU cores and 8Gi memory for production JetStream clusters
resources:
requests:
cpu: "2"
memory: 8Gi
limits:
cpu: "2"
memory: 8Gi
```
### Specify Image Version
The container image can now be overridden by specifying either the image tag, an image digest, or a full image name. Examples below illustrate the options:
- To set the tag:
```yaml
container:
image:
tag: x.y.z-alpine
```
- To use an image digest, which overrides the tag:
```yaml
container:
image:
repository: nats
digest: sha256:abcdef1234567890...
```
- To override the registry, repository, tag, and digest all at once, specify a full image name:
```yaml
container:
image:
fullImageName: custom-reg.io/myimage@sha256:abcdef1234567890...
```
### Operator Mode with NATS Resolver
Run `nsc generate config --nats-resolver` and replace the `OPERATOR_JWT`, `SYS_ACCOUNT_ID`, and `SYS_ACCOUNT_JWT` with your values.
Make sure that you do not include the trailing `,` in the `SYS_ACCOUNT_JWT`.
```
config:
resolver:
enabled: true
merge:
type: full
interval: 2m
timeout: 1.9s
merge:
operator: OPERATOR_JWT
system_account: SYS_ACCOUNT_ID
resolver_preload:
SYS_ACCOUNT_ID: SYS_ACCOUNT_JWT
```
## Accessing NATS
The chart contains 2 services by default, `service` and `headlessService`.
### `service`
The `service` is intended to be accessed by NATS Clients. It is a `ClusterIP` service by default, however it can easily be changed to a different service type.
The `nats`, `websocket`, `leafnodes`, and `mqtt` ports will be exposed through this service by default if they are enabled.
Example: change this service type to a `LoadBalancer`:
```yaml
service:
merge:
spec:
type: LoadBalancer
```
### `headlessService`
The `headlessService` is used for NATS Servers in the Stateful Set to discover one another. It is primarily intended to be used for Cluster Route connections.
### TLS Considerations
The TLS Certificate used for Client Connections should have a SAN covering DNS Name that clients access the `service` at.
The TLS Certificate used for Cluster Route Connections should have a SAN covering the DNS Name that routes access each other on the `headlessService` at. This is `*.<headless-service-name>` by default.
## Advanced Features
### Templating Values
Anything in `values.yaml` can be templated:
- maps matching the following syntax will be templated and parsed as YAML:
```yaml
$tplYaml: |
yaml template
```
- maps matching the follow syntax will be templated, parsed as YAML, and spread into the parent map/slice
```yaml
$tplYamlSpread: |
yaml template
```
Example - change service name:
```yaml
service:
name:
$tplYaml: >-
{{ include "nats.fullname" . }}-svc
```
### NATS Config Units and Variables
NATS configuration extends JSON, and can represent Units and Variables. They must be wrapped in `<< >>` in order to template correctly. Example:
```yaml
config:
merge:
authorization:
# variable
token: << $TOKEN >>
# units
max_payload: << 2MB >>
```
templates to the `nats.conf`:
```
{
"authorization": {
"token": $TOKEN
},
"max_payload": 2MB,
"port": 4222,
...
}
```
### NATS Config Includes
Any NATS Config key ending in `$include` will be replaced with an include directive. Included files should be in paths relative to `/etc/nats-config`. Multiple `$include` keys are supported by using a prefix, and will be sorted alphabetically. Example:
```yaml
config:
merge:
00$include: auth.conf
01$include: params.conf
configMap:
merge:
data:
auth.conf: |
accounts: {
A: {
users: [
{user: a, password: a}
]
},
B: {
users: [
{user: b, password: b}
]
},
}
params.conf: |
max_payload: 2MB
```
templates to the `nats.conf`:
```
include auth.conf;
"port": 4222,
...
include params.conf;
```
### Extra Resources
Enables adding additional arbitrary resources. Example - expose WebSocket via VirtualService in Istio:
```yaml
config:
websocket:
enabled: true
extraResources:
- apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
namespace:
$tplYamlSpread: >
{{ include "nats.metadataNamespace" $ }}
name:
$tplYaml: >
{{ include "nats.fullname" $ | quote }}
labels:
$tplYaml: |
{{ include "nats.labels" $ }}
spec:
hosts:
- demo.nats.io
gateways:
- my-gateway
http:
- name: default
match:
- name: root
uri:
exact: /
route:
- destination:
host:
$tplYaml: >
{{ .Values.service.name | quote }}
port:
number:
$tplYaml: >
{{ .Values.config.websocket.port }}
```

155
infra/charts/nats/UPGRADING.md Normal file
View File

@ -0,0 +1,155 @@
# Upgrading from 0.x to 1.x
Instructions for upgrading an existing `nats` 0.x release to 1.x.
## Rename Immutable Fields
There are a number of immutable fields in the NATS Stateful Set and NATS Box deployment. All 1.x `values.yaml` files targeting an existing 0.x release will require some or all of these settings:
```yaml
config:
# required if using JetStream file storage
jetstream:
# uncomment the next line if using JetStream file storage
# enabled: true
fileStore:
pvc:
name:
$tplYaml: >-
{{ include "nats.fullname" . }}-js-pvc
# set other PVC options here to make it match 0.x, refer to values.yaml for schema
# required if using a full or cache resolver
resolver:
# uncomment the next line if using a full or cache resolver
# enabled: true
pvc:
name: nats-jwt-pvc
# set other PVC options here to make it match 0.x, refer to values.yaml for schema
# required
statefulSet:
patch:
- op: remove
path: /spec/selector/matchLabels/app.kubernetes.io~1component
- $tplYamlSpread: |-
{{- if and
.Values.config.jetstream.enabled
.Values.config.jetstream.fileStore.enabled
.Values.config.jetstream.fileStore.pvc.enabled
.Values.config.resolver.enabled
.Values.config.resolver.pvc.enabled
}}
- op: move
from: /spec/volumeClaimTemplates/0
path: /spec/volumeClaimTemplates/1
{{- else}}
[]
{{- end }}
# required
headlessService:
name:
$tplYaml: >-
{{ include "nats.fullname" . }}
# required unless 0.x values explicitly set nats.serviceAccount.create=false
serviceAccount:
enabled: true
# required to use new ClusterIP service for Clients accessing NATS
# if using TLS, this may require adding another SAN
service:
# uncomment the next line to disable the new ClusterIP service
# enabled: false
name:
$tplYaml: >-
{{ include "nats.fullname" . }}-svc
# required if using NatsBox
natsBox:
deployment:
patch:
- op: replace
path: /spec/selector/matchLabels
value:
app: nats-box
- op: add
path: /spec/template/metadata/labels/app
value: nats-box
```
## Update NATS Config to new values.yaml schema
Most values that control the NATS Config have changed and moved under the `config` key. Refer to the 1.x Chart's [values.yaml](values.yaml) for the complete schema.
After migrating to the new values schema, ensure that changes you expect in the NATS Config files match by templating the old and new config files.
Template your old 0.x Config Map, this example uses a file called `values-old.yaml`:
```sh
helm template \
--version "0.x" \
-f values-old.yaml \
-s templates/configmap.yaml \
nats \
nats/nats
```
Template your new 1.x Config Map, this example uses a file called `values.yaml`:
```sh
helm template \
--version "^1-beta" \
-f values.yaml \
-s templates/config-map.yaml \
nats \
nats/nats
```
## Update Kubernetes Resources to new values.yaml schema
Most values that control Kubernetes Resources have been changed. Refer to the 1.x Chart's [values.yaml](values.yaml) for the complete schema.
After migrating to the new values schema, ensure that changes you expect in resources match by templating the old and new resources.
| Resource | 0.x Template File | 1.x Template File |
|-------------------------|---------------------------------|-------------------------------------------|
| Config Map | `templates/configmap.yaml` | `templates/config-map.yaml` |
| Stateful Set | `templates/statefulset.yaml` | `templates/stateful-set.yaml` |
| Headless Service | `templates/service.yaml` | `templates/headless-service.yaml` |
| ClusterIP Service | N/A | `templates/service.yaml` |
| Network Policy | `templates/networkpolicy.yaml` | N/A |
| Pod Disruption Budget | `templates/pdb.yaml` | `templates/pod-disruption-budget.yaml` |
| Service Account | `templates/rbac.yaml` | `templates/service-account.yaml` |
| Resource | `templates/` | `templates/` |
| Resource | `templates/` | `templates/` |
| Prometheus Monitor | `templates/serviceMonitor.yaml` | `templates/pod-monitor.yaml` |
| NatsBox Deployment | `templates/nats-box.yaml` | `templates/nats-box/deployment.yaml` |
| NatsBox Service Account | N/A | `templates/nats-box/service-account.yaml` |
| NatsBox Contents Secret | N/A | `templates/nats-box/contents-secret.yaml` |
| NatsBox Contexts Secret | N/A | `templates/nats-box/contexts-secret.yaml` |
For example, to check that the Stateful Set matches:
Template your old 0.x Stateful Set, this example uses a file called `values-old.yaml`:
```sh
helm template \
--version "0.x" \
-f values-old.yaml \
-s templates/statefulset.yaml \
nats \
nats/nats
```
Template your new 1.x Stateful Set, this example uses a file called `values.yaml`:
```sh
helm template \
--version "^1-beta" \
-f values.yaml \
-s templates/stateful-set.yaml \
nats \
nats/nats
```

10
infra/charts/nats/files/config-map.yaml Normal file
View File

@ -0,0 +1,10 @@
apiVersion: v1
kind: ConfigMap
metadata:
{{- include "nats.metadataNamespace" $ | nindent 2 }}
name: {{ .Values.configMap.name }}
labels:
{{- include "nats.labels" $ | nindent 4 }}
data:
nats.conf: |
{{- include "nats.formatConfig" .config | nindent 4 }}

32
infra/charts/nats/files/config/cluster.yaml Normal file
View File

@ -0,0 +1,32 @@
{{- with .Values.config.cluster }}
name: {{ $.Values.statefulSet.name }}
port: {{ .port }}
no_advertise: {{ .noAdvertise }}
routes:
{{- $proto := ternary "tls" "nats" .tls.enabled }}
{{- $auth := "" }}
{{- if and .routeURLs.user .routeURLs.password }}
{{- $auth = printf "%s:%s@" (urlquery .routeURLs.user) (urlquery .routeURLs.password) -}}
{{- end }}
{{- $domain := $.Values.headlessService.name }}
{{- if .routeURLs.useFQDN }}
{{- $domain = printf "%s.%s.svc.%s" $domain (include "nats.namespace" $) .routeURLs.k8sClusterDomain }}
{{- end }}
{{- $port := (int .port) }}
{{- range $i, $_ := until (int .replicas) }}
- {{ printf "%s://%s%s-%d.%s:%d" $proto $auth $.Values.statefulSet.name $i $domain $port }}
{{- end }}
{{- if and .routeURLs.user .routeURLs.password }}
authorization:
user: {{ .routeURLs.user | quote }}
password: {{ .routeURLs.password | quote }}
{{- end }}
{{- with .tls }}
{{- if .enabled }}
tls:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/tls.yaml" "ctx" (merge (dict "tls" .) $)) .) | nindent 2 }}
{{- end }}
{{- end }}
{{- end }}

114
infra/charts/nats/files/config/config.yaml Normal file
View File

@ -0,0 +1,114 @@
{{- with .Values.config }}
server_name: << $SERVER_NAME >>
lame_duck_grace_period: 10s
lame_duck_duration: 30s
pid_file: /var/run/nats/nats.pid
########################################
# NATS
########################################
{{- with .nats }}
port: {{ .port }}
{{- with .tls }}
{{- if .enabled }}
tls:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/tls.yaml" "ctx" (merge (dict "tls" .) $)) .) | nindent 2 }}
{{- end }}
{{- end }}
{{- end }}
########################################
# leafnodes
########################################
{{- with .leafnodes }}
{{- if .enabled }}
leafnodes:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/leafnodes.yaml" "ctx" $) .) | nindent 2 }}
{{- end }}
{{- end }}
########################################
# websocket
########################################
{{- with .websocket }}
{{- if .enabled }}
websocket:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/websocket.yaml" "ctx" $) .) | nindent 2 }}
{{- end }}
{{- end }}
########################################
# MQTT
########################################
{{- with .mqtt }}
{{- if .enabled }}
mqtt:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/mqtt.yaml" "ctx" $) .) | nindent 2 }}
{{- end }}
{{- end }}
########################################
# cluster
########################################
{{- with .cluster }}
{{- if .enabled }}
cluster:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/cluster.yaml" "ctx" $) .) | nindent 2 }}
{{- end }}
{{- end }}
########################################
# gateway
########################################
{{- with .gateway }}
{{- if .enabled }}
gateway:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/gateway.yaml" "ctx" $) .) | nindent 2 }}
{{- end }}
{{- end }}
########################################
# monitor
########################################
{{- with .monitor }}
{{- if .enabled }}
{{- if .tls.enabled }}
https_port: {{ .port }}
{{- else }}
http_port: {{ .port }}
{{- end }}
{{- end }}
{{- end }}
########################################
# profiling
########################################
{{- with .profiling }}
{{- if .enabled }}
prof_port: {{ .port }}
{{- end }}
{{- end }}
########################################
# jetstream
########################################
{{- with $.Values.config.jetstream -}}
{{- if .enabled }}
jetstream:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/jetstream.yaml" "ctx" $) .) | nindent 2 }}
{{- end }}
{{- end }}
########################################
# resolver
########################################
{{- with $.Values.config.resolver -}}
{{- if .enabled }}
resolver:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/resolver.yaml" "ctx" $) .) | nindent 2 }}
{{- end }}
{{- end }}
{{- end }}

11
infra/charts/nats/files/config/gateway.yaml Normal file
View File

@ -0,0 +1,11 @@
{{- with .Values.config.gateway }}
name: {{ $.Values.statefulSet.name }}
port: {{ .port }}
{{- with .tls }}
{{- if .enabled }}
tls:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/tls.yaml" "ctx" (merge (dict "tls" .) $)) .) | nindent 2 }}
{{- end }}
{{- end }}
{{- end }}

23
infra/charts/nats/files/config/jetstream.yaml Normal file
View File

@ -0,0 +1,23 @@
{{- with .Values.config.jetstream }}
{{- with .memoryStore }}
{{- if .enabled }}
{{- with .maxSize }}
max_memory_store: << {{ . }} >>
{{- end }}
{{- else }}
max_memory_store: 0
{{- end }}
{{- end }}
{{- with .fileStore }}
{{- if .enabled }}
store_dir: {{ .dir }}
{{- if .maxSize }}
max_file_store: << {{ .maxSize }} >>
{{- else if .pvc.enabled }}
max_file_store: << {{ .pvc.size }} >>
{{- end }}
{{- else }}
max_file_store: 0
{{- end }}
{{- end }}
{{- end }}

11
infra/charts/nats/files/config/leafnodes.yaml Normal file
View File

@ -0,0 +1,11 @@
{{- with .Values.config.leafnodes }}
port: {{ .port }}
no_advertise: true
{{- with .tls }}
{{- if .enabled }}
tls:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/tls.yaml" "ctx" (merge (dict "tls" .) $)) .) | nindent 2 }}
{{- end }}
{{- end }}
{{- end }}

10
infra/charts/nats/files/config/mqtt.yaml Normal file
View File

@ -0,0 +1,10 @@
{{- with .Values.config.mqtt }}
port: {{ .port }}
{{- with .tls }}
{{- if .enabled }}
tls:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/tls.yaml" "ctx" (merge (dict "tls" .) $)) .) | nindent 2 }}
{{- end }}
{{- end }}
{{- end }}

10
infra/charts/nats/files/config/protocol.yaml Normal file
View File

@ -0,0 +1,10 @@
{{- with .protocol }}
port: {{ .port }}
{{- with .tls }}
{{- if .enabled }}
tls:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/tls.yaml" "ctx" (merge (dict "tls" .) $)) .) | nindent 2 }}
{{- end }}
{{- end }}
{{- end }}

3
infra/charts/nats/files/config/resolver.yaml Normal file
View File

@ -0,0 +1,3 @@
{{- with .Values.config.resolver }}
dir: {{ .dir }}
{{- end }}

16
infra/charts/nats/files/config/tls.yaml Normal file
View File

@ -0,0 +1,16 @@
# tls
{{- with .tls }}
{{- if .secretName }}
{{- $dir := trimSuffix "/" .dir }}
cert_file: {{ printf "%s/%s" $dir (.cert | default "tls.crt") | quote }}
key_file: {{ printf "%s/%s" $dir (.key | default "tls.key") | quote }}
{{- end }}
{{- end }}
# tlsCA
{{- with $.Values.tlsCA }}
{{- if and .enabled (or .configMapName .secretName) }}
{{- $dir := trimSuffix "/" .dir }}
ca_file: {{ printf "%s/%s" $dir (.key | default "ca.crt") | quote }}
{{- end }}
{{- end }}

12
infra/charts/nats/files/config/websocket.yaml Normal file
View File

@ -0,0 +1,12 @@
{{- with .Values.config.websocket }}
port: {{ .port }}
{{- if .tls.enabled }}
{{- with .tls }}
tls:
{{- include "nats.loadMergePatch" (merge (dict "file" "config/tls.yaml" "ctx" (merge (dict "tls" .) $)) .) | nindent 2 }}
{{- end }}
{{- else }}
no_tls: true
{{- end }}
{{- end }}

24
infra/charts/nats/files/headless-service.yaml Normal file
View File

@ -0,0 +1,24 @@
apiVersion: v1
kind: Service
metadata:
{{- include "nats.metadataNamespace" $ | nindent 2 }}
name: {{ .Values.headlessService.name }}
labels:
{{- include "nats.labels" $ | nindent 4 }}
spec:
selector:
{{- include "nats.selectorLabels" $ | nindent 4 }}
clusterIP: None
publishNotReadyAddresses: true
ports:
{{- range $protocol := list "nats" "leafnodes" "websocket" "mqtt" "cluster" "gateway" "monitor" "profiling" }}
{{- $configProtocol := get $.Values.config $protocol }}
{{- if or (eq $protocol "nats") $configProtocol.enabled }}
{{- $tlsEnabled := false }}
{{- if hasKey $configProtocol "tls" }}
{{- $tlsEnabled = $configProtocol.tls.enabled }}
{{- end }}
{{- $appProtocol := or (eq $protocol "websocket") (eq $protocol "monitor") | ternary ($tlsEnabled | ternary "https" "http") ($tlsEnabled | ternary "tls" "tcp") }}
- {{ dict "name" $protocol "port" $configProtocol.port "targetPort" $protocol "appProtocol" $appProtocol | toYaml | nindent 4 }}
{{- end }}
{{- end }}

34
infra/charts/nats/files/ingress.yaml Normal file
View File

@ -0,0 +1,34 @@
{{- with .Values.config.websocket.ingress }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
{{- include "nats.metadataNamespace" $ | nindent 2 }}
name: {{ .name }}
labels:
{{- include "nats.labels" $ | nindent 4 }}
spec:
{{- with .className }}
ingressClassName: {{ . | quote }}
{{- end }}
rules:
{{- $path := .path }}
{{- $pathType := .pathType }}
{{- range .hosts }}
- host: {{ . | quote }}
http:
paths:
- path: {{ $path | quote }}
pathType: {{ $pathType | quote }}
backend:
service:
name: {{ $.Values.service.name }}
port:
name: websocket
{{- end }}
{{- if .tlsSecretName }}
tls:
- secretName: {{ .tlsSecretName | quote }}
hosts:
{{- toYaml .hosts | nindent 4 }}
{{- end }}
{{- end }}

17
infra/charts/nats/files/nats-box/contents-secret.yaml Normal file
View File

@ -0,0 +1,17 @@
apiVersion: v1
kind: Secret
metadata:
{{- include "nats.metadataNamespace" $ | nindent 2 }}
name: {{ .Values.natsBox.contentsSecret.name }}
labels:
{{- include "natsBox.labels" $ | nindent 4 }}
type: Opaque
stringData:
{{- range $ctxKey, $ctxVal := .Values.natsBox.contexts }}
{{- range $secretKey, $secretVal := dict "creds" "creds" "nkey" "nk" }}
{{- $secret := get $ctxVal $secretKey }}
{{- if and $secret $secret.contents }}
"{{ $ctxKey }}.{{ $secretVal }}": {{ $secret.contents | quote }}
{{- end }}
{{- end }}
{{- end }}

51
infra/charts/nats/files/nats-box/contexts-secret/context.yaml Normal file
View File

@ -0,0 +1,51 @@
{{- $contextName := .contextName }}
# url
{{- if .Values.service.enabled }}
url: nats://{{ .Values.service.name }}
{{- else }}
url: nats://{{ .Values.headlessService.name }}
{{- end }}
{{- with .context }}
# creds
{{- with .creds}}
{{- if .contents }}
creds: /etc/nats-contents/{{ $contextName }}.creds
{{- else if .secretName }}
{{- $dir := trimSuffix "/" .dir }}
creds: {{ printf "%s/%s" $dir (.key | default "nats.creds") | quote }}
{{- end }}
{{- end }}
# nkey
{{- with .nkey}}
{{- if .contents }}
nkey: /etc/nats-contents/{{ $contextName }}.nk
{{- else if .secretName }}
{{- $dir := trimSuffix "/" .dir }}
nkey: {{ printf "%s/%s" $dir (.key | default "nats.nk") | quote }}
{{- end }}
{{- end }}
# tls
{{- with .tls }}
{{- if .secretName }}
{{- $dir := trimSuffix "/" .dir }}
cert: {{ printf "%s/%s" $dir (.cert | default "tls.crt") | quote }}
key: {{ printf "%s/%s" $dir (.key | default "tls.key") | quote }}
{{- end }}
{{- end }}
# tlsCA
{{- if $.Values.config.nats.tls.enabled }}
{{- with $.Values.tlsCA }}
{{- if and .enabled (or .configMapName .secretName) }}
{{- $dir := trimSuffix "/" .dir }}
ca: {{ printf "%s/%s" $dir (.key | default "ca.crt") | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}

13
infra/charts/nats/files/nats-box/contexts-secret/contexts-secret.yaml Normal file
View File

@ -0,0 +1,13 @@
apiVersion: v1
kind: Secret
metadata:
{{- include "nats.metadataNamespace" $ | nindent 2 }}
name: {{ .Values.natsBox.contextsSecret.name }}
labels:
{{- include "natsBox.labels" $ | nindent 4 }}
type: Opaque
stringData:
{{- range $ctxKey, $ctxVal := .Values.natsBox.contexts }}
"{{ $ctxKey }}.json": |
{{- include "toPrettyRawJson" (include "nats.loadMergePatch" (dict "file" "nats-box/contexts-secret/context.yaml" "merge" (.merge | default dict) "patch" (.patch | default list) "ctx" (merge (dict "contextName" $ctxKey "context" $ctxVal) $)) | fromYaml) | nindent 4 }}
{{- end }}

49
infra/charts/nats/files/nats-box/deployment/container.yaml Normal file
View File

@ -0,0 +1,49 @@
name: nats-box
{{ include "nats.image" (merge (pick $.Values "global") .Values.natsBox.container.image) }}
{{- with .Values.natsBox.container.env }}
env:
{{- include "nats.env" . }}
{{- end }}
command:
- sh
- -ec
- |
work_dir="$(pwd)"
mkdir -p "$XDG_CONFIG_HOME/nats"
cd "$XDG_CONFIG_HOME/nats"
if ! [ -s context ]; then
ln -s /etc/nats-contexts context
fi
{{- if .Values.natsBox.defaultContextName }}
if ! [ -f context.txt ]; then
echo -n {{ .Values.natsBox.defaultContextName | quote }} > context.txt
fi
{{- end }}
cd "$work_dir"
exec /entrypoint.sh "$@"
- --
args:
- sh
- -ec
- trap true INT TERM; sleep infinity & wait
volumeMounts:
# contexts secret
- name: contexts
mountPath: /etc/nats-contexts
# contents secret
{{- if .hasContentsSecret }}
- name: contents
mountPath: /etc/nats-contents
{{- end }}
# tlsCA
{{- include "nats.tlsCAVolumeMount" $ }}
# secrets
{{- range (include "natsBox.secretNames" $ | fromJson).secretNames }}
- name: {{ .name | quote }}
mountPath: {{ .dir | quote }}
{{- end }}
resources:
{{- toYaml .Values.natsBox.container.resources | nindent 2 }}

16
infra/charts/nats/files/nats-box/deployment/deployment.yaml Normal file
View File

@ -0,0 +1,16 @@
apiVersion: apps/v1
kind: Deployment
metadata:
{{- include "nats.metadataNamespace" $ | nindent 2 }}
name: {{ .Values.natsBox.deployment.name }}
labels:
{{- include "natsBox.labels" $ | nindent 4 }}
spec:
selector:
matchLabels:
{{- include "natsBox.selectorLabels" $ | nindent 6 }}
replicas: 1
template:
{{- with .Values.natsBox.podTemplate }}
{{ include "nats.loadMergePatch" (merge (dict "file" "nats-box/deployment/pod-template.yaml" "ctx" $) .) | nindent 4 }}
{{- end }}

44
infra/charts/nats/files/nats-box/deployment/pod-template.yaml Normal file
View File

@ -0,0 +1,44 @@
metadata:
labels:
{{- include "natsBox.labels" $ | nindent 4 }}
spec:
containers:
{{- with .Values.natsBox.container }}
- {{ include "nats.loadMergePatch" (merge (dict "file" "nats-box/deployment/container.yaml" "ctx" $) .) | nindent 4 }}
{{- end }}
# service discovery uses DNS; don't need service env vars
enableServiceLinks: false
{{- with .Values.global.image.pullSecretNames }}
imagePullSecrets:
{{- range . }}
- name: {{ . | quote }}
{{- end }}
{{- end }}
{{- with .Values.natsBox.serviceAccount }}
{{- if .enabled }}
serviceAccountName: {{ .name | quote }}
{{- end }}
{{- end }}
volumes:
# contexts secret
- name: contexts
secret:
secretName: {{ .Values.natsBox.contextsSecret.name }}
# contents secret
{{- if .hasContentsSecret }}
- name: contents
secret:
secretName: {{ .Values.natsBox.contentsSecret.name }}
{{- end }}
# tlsCA
{{- include "nats.tlsCAVolume" $ | nindent 2 }}
# secrets
{{- range (include "natsBox.secretNames" $ | fromJson).secretNames }}
- name: {{ .name | quote }}
secret:
secretName: {{ .secretName | quote }}
{{- end }}

7
infra/charts/nats/files/nats-box/service-account.yaml Normal file
View File

@ -0,0 +1,7 @@
apiVersion: v1
kind: ServiceAccount
metadata:
{{- include "nats.metadataNamespace" $ | nindent 2 }}
name: {{ .Values.natsBox.serviceAccount.name }}
labels:
{{- include "natsBox.labels" $ | nindent 4 }}

12
infra/charts/nats/files/pod-disruption-budget.yaml Normal file
View File

@ -0,0 +1,12 @@
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
{{- include "nats.metadataNamespace" $ | nindent 2 }}
name: {{ .Values.podDisruptionBudget.name }}
labels:
{{- include "nats.labels" $ | nindent 4 }}
spec:
maxUnavailable: 1
selector:
matchLabels:
{{- include "nats.selectorLabels" $ | nindent 6 }}

13
infra/charts/nats/files/pod-monitor.yaml Normal file
View File

@ -0,0 +1,13 @@
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
{{- include "nats.metadataNamespace" $ | nindent 2 }}
name: {{ .Values.promExporter.podMonitor.name }}
labels:
{{- include "nats.labels" $ | nindent 4 }}
spec:
selector:
matchLabels:
{{- include "nats.selectorLabels" $ | nindent 6 }}
podMetricsEndpoints:
- port: prom-metrics

7
infra/charts/nats/files/service-account.yaml Normal file
View File

@ -0,0 +1,7 @@
apiVersion: v1
kind: ServiceAccount
metadata:
{{- include "nats.metadataNamespace" $ | nindent 2 }}
name: {{ .Values.serviceAccount.name }}
labels:
{{- include "nats.labels" $ | nindent 4 }}

23
infra/charts/nats/files/service.yaml Normal file
View File

@ -0,0 +1,23 @@
apiVersion: v1
kind: Service
metadata:
{{- include "nats.metadataNamespace" $ | nindent 2 }}
name: {{ .Values.service.name }}
labels:
{{- include "nats.labels" $ | nindent 4 }}
spec:
selector:
{{- include "nats.selectorLabels" $ | nindent 4 }}
ports:
{{- range $protocol := list "nats" "leafnodes" "websocket" "mqtt" "cluster" "gateway" "monitor" "profiling" }}
{{- $configProtocol := get $.Values.config $protocol }}
{{- $servicePort := get $.Values.service.ports $protocol }}
{{- if and (or (eq $protocol "nats") $configProtocol.enabled) $servicePort.enabled }}
{{- $tlsEnabled := false }}
{{- if hasKey $configProtocol "tls" }}
{{- $tlsEnabled = $configProtocol.tls.enabled }}
{{- end }}
{{- $appProtocol := or (eq $protocol "websocket") (eq $protocol "monitor") | ternary ($tlsEnabled | ternary "https" "http") ($tlsEnabled | ternary "tls" "tcp") }}
- {{ merge (dict "name" $protocol "targetPort" $protocol "appProtocol" $appProtocol) (omit $servicePort "enabled") (dict "port" $configProtocol.port) | toYaml | nindent 4 }}
{{- end }}
{{- end }}

13
infra/charts/nats/files/stateful-set/jetstream-pvc.yaml Normal file
View File

@ -0,0 +1,13 @@
{{- with .Values.config.jetstream.fileStore.pvc }}
metadata:
name: {{ .name }}
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ .size | quote }}
{{- with .storageClassName }}
storageClassName: {{ . | quote }}
{{- end }}
{{- end }}

109
infra/charts/nats/files/stateful-set/nats-container.yaml Normal file
View File

@ -0,0 +1,109 @@
name: nats
{{ include "nats.image" (merge (pick $.Values "global") .Values.container.image) }}
ports:
{{- range $protocol := list "nats" "leafnodes" "websocket" "mqtt" "cluster" "gateway" "monitor" "profiling" }}
{{- $configProtocol := get $.Values.config $protocol }}
{{- $containerPort := get $.Values.container.ports $protocol }}
{{- if or (eq $protocol "nats") $configProtocol.enabled }}
- {{ merge (dict "name" $protocol "containerPort" $configProtocol.port) $containerPort | toYaml | nindent 2 }}
{{- end }}
{{- end }}
args:
- --config
- /etc/nats-config/nats.conf
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: SERVER_NAME
value: {{ printf "%s$(POD_NAME)" .Values.config.serverNamePrefix | quote }}
{{- with .Values.container.env }}
{{- include "nats.env" . }}
{{- end }}
lifecycle:
preStop:
exec:
# send the lame duck shutdown signal to trigger a graceful shutdown
command:
- nats-server
- -sl=ldm=/var/run/nats/nats.pid
{{- with .Values.config.monitor }}
{{- if .enabled }}
startupProbe:
httpGet:
path: /healthz
port: monitor
{{- if .tls.enabled }}
scheme: HTTPS
{{- end}}
initialDelaySeconds: 10
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 90
readinessProbe:
httpGet:
path: /healthz?js-server-only=true
port: monitor
{{- if .tls.enabled }}
scheme: HTTPS
{{- end}}
initialDelaySeconds: 10
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 3
livenessProbe:
httpGet:
path: /healthz?js-enabled-only=true
port: monitor
{{- if .tls.enabled }}
scheme: HTTPS
{{- end}}
initialDelaySeconds: 10
timeoutSeconds: 5
periodSeconds: 30
successThreshold: 1
failureThreshold: 3
{{- end }}
{{- end }}
volumeMounts:
# nats config
- name: config
mountPath: /etc/nats-config
# PID volume
- name: pid
mountPath: /var/run/nats
# JetStream PVC
{{- with .Values.config.jetstream }}
{{- if and .enabled .fileStore.enabled .fileStore.pvc.enabled }}
{{- with .fileStore }}
- name: {{ .pvc.name }}
mountPath: {{ .dir | quote }}
{{- end }}
{{- end }}
{{- end }}
# resolver PVC
{{- with .Values.config.resolver }}
{{- if and .enabled .pvc.enabled }}
- name: {{ .pvc.name }}
mountPath: {{ .dir | quote }}
{{- end }}
{{- end }}
# tlsCA
{{- include "nats.tlsCAVolumeMount" $ }}
# secrets
{{- range (include "nats.secretNames" $ | fromJson).secretNames }}
- name: {{ .name | quote }}
mountPath: {{ .dir | quote }}
{{- end }}
resources:
{{- toYaml .Values.container.resources | nindent 2 }}

75
infra/charts/nats/files/stateful-set/pod-template.yaml Normal file
View File

@ -0,0 +1,75 @@
metadata:
labels:
{{- include "nats.labels" $ | nindent 4 }}
annotations:
{{- if .Values.podTemplate.configChecksumAnnotation }}
{{- $configMap := include "nats.loadMergePatch" (merge (dict "file" "config-map.yaml" "ctx" $) $.Values.configMap) }}
checksum/config: {{ sha256sum $configMap }}
{{- end }}
spec:
containers:
# nats
{{- $nats := dict }}
{{- with .Values.container }}
{{- $nats = include "nats.loadMergePatch" (merge (dict "file" "stateful-set/nats-container.yaml" "ctx" $) .) | fromYaml }}
- {{ toYaml $nats | nindent 4 }}
{{- end }}
# reloader
{{- with .Values.reloader }}
{{- if .enabled }}
- {{ include "nats.loadMergePatch" (merge (dict "file" "stateful-set/reloader-container.yaml" "ctx" (merge (dict "natsVolumeMounts" $nats.volumeMounts) $)) .) | nindent 4 }}
{{- end }}
{{- end }}
{{- with .Values.promExporter }}
{{- if .enabled }}
- {{ include "nats.loadMergePatch" (merge (dict "file" "stateful-set/prom-exporter-container.yaml" "ctx" $) .) | nindent 4 }}
{{- end }}
{{- end }}
# service discovery uses DNS; don't need service env vars
enableServiceLinks: false
{{- with .Values.global.image.pullSecretNames }}
imagePullSecrets:
{{- range . }}
- name: {{ . | quote }}
{{- end }}
{{- end }}
{{- with .Values.serviceAccount }}
{{- if .enabled }}
serviceAccountName: {{ .name | quote }}
{{- end }}
{{- end }}
{{- if .Values.reloader.enabled }}
shareProcessNamespace: true
{{- end }}
volumes:
# nats config
- name: config
configMap:
name: {{ .Values.configMap.name }}
# PID volume
- name: pid
emptyDir: {}
# tlsCA
{{- include "nats.tlsCAVolume" $ | nindent 2 }}
# secrets
{{- range (include "nats.secretNames" $ | fromJson).secretNames }}
- name: {{ .name | quote }}
secret:
secretName: {{ .secretName | quote }}
{{- end }}
{{- with .Values.podTemplate.topologySpreadConstraints }}
topologySpreadConstraints:
{{- range $k, $v := . }}
- {{ merge (dict "topologyKey" $k "labelSelector" (dict "matchLabels" (include "nats.selectorLabels" $ | fromYaml))) $v | toYaml | nindent 4 }}
{{- end }}
{{- end}}
# terminationGracePeriodSeconds determines how long to wait for graceful shutdown
# this should be at least `lameDuckGracePeriod` + 20s shutdown overhead
terminationGracePeriodSeconds: 60

31
infra/charts/nats/files/stateful-set/prom-exporter-container.yaml Normal file
View File

@ -0,0 +1,31 @@
name: prom-exporter
{{ include "nats.image" (merge (pick $.Values "global") .Values.promExporter.image) }}
ports:
- name: prom-metrics
containerPort: {{ .Values.promExporter.port }}
{{- with .Values.promExporter.env }}
env:
{{- include "nats.env" . }}
{{- end }}
args:
- -port={{ .Values.promExporter.port }}
- -connz
- -routez
- -subz
- -varz
- -prefix=nats
- -use_internal_server_id
{{- if .Values.config.jetstream.enabled }}
- -jsz=all
{{- end }}
{{- if .Values.config.leafnodes.enabled }}
- -leafz
{{- end }}
{{- if .Values.config.gateway.enabled }}
- -gatewayz
{{- end }}
{{- $monitorProto := ternary "https" "http" .Values.config.monitor.tls.enabled }}
- {{ $monitorProto }}://{{ .Values.promExporter.monitorDomain }}:{{ .Values.config.monitor.port }}/

27
infra/charts/nats/files/stateful-set/reloader-container.yaml Normal file
View File

@ -0,0 +1,27 @@
name: reloader
{{ include "nats.image" (merge (pick $.Values "global") .Values.reloader.image) }}
{{- with .Values.reloader.env }}
env:
{{- include "nats.env" . }}
{{- end }}
args:
- -pid
- /var/run/nats/nats.pid
- -config
- /etc/nats-config/nats.conf
{{ include "nats.reloaderConfig" (dict "config" .config "dir" "/etc/nats-config") }}
volumeMounts:
- name: pid
mountPath: /var/run/nats
{{- range $mnt := .natsVolumeMounts }}
{{- $found := false }}
{{- range $.Values.reloader.natsVolumeMountPrefixes }}
{{- if and (not $found) (hasPrefix . $mnt.mountPath) }}
{{- $found = true }}
- {{ toYaml $mnt | nindent 2}}
{{- end }}
{{- end }}
{{- end }}

13
infra/charts/nats/files/stateful-set/resolver-pvc.yaml Normal file
View File

@ -0,0 +1,13 @@
{{- with .Values.config.resolver.pvc }}
metadata:
name: {{ .name }}
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ .size | quote }}
{{- with .storageClassName }}
storageClassName: {{ . | quote }}
{{- end }}
{{- end }}

37
infra/charts/nats/files/stateful-set/stateful-set.yaml Normal file
View File

@ -0,0 +1,37 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
{{- include "nats.metadataNamespace" $ | nindent 2 }}
name: {{ .Values.statefulSet.name }}
labels:
{{- include "nats.labels" $ | nindent 4 }}
spec:
selector:
matchLabels:
{{- include "nats.selectorLabels" $ | nindent 6 }}
{{- if .Values.config.cluster.enabled }}
replicas: {{ .Values.config.cluster.replicas }}
{{- else }}
replicas: 1
{{- end }}
serviceName: {{ .Values.headlessService.name }}
podManagementPolicy: Parallel
template:
{{- with .Values.podTemplate }}
{{ include "nats.loadMergePatch" (merge (dict "file" "stateful-set/pod-template.yaml" "ctx" $) .) | nindent 4 }}
{{- end }}
volumeClaimTemplates:
{{- with .Values.config.jetstream }}
{{- if and .enabled .fileStore.enabled .fileStore.pvc.enabled }}
{{- with .fileStore.pvc }}
- {{ include "nats.loadMergePatch" (merge (dict "file" "stateful-set/jetstream-pvc.yaml" "ctx" $) .) | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}
{{- with .Values.config.resolver }}
{{- if and .enabled .pvc.enabled }}
{{- with .pvc }}
- {{ include "nats.loadMergePatch" (merge (dict "file" "stateful-set/resolver-pvc.yaml" "ctx" $) .) | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}

289
infra/charts/nats/templates/_helpers.tpl Normal file
View File

@ -0,0 +1,289 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "nats.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "nats.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "nats.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Print the namespace
*/}}
{{- define "nats.namespace" -}}
{{- default .Release.Namespace .Values.namespaceOverride }}
{{- end }}
{{/*
Print the namespace for the metadata section
*/}}
{{- define "nats.metadataNamespace" -}}
{{- with .Values.namespaceOverride }}
namespace: {{ . | quote }}
{{- end }}
{{- end }}
{{/*
Set default values.
*/}}
{{- define "nats.defaultValues" }}
{{- if not .defaultValuesSet }}
{{- $name := include "nats.fullname" . }}
{{- with .Values }}
{{- $_ := set .config.jetstream.fileStore.pvc "name" (.config.jetstream.fileStore.pvc.name | default (printf "%s-js" $name)) }}
{{- $_ := set .config.resolver.pvc "name" (.config.resolver.pvc.name | default (printf "%s-resolver" $name)) }}
{{- $_ := set .config.websocket.ingress "name" (.config.websocket.ingress.name | default (printf "%s-ws" $name)) }}
{{- $_ := set .configMap "name" (.configMap.name | default (printf "%s-config" $name)) }}
{{- $_ := set .headlessService "name" (.headlessService.name | default (printf "%s-headless" $name)) }}
{{- $_ := set .natsBox.contentsSecret "name" (.natsBox.contentsSecret.name | default (printf "%s-box-contents" $name)) }}
{{- $_ := set .natsBox.contextsSecret "name" (.natsBox.contextsSecret.name | default (printf "%s-box-contexts" $name)) }}
{{- $_ := set .natsBox.deployment "name" (.natsBox.deployment.name | default (printf "%s-box" $name)) }}
{{- $_ := set .natsBox.serviceAccount "name" (.natsBox.serviceAccount.name | default (printf "%s-box" $name)) }}
{{- $_ := set .podDisruptionBudget "name" (.podDisruptionBudget.name | default $name) }}
{{- $_ := set .service "name" (.service.name | default $name) }}
{{- $_ := set .serviceAccount "name" (.serviceAccount.name | default $name) }}
{{- $_ := set .statefulSet "name" (.statefulSet.name | default $name) }}
{{- $_ := set .promExporter.podMonitor "name" (.promExporter.podMonitor.name | default $name) }}
{{- end }}
{{- $values := get (include "tplYaml" (dict "doc" .Values "ctx" $) | fromJson) "doc" }}
{{- $_ := set . "Values" $values }}
{{- $hasContentsSecret := false }}
{{- range $ctxKey, $ctxVal := .Values.natsBox.contexts }}
{{- range $secretKey, $secretVal := dict "creds" "nats-creds" "nkey" "nats-nkeys" "tls" "nats-certs" }}
{{- $secret := get $ctxVal $secretKey }}
{{- if $secret }}
{{- $_ := set $secret "dir" ($secret.dir | default (printf "/etc/%s/%s" $secretVal $ctxKey)) }}
{{- if and (ne $secretKey "tls") $secret.contents }}
{{- $hasContentsSecret = true }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- $_ := set $ "hasContentsSecret" $hasContentsSecret }}
{{- with .Values.config }}
{{- $config := include "nats.loadMergePatch" (merge (dict "file" "config/config.yaml" "ctx" $) .) | fromYaml }}
{{- $_ := set $ "config" $config }}
{{- end }}
{{- $_ := set . "defaultValuesSet" true }}
{{- end }}
{{- end }}
{{/*
NATS labels
*/}}
{{- define "nats.labels" -}}
{{- with .Values.global.labels -}}
{{ toYaml . }}
{{ end -}}
helm.sh/chart: {{ include "nats.chart" . }}
{{ include "nats.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
NATS selector labels
*/}}
{{- define "nats.selectorLabels" -}}
app.kubernetes.io/name: {{ include "nats.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: nats
{{- end }}
{{/*
NATS Box labels
*/}}
{{- define "natsBox.labels" -}}
{{- with .Values.global.labels -}}
{{ toYaml . }}
{{ end -}}
helm.sh/chart: {{ include "nats.chart" . }}
{{ include "natsBox.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
NATS Box selector labels
*/}}
{{- define "natsBox.selectorLabels" -}}
app.kubernetes.io/name: {{ include "nats.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: nats-box
{{- end }}
{{/*
Print the image
*/}}
{{- define "nats.image" }}
{{- $image := "" }}
{{- if .digest }}
{{- $image = printf "%s@%s" .repository .digest }}
{{- else }}
{{- $image = printf "%s:%s" .repository .tag }}
{{- end }}
{{- if or .registry .global.image.registry }}
{{- $image = printf "%s/%s" (.registry | default .global.image.registry) $image }}
{{- end }}
{{- if .fullImageName }}
{{- $image = .fullImageName }}
{{- end }}
image: {{ $image }}
{{- if or .pullPolicy .global.image.pullPolicy }}
imagePullPolicy: {{ .pullPolicy | default .global.image.pullPolicy }}
{{- end }}
{{- end }}
{{- define "nats.secretNames" -}}
{{- $secrets := list }}
{{- range $protocol := list "nats" "leafnodes" "websocket" "mqtt" "cluster" "gateway" }}
{{- $configProtocol := get $.Values.config $protocol }}
{{- if and (or (eq $protocol "nats") $configProtocol.enabled) $configProtocol.tls.enabled $configProtocol.tls.secretName }}
{{- $secrets = append $secrets (merge (dict "name" (printf "%s-tls" $protocol)) $configProtocol.tls) }}
{{- end }}
{{- end }}
{{- toJson (dict "secretNames" $secrets) }}
{{- end }}
{{- define "natsBox.secretNames" -}}
{{- $secrets := list }}
{{- range $ctxKey, $ctxVal := .Values.natsBox.contexts }}
{{- range $secretKey, $secretVal := dict "creds" "nats-creds" "nkey" "nats-nkeys" "tls" "nats-certs" }}
{{- $secret := get $ctxVal $secretKey }}
{{- if and $secret $secret.secretName }}
{{- $secrets = append $secrets (merge (dict "name" (printf "ctx-%s-%s" $ctxKey $secretKey)) $secret) }}
{{- end }}
{{- end }}
{{- end }}
{{- toJson (dict "secretNames" $secrets) }}
{{- end }}
{{- define "nats.tlsCAVolume" -}}
{{- with .Values.tlsCA }}
{{- if and .enabled (or .configMapName .secretName) }}
- name: tls-ca
{{- if .configMapName }}
configMap:
name: {{ .configMapName | quote }}
{{- else if .secretName }}
secret:
secretName: {{ .secretName | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- define "nats.tlsCAVolumeMount" -}}
{{- with .Values.tlsCA }}
{{- if and .enabled (or .configMapName .secretName) }}
- name: tls-ca
mountPath: {{ .dir | quote }}
{{- end }}
{{- end }}
{{- end }}
{{/*
translates env var map to list
*/}}
{{- define "nats.env" -}}
{{- range $k, $v := . }}
{{- if kindIs "string" $v }}
- name: {{ $k | quote }}
value: {{ $v | quote }}
{{- else if kindIs "map" $v }}
- {{ merge (dict "name" $k) $v | toYaml | nindent 2 }}
{{- else }}
{{- fail (cat "env var" $k "must be string or map, got" (kindOf $v)) }}
{{- end }}
{{- end }}
{{- end }}
{{- /*
nats.loadMergePatch
input: map with 4 keys:
- file: name of file to load
- ctx: context to pass to tpl
- merge: interface{} to merge
- patch: []interface{} valid JSON Patch document
output: JSON encoded map with 1 key:
- doc: interface{} patched json result
*/}}
{{- define "nats.loadMergePatch" -}}
{{- $doc := tpl (.ctx.Files.Get (printf "files/%s" .file)) .ctx | fromYaml | default dict -}}
{{- $doc = mergeOverwrite $doc (deepCopy (.merge | default dict)) -}}
{{- get (include "jsonpatch" (dict "doc" $doc "patch" (.patch | default list)) | fromJson ) "doc" | toYaml -}}
{{- end }}
{{- /*
nats.reloaderConfig
input: map with 2 keys:
- config: interface{} nats config
- dir: dir config file is in
output: YAML list of reloader config files
*/}}
{{- define "nats.reloaderConfig" -}}
{{- $dir := trimSuffix "/" .dir -}}
{{- with .config -}}
{{- if kindIs "map" . -}}
{{- range $k, $v := . -}}
{{- if or (eq $k "cert_file") (eq $k "key_file") (eq $k "ca_file") }}
- -config
- {{ $v }}
{{- else if hasSuffix "$include" $k }}
- -config
- {{ clean (printf "%s/%s" $dir $v) }}
{{- else }}
{{- include "nats.reloaderConfig" (dict "config" $v "dir" $dir) }}
{{- end -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- /*
nats.formatConfig
input: map[string]interface{}
output: string with following format rules
1. keys ending in $natsRaw are unquoted
2. keys ending in $natsInclude are converted to include directives
*/}}
{{- define "nats.formatConfig" -}}
{{-
(regexReplaceAll "\"<<\\s+(.*?)\\s+>>\""
(regexReplaceAll "\".*\\$include\": \"(.*)\",?" (include "toPrettyRawJson" .) "include ${1};")
"${1}")
-}}
{{- end -}}

219
infra/charts/nats/templates/_jsonpatch.tpl Normal file
View File

@ -0,0 +1,219 @@
{{- /*
jsonpatch
input: map with 2 keys:
- doc: interface{} valid JSON document
- patch: []interface{} valid JSON Patch document
output: JSON encoded map with 1 key:
- doc: interface{} patched json result
*/}}
{{- define "jsonpatch" -}}
{{- $params := fromJson (toJson .) -}}
{{- $patches := $params.patch -}}
{{- $docContainer := pick $params "doc" -}}
{{- range $patch := $patches -}}
{{- if not (hasKey $patch "op") -}}
{{- fail "patch is missing op key" -}}
{{- end -}}
{{- if and (ne $patch.op "add") (ne $patch.op "remove") (ne $patch.op "replace") (ne $patch.op "copy") (ne $patch.op "move") (ne $patch.op "test") -}}
{{- fail (cat "patch has invalid op" $patch.op) -}}
{{- end -}}
{{- if not (hasKey $patch "path") -}}
{{- fail "patch is missing path key" -}}
{{- end -}}
{{- if and (or (eq $patch.op "add") (eq $patch.op "replace") (eq $patch.op "test")) (not (hasKey $patch "value")) -}}
{{- fail (cat "patch with op" $patch.op "is missing value key") -}}
{{- end -}}
{{- if and (or (eq $patch.op "copy") (eq $patch.op "move")) (not (hasKey $patch "from")) -}}
{{- fail (cat "patch with op" $patch.op "is missing from key") -}}
{{- end -}}
{{- $opPathKeys := list "path" -}}
{{- if or (eq $patch.op "copy") (eq $patch.op "move") -}}
{{- $opPathKeys = append $opPathKeys "from" -}}
{{- end -}}
{{- $reSlice := list -}}
{{- range $opPathKey := $opPathKeys -}}
{{- $obj := $docContainer -}}
{{- if and (eq $patch.op "copy") (eq $opPathKey "from") -}}
{{- $obj = (fromJson (toJson $docContainer)) -}}
{{- end -}}
{{- $key := "doc" -}}
{{- $lastMap := dict "root" $obj -}}
{{- $lastKey := "root" -}}
{{- $paths := (splitList "/" (get $patch $opPathKey)) -}}
{{- $firstPath := index $paths 0 -}}
{{- if ne (index $paths 0) "" -}}
{{- fail (cat "invalid" $opPathKey (get $patch $opPathKey) "must be empty string or start with /") -}}
{{- end -}}
{{- $paths = slice $paths 1 -}}
{{- range $path := $paths -}}
{{- $path = replace "~1" "/" $path -}}
{{- $path = replace "~0" "~" $path -}}
{{- if kindIs "slice" $obj -}}
{{- $mapObj := dict -}}
{{- range $i, $v := $obj -}}
{{- $_ := set $mapObj (toString $i) $v -}}
{{- end -}}
{{- $obj = $mapObj -}}
{{- $_ := set $lastMap $lastKey $obj -}}
{{- $reSlice = prepend $reSlice (dict "lastMap" $lastMap "lastKey" $lastKey "mapObj" $obj) -}}
{{- end -}}
{{- if kindIs "map" $obj -}}
{{- if not (hasKey $obj $key) -}}
{{- fail (cat "key" $key "does not exist") -}}
{{- end -}}
{{- $lastKey = $key -}}
{{- $lastMap = $obj -}}
{{- $obj = index $obj $key -}}
{{- $key = $path -}}
{{- else -}}
{{- fail (cat "cannot iterate into path" $key "on type" (kindOf $obj)) -}}
{{- end -}}
{{- end -}}
{{- $_ := set $patch (printf "%sKey" $opPathKey) $key -}}
{{- $_ := set $patch (printf "%sLastKey" $opPathKey) $lastKey -}}
{{- $_ = set $patch (printf "%sLastMap" $opPathKey) $lastMap -}}
{{- end -}}
{{- if eq $patch.op "move" }}
{{- if and (ne $patch.path $patch.from) (hasPrefix (printf "%s/" $patch.path) (printf "%s/" $patch.from)) -}}
{{- fail (cat "from" $patch.from "may not be a child of path" $patch.path) -}}
{{- end -}}
{{- end -}}
{{- if or (eq $patch.op "move") (eq $patch.op "copy") (eq $patch.op "test") }}
{{- $key := $patch.fromKey -}}
{{- $lastMap := $patch.fromLastMap -}}
{{- $lastKey := $patch.fromLastKey -}}
{{- $setKey := "value" -}}
{{- if eq $patch.op "test" }}
{{- $key = $patch.pathKey -}}
{{- $lastMap = $patch.pathLastMap -}}
{{- $lastKey = $patch.pathLastKey -}}
{{- $setKey = "testValue" -}}
{{- end -}}
{{- $obj := index $lastMap $lastKey -}}
{{- if kindIs "map" $obj -}}
{{- if not (hasKey $obj $key) -}}
{{- fail (cat $key "does not exist") -}}
{{- end -}}
{{- $_ := set $patch $setKey (index $obj $key) -}}
{{- else if kindIs "slice" $obj -}}
{{- $i := atoi $key -}}
{{- if ne $key (toString $i) -}}
{{- fail (cat "cannot convert" $key "to int") -}}
{{- end -}}
{{- if lt $i 0 -}}
{{- fail "slice index <0" -}}
{{- else if lt $i (len $obj) -}}
{{- $_ := set $patch $setKey (index $obj $i) -}}
{{- else -}}
{{- fail "slice index >= slice length" -}}
{{- end -}}
{{- else -}}
{{- fail (cat "cannot" $patch.op $key "on type" (kindOf $obj)) -}}
{{- end -}}
{{- end -}}
{{- if or (eq $patch.op "remove") (eq $patch.op "replace") (eq $patch.op "move") }}
{{- $key := $patch.pathKey -}}
{{- $lastMap := $patch.pathLastMap -}}
{{- $lastKey := $patch.pathLastKey -}}
{{- if eq $patch.op "move" }}
{{- $key = $patch.fromKey -}}
{{- $lastMap = $patch.fromLastMap -}}
{{- $lastKey = $patch.fromLastKey -}}
{{- end -}}
{{- $obj := index $lastMap $lastKey -}}
{{- if kindIs "map" $obj -}}
{{- if not (hasKey $obj $key) -}}
{{- fail (cat $key "does not exist") -}}
{{- end -}}
{{- $_ := unset $obj $key -}}
{{- else if kindIs "slice" $obj -}}
{{- $i := atoi $key -}}
{{- if ne $key (toString $i) -}}
{{- fail (cat "cannot convert" $key "to int") -}}
{{- end -}}
{{- if lt $i 0 -}}
{{- fail "slice index <0" -}}
{{- else if eq $i 0 -}}
{{- $_ := set $lastMap $lastKey (slice $obj 1) -}}
{{- else if lt $i (sub (len $obj) 1) -}}
{{- $_ := set $lastMap $lastKey (concat (slice $obj 0 $i) (slice $obj (add $i 1) (len $obj))) -}}
{{- else if eq $i (sub (len $obj) 1) -}}
{{- $_ := set $lastMap $lastKey (slice $obj 0 (sub (len $obj) 1)) -}}
{{- else -}}
{{- fail "slice index >= slice length" -}}
{{- end -}}
{{- else -}}
{{- fail (cat "cannot" $patch.op $key "on type" (kindOf $obj)) -}}
{{- end -}}
{{- end -}}
{{- if or (eq $patch.op "add") (eq $patch.op "replace") (eq $patch.op "move") (eq $patch.op "copy") }}
{{- $key := $patch.pathKey -}}
{{- $lastMap := $patch.pathLastMap -}}
{{- $lastKey := $patch.pathLastKey -}}
{{- $value := $patch.value -}}
{{- $obj := index $lastMap $lastKey -}}
{{- if kindIs "map" $obj -}}
{{- $_ := set $obj $key $value -}}
{{- else if kindIs "slice" $obj -}}
{{- $i := 0 -}}
{{- if eq $key "-" -}}
{{- $i = len $obj -}}
{{- else -}}
{{- $i = atoi $key -}}
{{- if ne $key (toString $i) -}}
{{- fail (cat "cannot convert" $key "to int") -}}
{{- end -}}
{{- end -}}
{{- if lt $i 0 -}}
{{- fail "slice index <0" -}}
{{- else if eq $i 0 -}}
{{- $_ := set $lastMap $lastKey (prepend $obj $value) -}}
{{- else if lt $i (len $obj) -}}
{{- $_ := set $lastMap $lastKey (concat (append (slice $obj 0 $i) $value) (slice $obj $i)) -}}
{{- else if eq $i (len $obj) -}}
{{- $_ := set $lastMap $lastKey (append $obj $value) -}}
{{- else -}}
{{- fail "slice index > slice length" -}}
{{- end -}}
{{- else -}}
{{- fail (cat "cannot" $patch.op $key "on type" (kindOf $obj)) -}}
{{- end -}}
{{- end -}}
{{- if eq $patch.op "test" }}
{{- if not (deepEqual $patch.value $patch.testValue) }}
{{- fail (cat "test failed, expected" (toJson $patch.value) "but got" (toJson $patch.testValue)) -}}
{{- end -}}
{{- end -}}
{{- range $reSliceOp := $reSlice -}}
{{- $sliceObj := list -}}
{{- range $i := until (len $reSliceOp.mapObj) -}}
{{- $sliceObj = append $sliceObj (index $reSliceOp.mapObj (toString $i)) -}}
{{- end -}}
{{- $_ := set $reSliceOp.lastMap $reSliceOp.lastKey $sliceObj -}}
{{- end -}}
{{- end -}}
{{- toJson $docContainer -}}
{{- end -}}

28
infra/charts/nats/templates/_toPrettyRawJson.tpl Normal file
View File

@ -0,0 +1,28 @@
{{- /*
toPrettyRawJson
input: interface{} valid JSON document
output: pretty raw JSON string
*/}}
{{- define "toPrettyRawJson" -}}
{{- include "toPrettyRawJsonStr" (toPrettyJson .) -}}
{{- end -}}
{{- /*
toPrettyRawJsonStr
input: pretty JSON string
output: pretty raw JSON string
*/}}
{{- define "toPrettyRawJsonStr" -}}
{{- $s :=
(regexReplaceAll "([^\\\\](?:\\\\\\\\)*)\\\\u003e"
(regexReplaceAll "([^\\\\](?:\\\\\\\\)*)\\\\u003c"
(regexReplaceAll "([^\\\\](?:\\\\\\\\)*)\\\\u0026" . "${1}&")
"${1}<")
"${1}>")
-}}
{{- if regexMatch "([^\\\\](?:\\\\\\\\)*)\\\\u00(26|3c|3e)" $s -}}
{{- include "toPrettyRawJsonStr" $s -}}
{{- else -}}
{{- $s -}}
{{- end -}}
{{- end -}}

114
infra/charts/nats/templates/_tplYaml.tpl Normal file
View File

@ -0,0 +1,114 @@
{{- /*
tplYaml
input: map with 2 keys:
- doc: interface{}
- ctx: context to pass to tpl function
output: JSON encoded map with 1 key:
- doc: interface{} with any keys called tpl or tplSpread values templated and replaced
maps matching the following syntax will be templated and parsed as YAML
{
$tplYaml: string
}
maps matching the follow syntax will be templated, parsed as YAML, and spread into the parent map/slice
{
$tplYamlSpread: string
}
*/}}
{{- define "tplYaml" -}}
{{- $patch := get (include "tplYamlItr" (dict "ctx" .ctx "parentKind" "" "parentPath" "" "path" "/" "value" .doc) | fromJson) "patch" -}}
{{- include "jsonpatch" (dict "doc" .doc "patch" $patch) -}}
{{- end -}}
{{- /*
tplYamlItr
input: map with 4 keys:
- path: string JSONPath to current element
- parentKind: string kind of parent element
- parentPath: string JSONPath to parent element
- value: interface{}
- ctx: context to pass to tpl function
output: JSON encoded map with 1 key:
- patch: list of patches to apply in order to template
*/}}
{{- define "tplYamlItr" -}}
{{- $params := . -}}
{{- $kind := kindOf $params.value -}}
{{- $patch := list -}}
{{- $joinPath := $params.path -}}
{{- if eq $params.path "/" -}}
{{- $joinPath = "" -}}
{{- end -}}
{{- $joinParentPath := $params.parentPath -}}
{{- if eq $params.parentPath "/" -}}
{{- $joinParentPath = "" -}}
{{- end -}}
{{- if eq $kind "slice" -}}
{{- $iAdj := 0 -}}
{{- range $i, $v := $params.value -}}
{{- $iPath := printf "%s/%d" $joinPath (add $i $iAdj) -}}
{{- $itrPatch := get (include "tplYamlItr" (dict "ctx" $params.ctx "parentKind" $kind "parentPath" $params.path "path" $iPath "value" $v) | fromJson) "patch" -}}
{{- $itrLen := len $itrPatch -}}
{{- if gt $itrLen 0 -}}
{{- $patch = concat $patch $itrPatch -}}
{{- if eq (get (index $itrPatch 0) "op") "remove" -}}
{{- $iAdj = add $iAdj (sub $itrLen 2) -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- else if eq $kind "map" -}}
{{- if and (eq (len $params.value) 1) (or (hasKey $params.value "$tplYaml") (hasKey $params.value "$tplYamlSpread")) -}}
{{- $tpl := get $params.value "$tplYaml" -}}
{{- $spread := false -}}
{{- if hasKey $params.value "$tplYamlSpread" -}}
{{- if eq $params.path "/" -}}
{{- fail "cannot $tplYamlSpread on root object" -}}
{{- end -}}
{{- $tpl = get $params.value "$tplYamlSpread" -}}
{{- $spread = true -}}
{{- end -}}
{{- $res := tpl $tpl $params.ctx -}}
{{- $res = get (fromYaml (tpl "tpl: {{ nindent 2 .res }}" (merge (dict "res" $res) $params.ctx))) "tpl" -}}
{{- if eq $spread false -}}
{{- $patch = append $patch (dict "op" "replace" "path" $params.path "value" $res) -}}
{{- else -}}
{{- $resKind := kindOf $res -}}
{{- if and (ne $resKind "invalid") (ne $resKind $params.parentKind) -}}
{{- fail (cat "can only $tplYamlSpread slice onto a slice or map onto a map; attempted to spread" $resKind "on" $params.parentKind "at path" $params.path) -}}
{{- end -}}
{{- $patch = append $patch (dict "op" "remove" "path" $params.path) -}}
{{- if eq $resKind "invalid" -}}
{{- /* no-op */ -}}
{{- else if eq $resKind "slice" -}}
{{- range $v := reverse $res -}}
{{- $patch = append $patch (dict "op" "add" "path" $params.path "value" $v) -}}
{{- end -}}
{{- else -}}
{{- range $k, $v := $res -}}
{{- $kPath := replace "~" "~0" $k -}}
{{- $kPath = replace "/" "~1" $kPath -}}
{{- $kPath = printf "%s/%s" $joinParentPath $kPath -}}
{{- $patch = append $patch (dict "op" "add" "path" $kPath "value" $v) -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- else -}}
{{- range $k, $v := $params.value -}}
{{- $kPath := replace "~" "~0" $k -}}
{{- $kPath = replace "/" "~1" $kPath -}}
{{- $kPath = printf "%s/%s" $joinPath $kPath -}}
{{- $itrPatch := get (include "tplYamlItr" (dict "ctx" $params.ctx "parentKind" $kind "parentPath" $params.path "path" $kPath "value" $v) | fromJson) "patch" -}}
{{- if gt (len $itrPatch) 0 -}}
{{- $patch = concat $patch $itrPatch -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- toJson (dict "patch" $patch) -}}
{{- end -}}

4
infra/charts/nats/templates/config-map.yaml Normal file
View File

@ -0,0 +1,4 @@
{{- include "nats.defaultValues" . }}
{{- with .Values.configMap }}
{{- include "nats.loadMergePatch" (merge (dict "file" "config-map.yaml" "ctx" $) .) }}
{{- end }}

5
infra/charts/nats/templates/extra-resources.yaml Normal file
View File

@ -0,0 +1,5 @@
{{- include "nats.defaultValues" . }}
{{- range .Values.extraResources }}
---
{{ . | toYaml }}
{{- end }}

4
infra/charts/nats/templates/headless-service.yaml Normal file
View File

@ -0,0 +1,4 @@
{{- include "nats.defaultValues" . }}
{{- with .Values.headlessService }}
{{- include "nats.loadMergePatch" (merge (dict "file" "headless-service.yaml" "ctx" $) .) }}
{{- end }}

6
infra/charts/nats/templates/ingress.yaml Normal file
View File

@ -0,0 +1,6 @@
{{- include "nats.defaultValues" . }}
{{- with .Values.config.websocket.ingress }}
{{- if and .enabled .hosts $.Values.config.websocket.enabled $.Values.service.enabled $.Values.service.ports.websocket.enabled }}
{{- include "nats.loadMergePatch" (merge (dict "file" "ingress.yaml" "ctx" $) .) }}
{{- end }}
{{- end }}

10
infra/charts/nats/templates/nats-box/contents-secret.yaml Normal file
View File

@ -0,0 +1,10 @@
{{- include "nats.defaultValues" . }}
{{- if .hasContentsSecret }}
{{- with .Values.natsBox }}
{{- if .enabled }}
{{- with .contentsSecret}}
{{- include "nats.loadMergePatch" (merge (dict "file" "nats-box/contents-secret.yaml" "ctx" $) .) }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}

8
infra/charts/nats/templates/nats-box/contexts-secret.yaml Normal file
View File

@ -0,0 +1,8 @@
{{- include "nats.defaultValues" . }}
{{- with .Values.natsBox }}
{{- if .enabled }}
{{- with .contextsSecret}}
{{- include "nats.loadMergePatch" (merge (dict "file" "nats-box/contexts-secret/contexts-secret.yaml" "ctx" $) .) }}
{{- end }}
{{- end }}
{{- end }}

8
infra/charts/nats/templates/nats-box/deployment.yaml Normal file
View File

@ -0,0 +1,8 @@
{{- include "nats.defaultValues" . }}
{{- with .Values.natsBox }}
{{- if .enabled }}
{{- with .deployment }}
{{- include "nats.loadMergePatch" (merge (dict "file" "nats-box/deployment/deployment.yaml" "ctx" $) .) }}
{{- end }}
{{- end }}
{{- end }}

8
infra/charts/nats/templates/nats-box/service-account.yaml Normal file
View File

@ -0,0 +1,8 @@
{{- include "nats.defaultValues" . }}
{{- if .Values.natsBox.enabled }}
{{- with .Values.natsBox.serviceAccount }}
{{- if .enabled }}
{{- include "nats.loadMergePatch" (merge (dict "file" "nats-box/service-account.yaml" "ctx" $) .) }}
{{- end }}
{{- end }}
{{- end }}

6
infra/charts/nats/templates/pod-disruption-budget.yaml Normal file
View File

@ -0,0 +1,6 @@
{{- include "nats.defaultValues" . }}
{{- with .Values.podDisruptionBudget }}
{{- if .enabled }}
{{- include "nats.loadMergePatch" (merge (dict "file" "pod-disruption-budget.yaml" "ctx" $) .) }}
{{- end }}
{{- end }}

8
infra/charts/nats/templates/pod-monitor.yaml Normal file
View File

@ -0,0 +1,8 @@
{{- include "nats.defaultValues" . }}
{{- with .Values.promExporter }}
{{- if and .enabled .podMonitor.enabled }}
{{- with .podMonitor }}
{{- include "nats.loadMergePatch" (merge (dict "file" "pod-monitor.yaml" "ctx" $) .) }}
{{- end }}
{{- end }}
{{- end }}

6
infra/charts/nats/templates/service-account.yaml Normal file
View File

@ -0,0 +1,6 @@
{{- include "nats.defaultValues" . }}
{{- with .Values.serviceAccount }}
{{- if .enabled }}
{{- include "nats.loadMergePatch" (merge (dict "file" "service-account.yaml" "ctx" $) .) }}
{{- end }}
{{- end }}

6
infra/charts/nats/templates/service.yaml Normal file
View File

@ -0,0 +1,6 @@
{{- include "nats.defaultValues" . }}
{{- with .Values.service }}
{{- if .enabled }}
{{- include "nats.loadMergePatch" (merge (dict "file" "service.yaml" "ctx" $) .) }}
{{- end }}
{{- end }}

4
infra/charts/nats/templates/stateful-set.yaml Normal file
View File

@ -0,0 +1,4 @@
{{- include "nats.defaultValues" . }}
{{- with .Values.statefulSet }}
{{- include "nats.loadMergePatch" (merge (dict "file" "stateful-set/stateful-set.yaml" "ctx" $) .) }}
{{- end }}

37
infra/charts/nats/templates/tests/request-reply.yaml Normal file
View File

@ -0,0 +1,37 @@
{{- include "nats.defaultValues" . }}
{{- with .Values.natsBox | deepCopy }}
{{- $natsBox := . }}
{{- if .enabled -}}
apiVersion: v1
kind: Pod
{{- with .container }}
{{- $_ := set . "merge" (dict
"args" (list
"sh"
"-ec"
"nats reply --echo echo & pid=\"$!\"; sleep 1; nats request echo hi > /tmp/resp; kill \"$pid\"; wait; grep -qF hi /tmp/resp"
)
) }}
{{- $_ := set . "patch" list }}
{{- end }}
{{- with .podTemplate }}
{{- $_ := set . "merge" (dict
"metadata" (dict
"name" (printf "%s-test-request-reply" $.Values.statefulSet.name)
"labels" (dict
"app.kubernetes.io/component" "test-request-reply"
)
"annotations" (dict
"helm.sh/hook" "test"
"helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded"
)
)
"spec" (dict
"restartPolicy" "Never"
)
) }}
{{- $_ := set . "patch" list }}
{{ include "nats.loadMergePatch" (merge (dict "file" "nats-box/deployment/pod-template.yaml" "ctx" (merge (dict "Values" (dict "natsBox" $natsBox)) $)) .) }}
{{- end }}
{{- end }}
{{- end }}

696
infra/charts/nats/values.yaml Normal file
View File

@ -0,0 +1,696 @@
################################################################################
# Global options
################################################################################
global:
image:
# global image pull policy to use for all container images in the chart
# can be overridden by individual image pullPolicy
pullPolicy:
# global list of secret names to use as image pull secrets for all pod specs in the chart
# secrets must exist in the same namespace
# https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
pullSecretNames: []
# global registry to use for all container images in the chart
# can be overridden by individual image registry
registry:
# global labels will be applied to all resources deployed by the chart
labels: {}
################################################################################
# Common options
################################################################################
# override name of the chart
nameOverride:
# override full name of the chart+release
fullnameOverride:
# override the namespace that resources are installed into
namespaceOverride:
# reference a common CA Certificate or Bundle in all nats config `tls` blocks and nats-box contexts
# note: `tls.verify` still must be set in the appropriate nats config `tls` blocks to require mTLS
tlsCA:
enabled: false
# set configMapName in order to mount an existing configMap to dir
configMapName:
# set secretName in order to mount an existing secretName to dir
secretName:
# directory to mount the configMap or secret to
dir: /etc/nats-ca-cert
# key in the configMap or secret that contains the CA Certificate or Bundle
key: ca.crt
################################################################################
# NATS Stateful Set and associated resources
################################################################################
############################################################
# NATS config
############################################################
config:
cluster:
enabled: false
port: 6222
# must be 2 or higher when jetstream is enabled
replicas: 3
# set to false to allow cluster nodes to advertise their addresses
# so that clients can reconnect without extra DNS lookups.
# Note: in case clients have external connectivity make sure to define the `advertise` section as well.
# If clients are behind a load balancer it is best to leave this as is.
noAdvertise: true
# apply to generated route URLs that connect to other pods in the StatefulSet
routeURLs:
# if both user and password are set, they will be added to route URLs
# and the cluster authorization block
user:
password:
# set to true to use FQDN in route URLs
useFQDN: false
k8sClusterDomain: cluster.local
tls:
enabled: false
# set secretName in order to mount an existing secret to dir
secretName:
dir: /etc/nats-certs/cluster
cert: tls.crt
key: tls.key
# merge or patch the tls config
# https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
merge: {}
patch: []
# merge or patch the cluster config
# https://docs.nats.io/running-a-nats-service/configuration/clustering/cluster_config
merge: {}
patch: []
jetstream:
enabled: false
fileStore:
enabled: true
dir: /data
############################################################
# stateful set -> volume claim templates -> jetstream pvc
############################################################
pvc:
enabled: true
size: 10Gi
storageClassName:
# merge or patch the jetstream pvc
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#persistentvolumeclaim-v1-core
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}-js"
name:
# defaults to the PVC size
maxSize:
memoryStore:
enabled: false
# ensure that container has a sufficient memory limit greater than maxSize
maxSize: 1Gi
# merge or patch the jetstream config
# https://docs.nats.io/running-a-nats-service/configuration#jetstream
merge: {}
patch: []
nats:
port: 4222
tls:
enabled: false
# set secretName in order to mount an existing secret to dir
secretName:
dir: /etc/nats-certs/nats
cert: tls.crt
key: tls.key
# merge or patch the tls config
# https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
merge: {}
patch: []
leafnodes:
enabled: false
port: 7422
tls:
enabled: false
# set secretName in order to mount an existing secret to dir
secretName:
dir: /etc/nats-certs/leafnodes
cert: tls.crt
key: tls.key
# merge or patch the tls config
# https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
merge: {}
patch: []
# merge or patch the leafnodes config
# https://docs.nats.io/running-a-nats-service/configuration/leafnodes/leafnode_conf
merge: {}
patch: []
websocket:
enabled: false
port: 8080
tls:
enabled: false
# set secretName in order to mount an existing secret to dir
secretName:
dir: /etc/nats-certs/websocket
cert: tls.crt
key: tls.key
# merge or patch the tls config
# https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
merge: {}
patch: []
############################################################
# ingress
############################################################
# service must be enabled also
ingress:
enabled: false
# must contain at least 1 host otherwise ingress will not be created
hosts: []
path: /
pathType: Exact
# sets to the ingress class name
className:
# set to an existing secret name to enable TLS on the ingress; applies to all hosts
tlsSecretName:
# merge or patch the ingress
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#ingress-v1-networking-k8s-io
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}-ws"
name:
# merge or patch the websocket config
# https://docs.nats.io/running-a-nats-service/configuration/websocket/websocket_conf
merge: {}
patch: []
mqtt:
enabled: false
port: 1883
tls:
enabled: false
# set secretName in order to mount an existing secret to dir
secretName:
dir: /etc/nats-certs/mqtt
cert: tls.crt
key: tls.key
# merge or patch the tls config
# https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
merge: {}
patch: []
# merge or patch the mqtt config
# https://docs.nats.io/running-a-nats-service/configuration/mqtt/mqtt_config
merge: {}
patch: []
gateway:
enabled: false
port: 7222
tls:
enabled: false
# set secretName in order to mount an existing secret to dir
secretName:
dir: /etc/nats-certs/gateway
cert: tls.crt
key: tls.key
# merge or patch the tls config
# https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
merge: {}
patch: []
# merge or patch the gateway config
# https://docs.nats.io/running-a-nats-service/configuration/gateways/gateway#gateway-configuration-block
merge: {}
patch: []
monitor:
enabled: true
port: 8222
tls:
# config.nats.tls must be enabled also
# when enabled, monitoring port will use HTTPS with the options from config.nats.tls
# if promExporter is also enabled, consider setting promExporter.monitorDomain
enabled: false
profiling:
enabled: false
port: 65432
resolver:
enabled: false
dir: /data/resolver
############################################################
# stateful set -> volume claim templates -> resolver pvc
############################################################
pvc:
enabled: true
size: 1Gi
storageClassName:
# merge or patch the pvc
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#persistentvolumeclaim-v1-core
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}-resolver"
name:
# merge or patch the resolver
# https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/jwt/resolver
merge: {}
patch: []
# adds a prefix to the server name, which defaults to the pod name
# helpful for ensuring server name is unique in a super cluster
serverNamePrefix: ""
# merge or patch the nats config
# https://docs.nats.io/running-a-nats-service/configuration
# following special rules apply
# 1. strings that start with << and end with >> will be unquoted
# use this for variables and numbers with units
# 2. keys ending in $include will be switched to include directives
# keys are sorted alphabetically, use prefix before $includes to control includes ordering
# paths should be relative to /etc/nats-config/nats.conf
# example:
#
# merge:
# $include: ./my-config.conf
# zzz$include: ./my-config-last.conf
# server_name: nats
# authorization:
# token: << $TOKEN >>
# jetstream:
# max_memory_store: << 1GB >>
#
# will yield the config:
# {
# include ./my-config.conf;
# "authorization": {
# "token": $TOKEN
# },
# "jetstream": {
# "max_memory_store": 1GB
# },
# "server_name": "nats",
# include ./my-config-last.conf;
# }
merge: {}
patch: []
############################################################
# stateful set -> pod template -> nats container
############################################################
container:
image:
repository: nats
tag: 2.12.5-alpine
pullPolicy:
registry:
# if digest is provided, it overrides tag (example: "sha256:abcdef1234567890")
digest:
# if fullImageName is provided, it overrides registry, repository, tag, and digest
fullImageName:
# container port options
# must be enabled in the config section also
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#containerport-v1-core
ports:
nats: {}
leafnodes: {}
websocket: {}
mqtt: {}
cluster: {}
gateway: {}
monitor: {}
profiling: {}
# map with key as env var name, value can be string or map
# example:
#
# env:
# GOMEMLIMIT: 7GiB
# TOKEN:
# valueFrom:
# secretKeyRef:
# name: nats-auth
# key: token
env: {}
# merge or patch the container
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
merge: {}
patch: []
# container resources
resources: {}
# requests:
# cpu: 100m
# memory: 128Mi
# limits:
# cpu: 100m
# memory: 128Mi
############################################################
# stateful set -> pod template -> reloader container
############################################################
reloader:
enabled: true
image:
repository: natsio/nats-server-config-reloader
tag: 0.21.1
pullPolicy:
registry:
digest:
fullImageName:
# env var map, see nats.env for an example
env: {}
# all nats container volume mounts with the following prefixes
# will be mounted into the reloader container
natsVolumeMountPrefixes:
- /etc/
# merge or patch the container
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
merge: {}
patch: []
############################################################
# stateful set -> pod template -> prom-exporter container
############################################################
# config.monitor must be enabled
promExporter:
enabled: false
image:
repository: natsio/prometheus-nats-exporter
tag: 0.18.0
pullPolicy:
registry:
digest:
fullImageName:
port: 7777
# if config.monitor.tls.enabled is set to true, monitorDomain must be set to the common name
# or a SAN used in the tls certificate
monitorDomain: localhost
# env var map, see nats.env for an example
env: {}
# merge or patch the container
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
merge: {}
patch: []
############################################################
# prometheus pod monitor
############################################################
podMonitor:
enabled: false
# merge or patch the pod monitor
# https://prometheus-operator.dev/docs/api-reference/api/#monitoring.coreos.com/v1.PodMonitor
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}"
name:
############################################################
# service
############################################################
service:
enabled: true
# service port options
# additional boolean field enable to control whether port is exposed in the service
# must be enabled in the config section also
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#serviceport-v1-core
ports:
nats:
enabled: true
leafnodes:
enabled: true
websocket:
enabled: true
mqtt:
enabled: true
cluster:
enabled: false
gateway:
enabled: false
monitor:
enabled: false
profiling:
enabled: false
# merge or patch the service
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#service-v1-core
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}"
name:
############################################################
# other nats extension points
############################################################
# stateful set
statefulSet:
# merge or patch the stateful set
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#statefulset-v1-apps
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}"
name:
# stateful set -> pod template
podTemplate:
# adds a hash of the ConfigMap as a pod annotation
# this will cause the StatefulSet to roll when the ConfigMap is updated
# set to true to force pod rollouts on config changes instead of using the reloader for hot updates
configChecksumAnnotation: false
# map of topologyKey: topologySpreadConstraint
# labelSelector will be added to match StatefulSet pods
#
# topologySpreadConstraints:
# kubernetes.io/hostname:
# maxSkew: 1
#
topologySpreadConstraints: {}
# merge or patch the pod template
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#pod-v1-core
merge: {}
patch: []
# headless service
headlessService:
# merge or patch the headless service
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#service-v1-core
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}-headless"
name:
# config map
configMap:
# merge or patch the config map
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#configmap-v1-core
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}-config"
name:
# pod disruption budget
podDisruptionBudget:
enabled: true
# merge or patch the pod disruption budget
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#poddisruptionbudget-v1-policy
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}"
name:
# service account
serviceAccount:
enabled: false
# merge or patch the service account
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#serviceaccount-v1-core
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}"
name:
############################################################
# natsBox
#
# NATS Box Deployment and associated resources
############################################################
natsBox:
enabled: true
############################################################
# NATS contexts
############################################################
contexts:
default:
creds:
# set contents in order to create a secret with the creds file contents
contents:
# set secretName in order to mount an existing secret to dir
secretName:
# defaults to /etc/nats-creds/<context-name>
dir:
key: nats.creds
nkey:
# set contents in order to create a secret with the nkey file contents
contents:
# set secretName in order to mount an existing secret to dir
secretName:
# defaults to /etc/nats-nkeys/<context-name>
dir:
key: nats.nk
# used to connect with client certificates
tls:
# set secretName in order to mount an existing secret to dir
secretName:
# defaults to /etc/nats-certs/<context-name>
dir:
cert: tls.crt
key: tls.key
# merge or patch the context
# https://docs.nats.io/using-nats/nats-tools/nats_cli#nats-contexts
merge: {}
patch: []
# name of context to select by default
defaultContextName: default
############################################################
# deployment -> pod template -> nats-box container
############################################################
container:
image:
repository: natsio/nats-box
tag: 0.19.3
pullPolicy:
registry:
digest:
fullImageName:
resources: {}
# env var map, see nats.env for an example
env: {}
# merge or patch the container
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
merge: {}
patch: []
############################################################
# other nats-box extension points
############################################################
# deployment
deployment:
# merge or patch the deployment
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#deployment-v1-apps
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}-box"
name:
# deployment -> pod template
podTemplate:
# merge or patch the pod template
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#pod-v1-core
merge: {}
patch: []
# contexts secret
contextsSecret:
# merge or patch the context secret
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secret-v1-core
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}-box-contexts"
name:
# contents secret
contentsSecret:
# merge or patch the contents secret
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secret-v1-core
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}-box-contents"
name:
# service account
serviceAccount:
enabled: false
# merge or patch the service account
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#serviceaccount-v1-core
merge: {}
patch: []
# defaults to "{{ include "nats.fullname" $ }}-box"
name:
################################################################################
# Extra user-defined resources
################################################################################
#
# add arbitrary user-generated resources
# example:
#
# config:
# websocket:
# enabled: true
# extraResources:
# - apiVersion: networking.istio.io/v1beta1
# kind: VirtualService
# metadata:
# name:
# $tplYaml: >
# {{ include "nats.fullname" $ | quote }}
# labels:
# $tplYaml: |
# {{ include "nats.labels" $ }}
# spec:
# hosts:
# - demo.nats.io
# gateways:
# - my-gateway
# http:
# - name: default
# match:
# - name: root
# uri:
# exact: /
# route:
# - destination:
# host:
# $tplYaml: >
# {{ .Values.service.name | quote }}
# port:
# number:
# $tplYaml: >
# {{ .Values.config.websocket.port }}
#
extraResources: []

87
infra/kustomization.yaml Normal file
View File

@ -0,0 +1,87 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: infra # <-- Add this here instead insidde of benthos-deployment.yaml
# 1. List your static YAML files here
resources:
#- nats-cluster.yaml
- benthos/benthos-deployment.yaml
#- benthos-service.yaml
helmCharts:
- name: nats
repo: https://nats-io.github.io/k8s/helm/charts/
releaseName: nats-cluster
#namespace: infra
#valuesFile: nats-values.yaml # It will look for this file in the same folder
valuesInline:
namespaceOverride: "infra" # if not there got created in default namespaces
replicaCount: 3
config:
cluster:
enabled: true
jetstream:
enabled: true
fileStore:
pvc:
enabled: true
storageClassName: "longhorn" # Use Longhorn for replication!
size: 5Gi
# Add the nodeSelector directly here instead of using a patch!
nodeSelector:
node-role.kubernetes.io/nworker: "true"
# 2. List your config sources here
# This tells K8s: "Take every file in these folders and make ConfigMaps"
configMapGenerator:
- name: benthos-streams
files:
- benthos/streams/ingest_request_log_id_serial.yaml
- benthos/streams/ingest_transaction_log_id_serial.yaml
#- streams/poll_request_log_id_serial.yaml
#- streams/poll_transaction_id_serial.yaml
- name: benthos-templates
files:
- benthos/templates/ingest_template_id_serial.yaml
- benthos/templates/poll_template_id_serial.yaml
- name: benthos-resources
files:
- benthos/resources/resources.yaml
patches:
- target:
kind: Deployment
name: benthos # Ensure this matches the name inside benthos-deployment.yaml
patch: |-
- op: add
path: /spec/template/spec/nodeSelector
value:
node-role.kubernetes.io/nworker: "true"
# secretGenerator:
# - name: postgres-ha-app
# literals:
# - password=0lkzPxlwj6JVOXwwoLYROZJsONJoPK3MtrqkxnH3iaXUs0gFg0WL78RxyDdB86Sk
# - target:
# kind: StatefulSet
# name: nats-cluster # Ensure this matches the 'metadata.name' in nats-cluster.yaml
# patch: |-
# - op: add
# path: /spec/template/spec/nodeSelector
# value:
# node-role.kubernetes.io/nworker: "true"
# helmCharts:
# - name: nats
# repo: https://nats-io.github.io/k8s/helm/charts/
# releaseName: nats-cluster
# namespace: infra
# valuesInline:
# nats:
# jetstream:
# enabled: true
# cluster:
# enabled: true
# replicas: 3

BIN
infra/nats-cluster.yaml Normal file
View File

Binary file not shown.

22
infra/nats-values.yaml Normal file
View File

@ -0,0 +1,22 @@
# To increase replicas for High Availability
replicaCount: 3
# JetStream Clustering (The "Replica Factor")
config:
cluster:
enabled: true
jetstream:
enabled: true
# This ensures your data survives if x-vm2 goes down
fileStore:
pvc:
enabled: true
#storageClassName: "local-path" # k3s default storage
storageClassName: "longhorn" # k3s default storage
size: 4Gi
# valuesInline vs. valuesFile
# valuesInline: Use this for small configurations (like 5-10 lines). It keeps everything in one file so it's easy to read.
# valuesFile: Use this if your configuration is huge (100+ lines). It keeps your kustomization.yaml from becoming a giant, messy wall of text.

Some files were not shown because too many files have changed in this diff Show More