diff --git a/ansible-k3s/hosts.ini b/ansible-k3s/hosts.ini index d9a6f06..2b5b198 100644 --- a/ansible-k3s/hosts.ini +++ b/ansible-k3s/hosts.ini @@ -12,26 +12,64 @@ ; 192.168.3.93 ; 192.168.3.92 ; 192.168.3.94 +;[master] +;192.168.3.91 ansible_user=master-1 +;192.168.3.120 ansible-user=master +;192.168.3.97 ansible-user=master-7 [master] -192.168.3.91 ansible_user=master-1 +invixel-ubuntu4 ansible_host=192.168.3.156 ansible_user=master +invixel-vm1 ansible_host=192.168.3.91 ansible_user=master-1 +invixel-vm7 ansible_host=192.168.3.97 ansible_user=master-7 +;ssh-keygen -t ed25519 -C "josh@admin" -> generiere ssh schlüssel paar einfach zweimal leer Enter drücken +;sudo visudo : in the server you want to add +;add master ALL=(ALL) NOPASSWD:ALL +; oder echo "master ALL=(ALL) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/master -> invixel-ubuntu2 +; for debian: echo "master ALL=(ALL) NOPASSWD:ALL" | tee /etc/sudoers.d/master +; --> alternativ ohne Änderung auf Visudo mit -K ask nach Root Password +; ansible-playbook -i hosts.ini k3s-remote-control.yaml -K +; for debian: chmod 0440 /etc/sudoers.d/master +;ssh-copy-id -i ~/.ssh/id_ed25519 master@192.168.3.156 -> kopiere auf die Server # different user name -[dbworkers] -192.168.3.92 ansible_user=master-2 -192.168.3.93 ansible_user=master-3 -192.168.3.94 ansible_user=master-4 +[pure_dbworkers] +invixel-ubuntu2 ansible_host=192.168.3.155 ansible_user=master +;192.168.3.122 ansible_user=master +invixel-vm5 ansible_host=192.168.3.95 ansible_user=master-5 +invixel-vm6 ansible_host=192.168.3.96 ansible_user=master-6 +invixel-ubuntu3 ansible_host=192.168.3.153 ansible_user=master +invixel-ubuntu5 ansible_host=192.168.3.154 ansible_user=master [workers] -192.168.3.95 ansible_user=master-5 -192.168.3.96 ansible_user=master-6 +invixel-debian1 ansible_host=192.168.3.151 ansible_user=master +invixel-ubuntu5 ansible_host=192.168.3.154 ansible_user=master +invixel-ubuntu3 ansible_host=192.168.3.153 ansible_user=master + +# Hier bündeln wir die echten DB-Worker inklusive dem Master vm7 +[dbworkers:children] +pure_dbworkers + +[dbworkers:hosts] +invixel-vm1 # Ansible weiß jetzt, dass vm1 hier dazugehört und nutzt die IP/User von oben! +invixel-vm7 # This group combines both for easy targeting [k8s_nodes:children] master workers - +[k3s_agents:children] +pure_dbworkers +workers # [k8s_nodes:vars] # # Ensure Ansible uses the correct Python on the remote nodes # ansible_python_interpreter=/usr/bin/python3 # # If you use the same SSH key for all: -# ansible_ssh_private_key_file=~/.ssh/id_rsa \ No newline at end of file +# ansible_ssh_private_key_file=~/.ssh/id_rsa + +# Das Endergebnis im Cluster +# Sobald dein Playbook durchgelaufen ist und du auf deinem Laptop kubectl get nodes eingibst, sieht deine Topologie so aus: + +# 3 Master (HA-Control-Plane mit etcd): invixel-vm1, invixel-vm7 und invixel-ubuntu4. Alle drei können auch Pods ausführen. + +# 3 dedizierte DB-Worker: .92, .93, .94 (plus invixel-vm7 als Master-DB-Hybrid). + +# 5 dedizierte normale Worker: .95, .96, .153, .154, .155 (plus invixel-ubuntu4 als Master-Worker-Hybrid). \ No newline at end of file diff --git a/ansible-k3s/k3s-remote-control.yaml b/ansible-k3s/k3s-remote-control.yaml index 327d04b..0eb9d1f 100644 --- a/ansible-k3s/k3s-remote-control.yaml +++ b/ansible-k3s/k3s-remote-control.yaml @@ -9,22 +9,52 @@ # ansible.builtin.lineinfile: # path: "~/.bashrc" # line: 'export ANSIBLE_CONFIG="/mnt/f/Invixel/k8s-infra-cluster/ansible.cfg"' -# state: present +# +# Run this on master node 192.168.3.91 +# sudo mkdir -p /etc/rancher/k3s +# sudo nano /etc/rancher/k3s/config.yaml +# state: present +# bind-address: "192.168.3.91" +# advertise-address: "192.168.3.91" +# tls-san: +# - "192.168.3.91" +# cluster-init: true + +# Reset on invixel-vm1 192.168.3.91 +#sudo systemctl stop k3s +#sudo k3s server --cluster-reset +#sudo systemctl start k3s + + + +# sudo systemctl start k3s - name: Setup Laptop to Control K3s Cluster - hosts: master + hosts: master[0] become: yes vars: # Replace this with your Master's actual LAN or Tailscale IP - master_public_ip: "192.168.3.91" + master_public_ip: "192.168.3.156" local_kube_path: "~/.kube/config" tasks: - - name: Ensure K3s is running with TLS SAN for the Public IP + # - name: Ensure K3s is running with etcd HA mode (--cluster-init) + # # Ensure K3s is running with TLS SAN for the Public IP + # # curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--tls-san {{ master_public_ip }}" sh - + # ansible.builtin.shell: | + # # Auf dem ERSTEN, existierenden Master ausführen: + # curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--tls-san {{ master_public_ip }} --cluster-init" sh - + # register: k3s_install + + + # Run this only one time + - name: Ensure K3s is running with etcd HA mode (--cluster-init) ansible.builtin.shell: | - curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--tls-san {{ master_public_ip }}" sh - - register: k3s_install + curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--bind-address={{ master_public_ip }} --advertise-address={{ master_public_ip }} --tls-san {{ master_public_ip }} --cluster-init" sh - + # Verhindert mehrfaches Ausführen, wenn K3s bereits existiert: + args: + creates: /usr/local/bin/k3s - name: Wait for k3s.yaml to be fully generated ansible.builtin.wait_for: path: /etc/rancher/k3s/k3s.yaml @@ -45,6 +75,20 @@ # dest: "{{ local_kube_path }}" # mode: '0600' + # For Phase 2 + - name: Fetch the cluster join token from the first master + ansible.builtin.slurp: + src: /var/lib/rancher/k3s/server/node-token + register: master_token_encoded + + - name: Share the token with the rest of the playbook run + ansible.builtin.set_fact: + cluster_token: "{{ master_token_encoded.content | b64decode | trim }}" + delegate_to: localhost + delegate_facts: true + # For Phase 2 + + - name: Read k3s.yaml from master ansible.builtin.slurp: src: /etc/rancher/k3s/k3s.yaml @@ -58,4 +102,76 @@ dest: "{{ local_kube_path }}" mode: '0600' +# ============================================================================== +# PHASE 2: WEITERE CONTROL PLANES HINZUFÜGEN (JOIN) +# ============================================================================== +- name: Join Additional Control Planes to the Cluster + hosts: master[1:] # Führt dies auf ALLEN ANDEREN Servern in der Master-Gruppe aus (Server 2, 3, etc.) + become: yes + vars: + first_master_ip: "192.168.3.156" # Die IP des ersten Masters, an den sich alle anmelde + tasks: + - name: Join as additional HA Control Plane + ansible.builtin.shell: | + # Auf dem NEUEN, zweiten Server ausführen: + curl -sfL https://get.k3s.io | K3S_TOKEN="{{ hostvars['localhost']['cluster_token'] }}" INSTALL_K3S_EXEC="--server https://{{ first_master_ip }}:6443 --tls-san {{ ansible_host }}" sh - + # Hinweis: {{ ansible_host }} nimmt automatisch die IP des jeweiligen Servers aus deiner hosts.ini + + +# ============================================================================== +# PHASE 3: ALLE WORKER NODES (DB- & NORMALE WORKER) SAUBER HINZUFÜGEN +# ============================================================================== +- name: Join All Worker Nodes to the Cluster + hosts: k3s_agents # Benutze die Sammelgruppe aus deiner hosts.ini + become: yes + vars: + first_master_ip: "192.168.3.156" + tasks: + - name: Check if an old K3s installation exists + ansible.builtin.stat: + path: /usr/local/bin/k3s-agent-uninstall.sh + register: old_k3s_agent + + - name: Clean old cluster configuration if present + ansible.builtin.shell: /usr/local/bin/k3s-agent-uninstall.sh + when: old_k3s_agent.stat.exists + + - name: Join as K3s Agent (Fresh or Re-Join) + ansible.builtin.shell: | + curl -sfL https://get.k3s.io | K3S_URL="https://{{ first_master_ip }}:6443" K3S_TOKEN="{{ hostvars['localhost']['cluster_token'] }}" sh - + +# ============================================================================== +# PHASE 4: KUBERNETES ROLES AUTOMATISCH ZUWEISEN +# ============================================================================== +- name: Assign Kubernetes Roles to Nodes + hosts: master[0] + become: yes + tasks: + - name: Label DB Workers + ansible.builtin.command: + # cmd: "kubectl label node {{ item }} node-role.kubernetes.io/dbworker= --overwrite" + # sudo kubectl label node invixel-ubuntu2 node-role.kubernetes.io/nworker= + # sudo kubectl label node invixel-ubuntu3 node-role.kubernetes.io/nworker= + # sudo kubectl label node invixel-ubuntu-5 node-role.kubernetes.io/nworker= + cmd: > + kubectl label node + {{ hostvars[item]['ansible_hostname'] | default(hostvars[item]['inventory_hostname']) }} + node-role.kubernetes.io/dbworker= --overwrite + loop: "{{ groups['pure_dbworkers'] }}" + ignore_errors: yes + + - name: Label Normal Workers + ansible.builtin.command: + # cmd: "kubectl label node {{ item }} node-role.kubernetes.io/nworker= --overwrite" + cmd: > + kubectl label node + {{ hostvars[item]['ansible_hostname'] | default(hostvars[item]['inventory_hostname']) }} + node-role.kubernetes.io/nworker= --overwrite + # Verwendet die Namen aus deiner [workers] Gruppe in der hosts.ini + loop: "{{ groups['workers'] }}" + ignore_errors: yes + + +# export ANSIBLE_CONFIG=./ansible.cfg + # ansible-playbook -i hosts.ini k3s-remote-control.yml \ No newline at end of file diff --git a/manuals.md b/manuals.md index 533cda0..774c034 100644 --- a/manuals.md +++ b/manuals.md @@ -1,5 +1,18 @@ ### Getting Started +#### Join new nodes +``` +# retrieve token on control-plane node +sudo cat /var/lib/rancher/k3s/server/node-token +K10094c77f9748b705ce6fd128b916dd96793160853577fdcba986c57de9b6f4a3c::server:b5743ed19b035f78f59607b2fa3bda2e +# join via +curl -sfL https://get.k3s.io | K3S_URL=https://:6443 K3S_TOKEN= sh - + +K3S_URL: The address of your control plane node + +curl -sfL https://get.k3s.io | K3S_URL=https://192.168.3.91:6443 K3S_TOKEN=K10094c77f9748b705ce6fd128b916dd96793160853577fdcba986c57de9b6f4a3c::server:b5743ed19b035f78f59607b2fa3bda2e sh - + +``` #### 1. Using ansible ``` a. ansible-playbook -i hosts.ini ansible-prepare-k3s.yml @@ -127,6 +140,18 @@ kubectl get secret postgres-ha-app -n db -o json | jq 'del(.metadata.namespace,. kubectl get secret postgres-ha-superuser -n db -o json | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid)' | kubectl apply -n infra -f - ``` +#### Avoiding sudo - Unable to read Error while executing kubectl command option +``` +# Option 1: Change permissions (allows your user to read) +sudo chmod 644 /etc/rancher/k3s/k3s.yaml + +# Option 2: Copy the config to your user's directory +mkdir -p ~/.kube +sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config +sudo chown $(id -u):$(id -g) ~/.kube/config +chmod 600 ~/.kube/config +``` + #### Debug ``` kubectl get all -A @@ -177,12 +202,34 @@ sudo chmod 644 /etc/rancher/k3s/k3s.yaml Run in PS kustomize build . --enable-helm | kubectl apply --server-side --force-conflicts -f - -4. pods exist in PS but not in wsl +4. pods exist in PS but not in wsl -- Join/Query from Laptop to k3s servers +# On WSL mkdir -p ~/.kube sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config sudo chown $(id -u):$(id -g) ~/.kube/config export KUBECONFIG=~/.kube/config + +change IP manually from 127.0.0.1 to 192.168.3.91 + +or + +mkdir -p ~/.kube +scp master-1@192.168.3.91:/etc/rancher/k3s/k3s.yaml ~/.kube/config +sed -i 's/127.0.0.1/192.168.3.91/g' ~/.kube/config +chmod 600 ~/.kube/config + +# Testen +kubectl get nodes + +# on PS +ssh master@192.168.3.156 "sudo cat /etc/rancher/k3s/k3s.yaml" > "$HOME\.kube\config" + +# Localhost-IP durch die echte Master-IP ersetzen +(Get-Content "$HOME\.kube\config") -replace '127.0.0.1', '192.168.3.156' | Set-Content "$HOME\.kube\config" + +# Testen +kubectl get nodes ``` diff --git a/manuals/deploy_k8s_components.md b/manuals/deploy_k8s_components.md index a56d916..c8eb7a2 100644 --- a/manuals/deploy_k8s_components.md +++ b/manuals/deploy_k8s_components.md @@ -729,4 +729,31 @@ Delete: httproute.yaml: This is for the "Gateway API." Unless you have a specific Gateway controller installed, standard ingress.yaml is what you'll use. NOTES.txt: Usually just contains generic text. -```` \ No newline at end of file +``` + +#### What ToDo after deleting the primary node that bootstrapped the cluster. +``` +ssh to other k3s control plane + +sudo systemctl status k3s +sudo systemctl stop k3s +sudo k3s server --cluster-reset +sudo systemctl start k3s +sudo kubectl get nodes + +sudo nano /etc/systemd/system/k3s.service + +from +ExecStart=/usr/local/bin/k3s server --server https://192.168.3.91:6443 --tls-san 192.168.3.156 + +to +ExecStart=/usr/local/bin/k3s server --cluster-init --tls-san 192.168.3.156 +``` + +#### Delete stale nodes +``` +sudo kubectl delete node invixel-vm1 +sudo kubectl delete node invixel-vm6 +sudo kubectl delete node invixel-vm7 +sudo kubectl delete node invixel-ubuntu1 +``` \ No newline at end of file diff --git a/manuals/important_commands.md b/manuals/important_commands.md index eed0043..d641124 100644 --- a/manuals/important_commands.md +++ b/manuals/important_commands.md @@ -30,7 +30,7 @@ kubectl exec -it nats-box -n infra -- nats -s nats://nats-cluster.infra.svc.clus kubectl exec -it nats-box -n infra -- nats -s nats://nats-cluster:4222 stream info request_stream kubectl exec -it nats-cluster-0 -n infra -c nats -- netstat -tuln kubectl get pod nats-cluster-0 -n infra -o wide - +kubectl get svc --all-namespaces kubectl get endpoints nats-cluster -n infra @@ -132,6 +132,10 @@ kubectl run net-test --image=alpine --restart=Never -n infra -- nslookup google. kubectl exec -it yb-tserver-0 -n db -- ysqlsh -h kubectl port-forward svc/cosmo-router 3002:3002 -n apps kubectl port-forward svc/postgres-ha-rw 5436:5432 -n db + + +# (Best Practice für Zero-Downtime): Evakuiere die alten Nodes zuerst manuell (Drain): Für die alten dbworker und nworker ausführen: +kubectl drain --ignore-daemonsets --delete-emptydir-data ``` #### The Difference: NATS Core vs. JetStream - NATS Core (nats sub): This is "Fire and Forget." If no one is listening when Benthos sends the message, the message disappears forever. diff --git a/manuals/interconnection.md b/manuals/interconnection.md index 36a6012..fba1358 100644 --- a/manuals/interconnection.md +++ b/manuals/interconnection.md @@ -91,4 +91,833 @@ Reason to use: This is the most "production-ready" version of what you are doing |kubectl port-forward|Layer 7|"A temporary ""straw"" to sip data from one |specific cup.| |Ingress|Layer 7|A ""Receptionist"" directing visitors based on the name on the |envelope.| |LoadBalancer|Layer 4|"A ""Direct Pipeline"" to a specific door (port).| -|Tailscale / VPN|Layer 3|"A ""Secret Tunnel"" that puts your whole house next to the office.| \ No newline at end of file +|Tailscale / VPN|Layer 3|"A ""Secret Tunnel"" that puts your whole house next to the office.| + +#### Finding IP +public: curl ifconfig.me + +|Target|Command|Requirement| +|----|----|-----| +|Inside Container| hostname -I /ip addr show eth0|LXC Container| +|LXC Local IP|pct exec ID ip a|Access to Proxmox Host| +|Proxmox Local IP|hostname -I|Access to Proxmox Host| +|Proxmox Public IP|curl ifconfig.me|Internet Access| + + +#### Ipaymu Integration need static IP + +[Cloudflare Worker] ──1. Request──► [Your Nginx Server] ──2. Request──► [iPaymu API] + │ │ +[Cloudflare Worker] ◄─4. Response── [Your Nginx Server] ◄─3. Response─── [iPaymu API] +Problem: we deploy using cloudflared worker which has dynamic ips + +- Find IP of the server +- curl -4 ifconfig.me +- curl -4 icanhazip.com +- curl -4 ipinfo.io/ip +- ip -4 addr show + +1. Option 1: Cloudflare Tunnel (cloudflared) — Easiest & Most Secure +A. cloudflared inside an LXC Container (Separate from K3s) +An LXC container on the same LAN/Proxmox host can run cloudflared and route incoming traffic directly to your K3s cluster. + +[Cloudflare Worker] + │ (Public HTTPS) + ▼ +[Cloudflare Edge Network] + │ (Outbound Tunnel) +[LXC Container running cloudflared] + │ (Local LAN / Bridge Traffic) +[K3s Cluster Node (Static IP)] ──► [Nginx Proxy Pod] ──► [iPaymu API] + +How to configure the LXC Tunnel: +When you set up the Public Hostname rule in the Cloudflare Zero Trust Dashboard (or config.yaml), point the origin URL directly to your K3s Node IP and port: + + - Service Type: HTTP + + - URL: : +(e.g., [http://192.168.1.50:80](http://192.168.1.50:80) if calling Traefik/Ingress, or [http://192.168.1.50:30080](http://192.168.1.50:30080) if using a NodePort service). + +Pros & Cons: + - Pros: Keeps your K3s cluster lightweight; if K3s reboots, the tunnel daemon stays alive in the LXC container. + - Cons: Requires exposing a NodePort or LoadBalancer (Traefik) on your K3s node so the LXC container can hit it on the local network. + +B. cloudflared as a Pod inside K3s +Deploying cloudflared directly inside K3s as a Deployment is the most common Kubernetes-native method. +[Cloudflare Worker] + │ + ▼ (HTTPS over public internet) +[https://proxy.yourdomain.com] <-- Cloudflare Edge + │ + ▼ (Secure outbound Tunnel) +[cloudflared Pod in K3s / Swarm] + │ + ▼ (Internal Cluster Traffic) +[Nginx Proxy Pod (Static-IP Node)] + │ + ▼ (Outbound call from Static IPv4) +[iPaymu API] + +2. Option 2: Traefik Ingress Controller + NodePort / Public IP +If your static-IP cluster node already has a Public Static IPv4 Address directly bound to its network interface: + +[Cloudflare Worker] + │ + ▼ (HTTPS to Public Static IP) +[Public IP:443] ──► [Traefik Ingress] ──► [Nginx Proxy Service] ──► [iPaymu] + +How to configure the K3s Tunnel: +Inside the Cloudflare dashboard, you point the public hostname to K3s's internal DNS service name: + +- Service Type: HTTP +- URL: ipaymu-proxy-service.default.svc.cluster.local:80 + +Pros & Cons: + - Pros: Highly secure—you do not need to open any ports or expose NodePort services on your LAN. Traffic moves entirely within K3s cluster networking. + - Cons: Runs inside the Kubernetes runtime lifecycle. + +3. Option 3: Port Mapping directly on the Static Node (hostPort) +If you don't want to deal with Traefik or Ingress controllers, you can expose your Nginx proxy directly on a port (e.g., 8080 or 8443) of your static-IP node using hostPort in K3s or ports mapping in Swarm. + +K3s Manifest Snippet: +containers: +- name: nginx + image: nginx:alpine + ports: + - containerPort: 80 + hostPort: 8080 # Exposes port 8080 directly on the physical host + + +Vour Cloudflare Worker calls http://:8080/ipaymu/api/v2/payment. + +- How the Request Flow Works End-to-End +Once exposed via Option 1, 2, or 3, here is the full round-trip execution path: +``` +// 1. Cloudflare Worker executes this code: +const response = await fetch("https://proxy.yourdomain.com/ipaymu/api/v2/payment", { + method: "POST", + headers: { + "X-Proxy-Secret": "YOUR_INTERNAL_SECRET", + "va": "1176000000000000", + "signature": "CALCULATED_HMAC_SIGNATURE", + "Content-Type": "application/json" + }, + body: JSON.stringify(paymentPayload) +}); + +// 2. Request reaches https://proxy.yourdomain.com (Your Cluster via Tunnel/Ingress) +// 3. Ingress routes it internally to your Nginx proxy pod +// 4. Nginx verifies "X-Proxy-Secret", then forwards the request to https://api.ipaymu.com +// 5. iPaymu receives the call, sees your Node's STATIC IP, and accepts it +// 6. iPaymu returns payment data -> Nginx -> Ingress -> Cloudflare Worker -> Client + +``` + + +#### Step by step using ONLY Traefik in K3s +``` +Step 1: Create Traefik Middlewares for Security and Stripping +Instead of Nginx if ($http_x_proxy_secret) directives, you define Traefik Middleware resources: + +# 1. Security Header Check Middleware +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: ipaymu-auth-check + namespace: default +spec: + headers: + customRequestHeaders: + # Ensures requests forwarded to iPaymu clean up internal proxy secrets + X-Proxy-Secret: "" +--- +# 2. Path Strip Middleware (Removes /ipaymu prefix before forwarding) +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: ipaymu-strip-prefix + namespace: default +spec: + stripPrefix: + prefixes: + - /ipaymu + +Step 2: Define an ExternalName Service for iPaymu +To tell Traefik to forward traffic out to iPaymu's external domain, create a Kubernetes ExternalName service: + +apiVersion: v1 +kind: Service +metadata: + name: ipaymu-external-api + namespace: default +spec: + type: ExternalName + externalName: api.ipaymu.com + ports: + - port: 443 + targetPort: 443 + protocol: TCP + +Step 3: Route Traffic with Traefik IngressRoute +Now, create a Traefik IngressRoute pinned to your static IP node: + +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: ipaymu-proxy-route + namespace: default +spec: + entryPoints: + - web + - websecure + routes: + - match: Host(`proxy.yourdomain.com`) && PathPrefix(`/ipaymu`) + kind: Rule + middlewares: + - name: ipaymu-strip-prefix + services: + - name: ipaymu-external-api + port: 443 + scheme: https + +When should you STILL use Nginx? +While Traefik replaces Nginx for 90% of use cases, keeping a tiny Nginx pod is still useful if: + +Complex Logic: You want custom Nginx Lua scripts or complex header manipulation (e.g., computing dynamic HMAC signatures or custom retry logic on upstream failures). + +Familiarity: You are already comfortable writing .conf files and don't want to learn Traefik-specific Kubernetes CRD syntax (IngressRoute / Middleware). +``` + +#### Step by step using traeffik and nginx in k3s +``` +1. Label your target node (Crucial Step) +Label the K3s node that holds your static public IPv4 address so K3s knows where to schedule the proxy pods: + +kubectl label nodes egress-ip=static-ipaymu + +2. k3s-ipaymu-proxy.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: ipaymu-proxy-config + namespace: default +data: + default.conf: | + upstream ipaymu_backend { + server api.ipaymu.com:443; + keepalive 32; + } + + server { + listen 80; + + location /ipaymu/ { + if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET_KEY") { + return 403; + } + + proxy_pass https://ipaymu_backend/; + proxy_set_header Host api.ipaymu.com; + + # SSL Handshake settings + proxy_ssl_server_name on; + proxy_ssl_protocols TLSv1.2 TLSv1.3; + + # HTTP Keepalive + proxy_http_version 1.1; + proxy_set_header Connection ""; + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ipaymu-proxy + namespace: default +spec: + replicas: 2 # Scale up replicas as needed + selector: + matchLabels: + app: ipaymu-proxy + template: + metadata: + labels: + app: ipaymu-proxy + spec: + # Pin all replicas to the node with the static public IP + nodeSelector: + egress-ip: static-ipaymu + containers: + - name: nginx + image: nginx:alpine + ports: + - containerPort: 80 + resources: + limits: + cpu: "200m" + memory: "128Mi" + requests: + cpu: "50m" + memory: "32Mi" + volumeMounts: + - name: nginx-config + mountPath: /etc/nginx/conf.d/default.conf + subPath: default.conf + volumes: + - name: nginx-config + configMap: + name: ipaymu-proxy-config +--- +apiVersion: v1 +kind: Service +metadata: + name: ipaymu-proxy-service + namespace: default +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: 80 + protocol: TCP + selector: + app: ipaymu-proxy + +3. Deploy : kubectl apply -f k3s-ipaymu-proxy.yaml +4. How to call it inside K3s +Any other pod inside your K3s cluster can now make payment requests directly using internal Kubernetes DNS: +[http://ipaymu-proxy-service.default.svc.cluster.local/ipaymu/api/v2/payment](http://ipaymu-proxy-service.default.svc.cluster.local/ipaymu/api/v2/payment) +``` + +#### Step by step using traeffik and cloudflared in k3s +``` +Step 2.1: Enable allowExternalNameServices in Traefik +By default, K3s disables Traefik from routing to external domains via ExternalName. Create a HelmChartConfig manifest on your K3s server node to enable it: + +Create file /var/lib/rancher/k3s/server/manifests/traefik-config.yaml: + +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: traefik + namespace: kube-system +spec: + valuesContent: |- + providers: + kubernetesCRD: + allowExternalNameServices: true + +K3s automatically applies this change in under 30 seconds. + +Step 2.2: Label your Static-IP Node +Label the node with your static public IPv4 address so K3s knows where to schedule the tunnel: + +kubectl label nodes egress-ip=static-ipaymu + +Step 2.3: Deploy the iPaymu Traefik Route +Save this to ipaymu-traefik.yaml and run kubectl apply -f ipaymu-traefik.yaml: + +# 1. External Name Service pointing to iPaymu +apiVersion: v1 +kind: Service +metadata: + name: ipaymu-external-service + namespace: default +spec: + type: ExternalName + externalName: api.ipaymu.com + ports: + - name: https + port: 443 + targetPort: 443 + protocol: TCP +--- +# 2. TLS Transport for SNI support +apiVersion: traefik.io/v1alpha1 +kind: ServersTransport +metadata: + name: ipaymu-transport + namespace: default +spec: + serverName: api.ipaymu.com +--- +# 3. Path Stripper Middleware (/ipaymu/api/v2/payment -> /api/v2/payment) +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: ipaymu-strip-prefix + namespace: default +spec: + stripPrefix: + prefixes: + - /ipaymu +--- +# 4. Traefik IngressRoute +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: ipaymu-proxy-route + namespace: default +spec: + entryPoints: + - web + - websecure + routes: + - match: Host(`proxy.yourdomain.com`) && PathPrefix(`/ipaymu`) + kind: Rule + middlewares: + - name: ipaymu-strip-prefix + services: + - name: ipaymu-external-service + port: 443 + scheme: https + serversTransport: ipaymu-transport + +Step 2.4: +- Get Your Tunnel Secret Token + 1. Log into your Cloudflare Zero Trust Dashboard. + 2. Navigate to Networks → Tunnels → Create a Tunnel. + 3. Choose Cloudflared as the connector and name it (e.g., k3s-ipaymu-tunnel). + 4. On the installation page, select Docker/Kubernetes. Look for the command and copy the long base64 token string after --token. + +- Deploy Cloudflare Tunnel inside K3s +Save this to cloudflared.yaml (replace YOUR_TOKEN_HERE with your Cloudflare Zero Trust tunnel token) and run kubectl apply -f cloudflared.yaml: + +apiVersion: v1 +kind: Secret +metadata: + name: cloudflared-token + namespace: default +type: Opaque +stringData: + TUNNEL_TOKEN: "YOUR_TOKEN_HERE" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cloudflared + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: cloudflared + template: + metadata: + labels: + app: cloudflared + spec: + nodeSelector: + egress-ip: static-ipaymu + containers: + - name: cloudflared + image: cloudflare/cloudflared:latest + args: + - tunnel + - --no-autoupdate + - run + env: + - name: TUNNEL_TOKEN + valueFrom: + secretKeyRef: + name: cloudflared-token + key: TUNNEL_TOKEN + + +kubectl apply -f cloudflared.yaml + +Step 2.5: Cloudflare Dashboard Routing +In Cloudflare Zero Trust Dashboard (→ Networks → Tunnels → Public Hostname): + +Public Hostname: proxy.yourdomain.com + +Path: /ipaymu + +Type: HTTP + +URL: traefik.kube-system.svc.cluster.local:80 +``` +#### Step by step using nginx and cloudflared in docker-compose +``` +Step 3.1: Create Directory Structure +On your target server, create a project directory: +mkdir -p ipaymu-proxy +cd ipaymu-proxy + +ipaymu-proxy/ +├── docker-compose.yml +└── nginx.conf + +Step 3.2: Create nginx.conf +events { worker_connections 1024; } + +http { + upstream ipaymu_api { + server api.ipaymu.com:443; + keepalive 16; + } + + server { + listen 80; + server_name _; + + location /ipaymu/ { + # Check secret key header sent by your Cloudflare Worker + if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET_KEY") { + return 403; + } + + proxy_pass https://ipaymu_api/; + proxy_set_header Host api.ipaymu.com; + + # Ensure TLS handshake works with iPaymu + proxy_ssl_server_name on; + proxy_ssl_protocols TLSv1.2 TLSv1.3; + + # HTTP Keepalive optimizations + proxy_http_version 1.1; + proxy_set_header Connection ""; + } + } +} + + + +Step 3.3: Create docker-compose.yml +version: '3.8' + +services: + ipaymu-proxy: + image: nginx:alpine + container_name: ipaymu_proxy + restart: always + ports: + - "80:80" # Or expose via reverse proxy / Cloudflare tunnel + volumes: + - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro + healthcheck: + test: ["CMD", "nginx", "-t"] + interval: 30s + timeout: 10s + retries: 3 + +or with nginx: + +#docker-compose.yml +version: '3.8' +services: + ipaymu-nginx: + image: nginx:alpine + container_name: ipaymu_nginx + restart: always + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + networks: + - proxy-net + + cloudflared: + image: cloudflare/cloudflared:latest + container_name: ipaymu_tunnel + restart: always + command: tunnel --no-autoupdate run + environment: + - TUNNEL_TOKEN=YOUR_CLOUDFLARE_TUNNEL_TOKEN_HERE + networks: + - proxy-net + +networks: + proxy-net: + driver: bridge + +Step 3.4: Deploy Docker Compose: docker compose up -d +Step 3.5: Cloudflare Dashboard RoutingIn Cloudflare Zero Trust Dashboard ($\rightarrow$ Networks $\rightarrow$ Tunnels $\rightarrow$ Public Hostname): +Public Hostname: proxy.yourdomain.com +Path: /ipaymuType: HTTP +URL: ipaymu_nginx:80 (Connects directly using Docker's internal container DNS). + +``` + +#### Step by step using docker swarm + +``` +version: '3.8' + +services: + ipaymu-proxy: + image: nginx:alpine + ports: + - "80:80" + - "443:443" + configs: + - source: nginx_config + target: /etc/nginx/conf.d/default.conf + deploy: + mode: replicated + replicas: 1 + placement: + constraints: + # Pin execution to the specific node with the static public IP + - node.hostname == node-with-static-ip + +configs: + nginx_config: + inline: | + server { + listen 80; + server_name ipaymu-proxy.yourdomain.com; + + location /ipaymu/ { + if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET") { + return 403; + } + + proxy_pass https://api.ipaymu.com/; + proxy_set_header Host api.ipaymu.com; + proxy_ssl_server_name on; + + proxy_ssl_protocols TLSv1.2 TLSv1.3; + + # Pass client details + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } + +[Cloudflare Worker] + │ + ├─── /ipaymu/v2/payment ─────────► [Nginx] ──► https://my.ipaymu.com/api/v2/payment + │ + └─── /ipaymu-sandbox/v2/payment ─► [Nginx] ──► https://sandbox.ipaymu.com/api/v2/payment + +1.a Create the dedicated folder in /opt/ +mkdir -p /opt/ipaymu-swarm-proxy +cd /opt/ipaymu-swarm-proxy + + +b create the nginx configuration file nginx.conf: +``` +events { + worker_connections 1024; +} + +http { + # Upstream targets using persistent HTTP keepalives + upstream ipaymu_live { + server my.ipaymu.com:443; + keepalive 16; + } + + upstream ipaymu_sandbox { + server sandbox.ipaymu.com:443; + keepalive 16; + } + + server { + listen 80; + server_name _; + + # ------------------------------------------------------------- + # 1. Production / Live Endpoint + # ------------------------------------------------------------- + location /ipaymu/v2/ { + # Security Header Check + if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET_KEY") { + return 403; + } + + proxy_pass https://ipaymu_live/api/v2/; + proxy_set_header Host my.ipaymu.com; + + # Ensure TLS/SNI Handshake matches my.ipaymu.com + proxy_ssl_server_name on; + proxy_ssl_name my.ipaymu.com; + proxy_ssl_protocols TLSv1.2 TLSv1.3; + + # Connection optimizations + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_pass_request_headers on; + } + + # ------------------------------------------------------------- + # 2. Development / Sandbox Endpoint + # ------------------------------------------------------------- + location /ipaymu-sandbox/v2/ { + # Security Header Check + if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET_KEY") { + return 403; + } + + proxy_pass https://ipaymu_sandbox/api/v2/; + proxy_set_header Host sandbox.ipaymu.com; + + # Ensure TLS/SNI Handshake matches sandbox.ipaymu.com + proxy_ssl_server_name on; + proxy_ssl_name sandbox.ipaymu.com; + proxy_ssl_protocols TLSv1.2 TLSv1.3; + + # Connection optimizations + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_pass_request_headers on; + } + } +} +``` + +2. Change ipaymu +``` +# erstelle einen starker schlüssel YOUR_INTERNAL_SECRET_KEY: +openssl rand -hex 32 + +# Setze diesen Schlüssel in deinen Nginx-Code (nginx.conf): +if ($http_x_proxy_secret != "0fb1d9712f16bb64e0dff4f911af02e8a2280722c80097a845e248c7a49781d7") { + return 403; +} + +Setze denselben Schlüssel in deinen Cloudflare Worker / App-Code: + +const headers = { + "Content-Type": "application/json", + "X-Proxy-Secret": "0fb1d9712f16bb64e0dff4f911af02e8a2280722c80097a845e248c7a49781d7", // Gleicher Schlüssel! + // ... +}; + + +CONTAINER_ID=$(docker ps -q -f name=ipaymu_nginx-proxy | head -n1) +docker exec -it $CONTAINER_ID apk add --no-cache curl +docker exec -it $CONTAINER_ID curl -4 ifconfig.me + +curl -i -X POST https://proxy.yourdomain.com/ipaymu/v2/payment \ + -H "X-Proxy-Secret: YOUR_INTERNAL_SECRET_KEY" \ + -H "Content-Type: application/json" +``` +1. Das Label auf den gewählten Node setzen +Sobald du weißt, welcher Node die statische IP hat (z. B. node-01), vergibst du das Label egress=static-ipaymu: +``` +docker node update --label-add egress=static-ipaymu node-01 + +docker node update --label-add egress=static-ipaymu +docker node inspect node-01 --format '{{ json .Spec.Labels }}' +``` + +4. Deploy And check +docker run cloudflare/cloudflared:latest tunnel --no-autoupdate run --token eyJhIjoiNjFhMTZkYzg3MmM4ZjlmZTQ4Mzc0ZWQ5NzVmNTcwNmIiLCJ0IjoiNTA5ZmRlZjQtZjEzYy00ZTQ1LTllZDQtYjZhMjExOTFlY2Y3IiwicyI6Ik5UWmtZVFU1T1dZdE5XVmtZUzAwTldZeExXSm1Oek10TlRRd1pqUTBNR1kxWW1RMSJ9 + +- Option 1 Using Docker Swarm env ( does not work ) +``` +version: '3.8' + +services: + nginx-proxy: + image: nginx:alpine + configs: + - source: nginx_config_v1 + target: /etc/nginx/nginx.conf + networks: + - payment-network + deploy: + mode: replicated + replicas: 2 + placement: + constraints: + # Pin to the node holding your static IP for iPaymu whitelisting + - node.labels.egress == static-ipaymu + restart_policy: + condition: on-failure + + cloudflared: + image: cloudflare/cloudflared:latest + command: tunnel --no-autoupdate run + environment: + - TUNNEL_TOKEN=${CLOUDFLARE_TUNNEL_TOKEN} + networks: + - payment-network + deploy: + mode: replicated + replicas: 2 + placement: + constraints: + - node.labels.egress == static-ipaymu + restart_policy: + condition: on-failure + +configs: + nginx_config_v1: + file: ./nginx.conf + +networks: + payment-network: + driver: overlay + attachable: true +``` +create .env file in the same directory: +sudo nano /opt/ipaymu-swarm-proxy/.env +CLOUDFLARE_TUNNEL_TOKEN=eyJhIjoiNjFhMTZkYzg3MmM4ZjlmZTQ4Mzc0ZWQ5NzVmNTcwNmIiLCJ0IjoiNTA5ZmRlZjQtZjEzYy00ZTQ1LTllZDQtYjZhMjExOTFlY2Y3IiwicyI6Ik5UWmtZVFU1T1dZdE5XVmtZUzAwTldZeExXSm1Oek10TlRRd1pqUTBNR1kxWW1RMSJ9 + +docker run cloudflare/cloudflared:latest tunnel --no-autoupdate run --token eyJhIjoiNjFhMTZkYzg3MmM4ZjlmZTQ4Mzc0ZWQ5NzVmNTcwNmIiLCJ0IjoiNTA5ZmRlZjQtZjEzYy00ZTQ1LTllZDQtYjZhMjExOTFlY2Y3IiwicyI6Ik5UWmtZVFU1T1dZdE5XVmtZUzAwTldZeExXSm1Oek10TlRRd1pqUTBNR1kxWW1RMSJ9 + +docker stack deploy -c ipaymu-swarm.yml ipaymu +or +export $(cat /opt/ipaymu-swarm-proxy/.env | xargs) && sudo -E docker stack deploy -c ipaymu-swarm.yml ipaymu + +or + +set -a && source /opt/ipaymu-swarm-proxy/.env && set +a && sudo -E docker stack deploy -c ipaymu-swarm.yml ipaymu + +or + +CLOUDFLARE_TUNNEL_TOKEN="eyJhIjoiNjFhMTZkYzg3MmM4ZjlmZTQ4Mzc0ZWQ5NzVmNTcwNmIiLCJ0IjoiNTA5ZmRlZjQtZjEzYy00ZTQ1LTllZDQtYjZhMjExOTFlY2Y3IiwicyI6Ik5UWmtZVFU1T1dZdE5XVmtZUzAwTldZeExXSm1Oek10TlRRd1pqUTBNR1kxWW1RMSJ9" sudo -E docker stack deploy -c ipaymu-swarm.yml ipaymu + +# Option 2 Docker Swarm Secret +Create a Docker Secret on your Manager node: +printf "your_actual_token_here" | docker secret create cloudflared_tunnel_token - + +version: '3.8' + +services: + cloudflared: + image: cloudflare/cloudflared:latest + # Tells cloudflared to read the token file created by Docker Secrets + command: tunnel --no-autoupdate run --token-file /run/secrets/cloudflared_token + secrets: + - source: cloudflared_tunnel_token + target: cloudflared_token + networks: + - ipaymu-overlay + deploy: + mode: replicated + replicas: 2 + placement: + constraints: + - node.labels.egress == static-ipaymu + +secrets: + cloudflared_tunnel_token: + external: true + +Summary Recommendation +For simple setups: Use Method 1 (.env file). +For strict security requirements: Use Method 4 (Docker Secrets). + +docker stack services ipaymu +docker service logs ipaymu_cloudflared --tail 20 + +Now you need to label the node like in docker-stack.yml: +docker node update --label-add egress=static-ipaymu + +docker node update --label-add egress=static-ipaymu invixel-debian1 +docker node update --label-add egress=static-ipaymu invixel-vm5 + +How to verify the exit IP +After deploying to K3s or Swarm, exec into your Nginx container/pod and test what public IP iPaymu will see: + +docker exec -it curl -4 ifconfig.me + +kubectl exec -it deployment/ipaymu-proxy -- curl -4 ifconfig.me + + +curl -i -X POST https://bali-car-dev.easy-rent.com/ipaymu/v2/payment \ + -H "X-Proxy-Secret: 0fb1d9712f16bb64e0dff4f911af02e8a2280722c80097a845e248c7a49781d7" \ + -H "Content-Type: application/json" +``` \ No newline at end of file