initial setup - most importants
This commit is contained in:
wendi gunawan
parent
4119e44d83
commit
b458d5805f
@ -12,26 +12,64 @@
|
|||||||
; 192.168.3.93
|
; 192.168.3.93
|
||||||
; 192.168.3.92
|
; 192.168.3.92
|
||||||
; 192.168.3.94
|
; 192.168.3.94
|
||||||
|
;[master]
|
||||||
|
;192.168.3.91 ansible_user=master-1
|
||||||
|
;192.168.3.120 ansible-user=master
|
||||||
|
;192.168.3.97 ansible-user=master-7
|
||||||
[master]
|
[master]
|
||||||
192.168.3.91 ansible_user=master-1
|
invixel-ubuntu4 ansible_host=192.168.3.156 ansible_user=master
|
||||||
|
invixel-vm1 ansible_host=192.168.3.91 ansible_user=master-1
|
||||||
|
invixel-vm7 ansible_host=192.168.3.97 ansible_user=master-7
|
||||||
|
;ssh-keygen -t ed25519 -C "josh@admin" -> generiere ssh schlüssel paar einfach zweimal leer Enter drücken
|
||||||
|
;sudo visudo : in the server you want to add
|
||||||
|
;add master ALL=(ALL) NOPASSWD:ALL
|
||||||
|
; oder echo "master ALL=(ALL) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/master -> invixel-ubuntu2
|
||||||
|
; for debian: echo "master ALL=(ALL) NOPASSWD:ALL" | tee /etc/sudoers.d/master
|
||||||
|
; --> alternativ ohne Änderung auf Visudo mit -K ask nach Root Password
|
||||||
|
; ansible-playbook -i hosts.ini k3s-remote-control.yaml -K
|
||||||
|
; for debian: chmod 0440 /etc/sudoers.d/master
|
||||||
|
;ssh-copy-id -i ~/.ssh/id_ed25519 master@192.168.3.156 -> kopiere auf die Server
|
||||||
|
|
||||||
# different user name
|
# different user name
|
||||||
[dbworkers]
|
[pure_dbworkers]
|
||||||
192.168.3.92 ansible_user=master-2
|
invixel-ubuntu2 ansible_host=192.168.3.155 ansible_user=master
|
||||||
192.168.3.93 ansible_user=master-3
|
;192.168.3.122 ansible_user=master
|
||||||
192.168.3.94 ansible_user=master-4
|
invixel-vm5 ansible_host=192.168.3.95 ansible_user=master-5
|
||||||
|
invixel-vm6 ansible_host=192.168.3.96 ansible_user=master-6
|
||||||
|
invixel-ubuntu3 ansible_host=192.168.3.153 ansible_user=master
|
||||||
|
invixel-ubuntu5 ansible_host=192.168.3.154 ansible_user=master
|
||||||
|
|
||||||
[workers]
|
[workers]
|
||||||
192.168.3.95 ansible_user=master-5
|
invixel-debian1 ansible_host=192.168.3.151 ansible_user=master
|
||||||
192.168.3.96 ansible_user=master-6
|
invixel-ubuntu5 ansible_host=192.168.3.154 ansible_user=master
|
||||||
|
invixel-ubuntu3 ansible_host=192.168.3.153 ansible_user=master
|
||||||
|
|
||||||
|
# Hier bündeln wir die echten DB-Worker inklusive dem Master vm7
|
||||||
|
[dbworkers:children]
|
||||||
|
pure_dbworkers
|
||||||
|
|
||||||
|
[dbworkers:hosts]
|
||||||
|
invixel-vm1 # Ansible weiß jetzt, dass vm1 hier dazugehört und nutzt die IP/User von oben!
|
||||||
|
invixel-vm7
|
||||||
# This group combines both for easy targeting
|
# This group combines both for easy targeting
|
||||||
[k8s_nodes:children]
|
[k8s_nodes:children]
|
||||||
master
|
master
|
||||||
workers
|
workers
|
||||||
|
|
||||||
|
[k3s_agents:children]
|
||||||
|
pure_dbworkers
|
||||||
|
workers
|
||||||
# [k8s_nodes:vars]
|
# [k8s_nodes:vars]
|
||||||
# # Ensure Ansible uses the correct Python on the remote nodes
|
# # Ensure Ansible uses the correct Python on the remote nodes
|
||||||
# ansible_python_interpreter=/usr/bin/python3
|
# ansible_python_interpreter=/usr/bin/python3
|
||||||
# # If you use the same SSH key for all:
|
# # If you use the same SSH key for all:
|
||||||
# ansible_ssh_private_key_file=~/.ssh/id_rsa
|
# ansible_ssh_private_key_file=~/.ssh/id_rsa
|
||||||
|
|
||||||
|
# Das Endergebnis im Cluster
|
||||||
|
# Sobald dein Playbook durchgelaufen ist und du auf deinem Laptop kubectl get nodes eingibst, sieht deine Topologie so aus:
|
||||||
|
|
||||||
|
# 3 Master (HA-Control-Plane mit etcd): invixel-vm1, invixel-vm7 und invixel-ubuntu4. Alle drei können auch Pods ausführen.
|
||||||
|
|
||||||
|
# 3 dedizierte DB-Worker: .92, .93, .94 (plus invixel-vm7 als Master-DB-Hybrid).
|
||||||
|
|
||||||
|
# 5 dedizierte normale Worker: .95, .96, .153, .154, .155 (plus invixel-ubuntu4 als Master-Worker-Hybrid).
|
||||||
@ -9,22 +9,52 @@
|
|||||||
# ansible.builtin.lineinfile:
|
# ansible.builtin.lineinfile:
|
||||||
# path: "~/.bashrc"
|
# path: "~/.bashrc"
|
||||||
# line: 'export ANSIBLE_CONFIG="/mnt/f/Invixel/k8s-infra-cluster/ansible.cfg"'
|
# line: 'export ANSIBLE_CONFIG="/mnt/f/Invixel/k8s-infra-cluster/ansible.cfg"'
|
||||||
# state: present
|
#
|
||||||
|
# Run this on master node 192.168.3.91
|
||||||
|
# sudo mkdir -p /etc/rancher/k3s
|
||||||
|
# sudo nano /etc/rancher/k3s/config.yaml
|
||||||
|
# state: present
|
||||||
|
# bind-address: "192.168.3.91"
|
||||||
|
# advertise-address: "192.168.3.91"
|
||||||
|
# tls-san:
|
||||||
|
# - "192.168.3.91"
|
||||||
|
# cluster-init: true
|
||||||
|
|
||||||
|
# Reset on invixel-vm1 192.168.3.91
|
||||||
|
#sudo systemctl stop k3s
|
||||||
|
#sudo k3s server --cluster-reset
|
||||||
|
#sudo systemctl start k3s
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# sudo systemctl start k3s
|
||||||
|
|
||||||
- name: Setup Laptop to Control K3s Cluster
|
- name: Setup Laptop to Control K3s Cluster
|
||||||
hosts: master
|
hosts: master[0]
|
||||||
become: yes
|
become: yes
|
||||||
|
|
||||||
vars:
|
vars:
|
||||||
# Replace this with your Master's actual LAN or Tailscale IP
|
# Replace this with your Master's actual LAN or Tailscale IP
|
||||||
master_public_ip: "192.168.3.91"
|
master_public_ip: "192.168.3.156"
|
||||||
local_kube_path: "~/.kube/config"
|
local_kube_path: "~/.kube/config"
|
||||||
|
|
||||||
tasks:
|
tasks:
|
||||||
- name: Ensure K3s is running with TLS SAN for the Public IP
|
# - name: Ensure K3s is running with etcd HA mode (--cluster-init)
|
||||||
|
# # Ensure K3s is running with TLS SAN for the Public IP
|
||||||
|
# # curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--tls-san {{ master_public_ip }}" sh -
|
||||||
|
# ansible.builtin.shell: |
|
||||||
|
# # Auf dem ERSTEN, existierenden Master ausführen:
|
||||||
|
# curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--tls-san {{ master_public_ip }} --cluster-init" sh -
|
||||||
|
# register: k3s_install
|
||||||
|
|
||||||
|
|
||||||
|
# Run this only one time
|
||||||
|
- name: Ensure K3s is running with etcd HA mode (--cluster-init)
|
||||||
ansible.builtin.shell: |
|
ansible.builtin.shell: |
|
||||||
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--tls-san {{ master_public_ip }}" sh -
|
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--bind-address={{ master_public_ip }} --advertise-address={{ master_public_ip }} --tls-san {{ master_public_ip }} --cluster-init" sh -
|
||||||
register: k3s_install
|
# Verhindert mehrfaches Ausführen, wenn K3s bereits existiert:
|
||||||
|
args:
|
||||||
|
creates: /usr/local/bin/k3s
|
||||||
- name: Wait for k3s.yaml to be fully generated
|
- name: Wait for k3s.yaml to be fully generated
|
||||||
ansible.builtin.wait_for:
|
ansible.builtin.wait_for:
|
||||||
path: /etc/rancher/k3s/k3s.yaml
|
path: /etc/rancher/k3s/k3s.yaml
|
||||||
@ -45,6 +75,20 @@
|
|||||||
# dest: "{{ local_kube_path }}"
|
# dest: "{{ local_kube_path }}"
|
||||||
# mode: '0600'
|
# mode: '0600'
|
||||||
|
|
||||||
|
# For Phase 2
|
||||||
|
- name: Fetch the cluster join token from the first master
|
||||||
|
ansible.builtin.slurp:
|
||||||
|
src: /var/lib/rancher/k3s/server/node-token
|
||||||
|
register: master_token_encoded
|
||||||
|
|
||||||
|
- name: Share the token with the rest of the playbook run
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
cluster_token: "{{ master_token_encoded.content | b64decode | trim }}"
|
||||||
|
delegate_to: localhost
|
||||||
|
delegate_facts: true
|
||||||
|
# For Phase 2
|
||||||
|
|
||||||
|
|
||||||
- name: Read k3s.yaml from master
|
- name: Read k3s.yaml from master
|
||||||
ansible.builtin.slurp:
|
ansible.builtin.slurp:
|
||||||
src: /etc/rancher/k3s/k3s.yaml
|
src: /etc/rancher/k3s/k3s.yaml
|
||||||
@ -58,4 +102,76 @@
|
|||||||
dest: "{{ local_kube_path }}"
|
dest: "{{ local_kube_path }}"
|
||||||
mode: '0600'
|
mode: '0600'
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# PHASE 2: WEITERE CONTROL PLANES HINZUFÜGEN (JOIN)
|
||||||
|
# ==============================================================================
|
||||||
|
- name: Join Additional Control Planes to the Cluster
|
||||||
|
hosts: master[1:] # Führt dies auf ALLEN ANDEREN Servern in der Master-Gruppe aus (Server 2, 3, etc.)
|
||||||
|
become: yes
|
||||||
|
vars:
|
||||||
|
first_master_ip: "192.168.3.156" # Die IP des ersten Masters, an den sich alle anmelde
|
||||||
|
tasks:
|
||||||
|
- name: Join as additional HA Control Plane
|
||||||
|
ansible.builtin.shell: |
|
||||||
|
# Auf dem NEUEN, zweiten Server ausführen:
|
||||||
|
curl -sfL https://get.k3s.io | K3S_TOKEN="{{ hostvars['localhost']['cluster_token'] }}" INSTALL_K3S_EXEC="--server https://{{ first_master_ip }}:6443 --tls-san {{ ansible_host }}" sh -
|
||||||
|
# Hinweis: {{ ansible_host }} nimmt automatisch die IP des jeweiligen Servers aus deiner hosts.ini
|
||||||
|
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# PHASE 3: ALLE WORKER NODES (DB- & NORMALE WORKER) SAUBER HINZUFÜGEN
|
||||||
|
# ==============================================================================
|
||||||
|
- name: Join All Worker Nodes to the Cluster
|
||||||
|
hosts: k3s_agents # Benutze die Sammelgruppe aus deiner hosts.ini
|
||||||
|
become: yes
|
||||||
|
vars:
|
||||||
|
first_master_ip: "192.168.3.156"
|
||||||
|
tasks:
|
||||||
|
- name: Check if an old K3s installation exists
|
||||||
|
ansible.builtin.stat:
|
||||||
|
path: /usr/local/bin/k3s-agent-uninstall.sh
|
||||||
|
register: old_k3s_agent
|
||||||
|
|
||||||
|
- name: Clean old cluster configuration if present
|
||||||
|
ansible.builtin.shell: /usr/local/bin/k3s-agent-uninstall.sh
|
||||||
|
when: old_k3s_agent.stat.exists
|
||||||
|
|
||||||
|
- name: Join as K3s Agent (Fresh or Re-Join)
|
||||||
|
ansible.builtin.shell: |
|
||||||
|
curl -sfL https://get.k3s.io | K3S_URL="https://{{ first_master_ip }}:6443" K3S_TOKEN="{{ hostvars['localhost']['cluster_token'] }}" sh -
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# PHASE 4: KUBERNETES ROLES AUTOMATISCH ZUWEISEN
|
||||||
|
# ==============================================================================
|
||||||
|
- name: Assign Kubernetes Roles to Nodes
|
||||||
|
hosts: master[0]
|
||||||
|
become: yes
|
||||||
|
tasks:
|
||||||
|
- name: Label DB Workers
|
||||||
|
ansible.builtin.command:
|
||||||
|
# cmd: "kubectl label node {{ item }} node-role.kubernetes.io/dbworker= --overwrite"
|
||||||
|
# sudo kubectl label node invixel-ubuntu2 node-role.kubernetes.io/nworker=
|
||||||
|
# sudo kubectl label node invixel-ubuntu3 node-role.kubernetes.io/nworker=
|
||||||
|
# sudo kubectl label node invixel-ubuntu-5 node-role.kubernetes.io/nworker=
|
||||||
|
cmd: >
|
||||||
|
kubectl label node
|
||||||
|
{{ hostvars[item]['ansible_hostname'] | default(hostvars[item]['inventory_hostname']) }}
|
||||||
|
node-role.kubernetes.io/dbworker= --overwrite
|
||||||
|
loop: "{{ groups['pure_dbworkers'] }}"
|
||||||
|
ignore_errors: yes
|
||||||
|
|
||||||
|
- name: Label Normal Workers
|
||||||
|
ansible.builtin.command:
|
||||||
|
# cmd: "kubectl label node {{ item }} node-role.kubernetes.io/nworker= --overwrite"
|
||||||
|
cmd: >
|
||||||
|
kubectl label node
|
||||||
|
{{ hostvars[item]['ansible_hostname'] | default(hostvars[item]['inventory_hostname']) }}
|
||||||
|
node-role.kubernetes.io/nworker= --overwrite
|
||||||
|
# Verwendet die Namen aus deiner [workers] Gruppe in der hosts.ini
|
||||||
|
loop: "{{ groups['workers'] }}"
|
||||||
|
ignore_errors: yes
|
||||||
|
|
||||||
|
|
||||||
|
# export ANSIBLE_CONFIG=./ansible.cfg
|
||||||
|
|
||||||
# ansible-playbook -i hosts.ini k3s-remote-control.yml
|
# ansible-playbook -i hosts.ini k3s-remote-control.yml
|
||||||
49
manuals.md
49
manuals.md
@ -1,5 +1,18 @@
|
|||||||
### Getting Started
|
### Getting Started
|
||||||
|
#### Join new nodes
|
||||||
|
```
|
||||||
|
# retrieve token on control-plane node
|
||||||
|
sudo cat /var/lib/rancher/k3s/server/node-token
|
||||||
|
K10094c77f9748b705ce6fd128b916dd96793160853577fdcba986c57de9b6f4a3c::server:b5743ed19b035f78f59607b2fa3bda2e
|
||||||
|
|
||||||
|
# join via
|
||||||
|
curl -sfL https://get.k3s.io | K3S_URL=https://<your-server-ip>:6443 K3S_TOKEN=<NodeToken> sh -
|
||||||
|
|
||||||
|
K3S_URL: The address of your control plane node
|
||||||
|
|
||||||
|
curl -sfL https://get.k3s.io | K3S_URL=https://192.168.3.91:6443 K3S_TOKEN=K10094c77f9748b705ce6fd128b916dd96793160853577fdcba986c57de9b6f4a3c::server:b5743ed19b035f78f59607b2fa3bda2e sh -
|
||||||
|
|
||||||
|
```
|
||||||
#### 1. Using ansible
|
#### 1. Using ansible
|
||||||
```
|
```
|
||||||
a. ansible-playbook -i hosts.ini ansible-prepare-k3s.yml
|
a. ansible-playbook -i hosts.ini ansible-prepare-k3s.yml
|
||||||
@ -127,6 +140,18 @@ kubectl get secret postgres-ha-app -n db -o json | jq 'del(.metadata.namespace,.
|
|||||||
kubectl get secret postgres-ha-superuser -n db -o json | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid)' | kubectl apply -n infra -f -
|
kubectl get secret postgres-ha-superuser -n db -o json | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid)' | kubectl apply -n infra -f -
|
||||||
|
|
||||||
```
|
```
|
||||||
|
#### Avoiding sudo - Unable to read Error while executing kubectl command option
|
||||||
|
```
|
||||||
|
# Option 1: Change permissions (allows your user to read)
|
||||||
|
sudo chmod 644 /etc/rancher/k3s/k3s.yaml
|
||||||
|
|
||||||
|
# Option 2: Copy the config to your user's directory
|
||||||
|
mkdir -p ~/.kube
|
||||||
|
sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
|
||||||
|
sudo chown $(id -u):$(id -g) ~/.kube/config
|
||||||
|
chmod 600 ~/.kube/config
|
||||||
|
```
|
||||||
|
|
||||||
#### Debug
|
#### Debug
|
||||||
```
|
```
|
||||||
kubectl get all -A
|
kubectl get all -A
|
||||||
@ -177,12 +202,34 @@ sudo chmod 644 /etc/rancher/k3s/k3s.yaml
|
|||||||
Run in PS
|
Run in PS
|
||||||
kustomize build . --enable-helm | kubectl apply --server-side --force-conflicts -f -
|
kustomize build . --enable-helm | kubectl apply --server-side --force-conflicts -f -
|
||||||
|
|
||||||
4. pods exist in PS but not in wsl
|
4. pods exist in PS but not in wsl -- Join/Query from Laptop to k3s servers
|
||||||
|
|
||||||
|
# On WSL
|
||||||
mkdir -p ~/.kube
|
mkdir -p ~/.kube
|
||||||
sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
|
sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
|
||||||
sudo chown $(id -u):$(id -g) ~/.kube/config
|
sudo chown $(id -u):$(id -g) ~/.kube/config
|
||||||
export KUBECONFIG=~/.kube/config
|
export KUBECONFIG=~/.kube/config
|
||||||
|
|
||||||
|
change IP manually from 127.0.0.1 to 192.168.3.91
|
||||||
|
|
||||||
|
or
|
||||||
|
|
||||||
|
mkdir -p ~/.kube
|
||||||
|
scp master-1@192.168.3.91:/etc/rancher/k3s/k3s.yaml ~/.kube/config
|
||||||
|
sed -i 's/127.0.0.1/192.168.3.91/g' ~/.kube/config
|
||||||
|
chmod 600 ~/.kube/config
|
||||||
|
|
||||||
|
# Testen
|
||||||
|
kubectl get nodes
|
||||||
|
|
||||||
|
# on PS
|
||||||
|
ssh master@192.168.3.156 "sudo cat /etc/rancher/k3s/k3s.yaml" > "$HOME\.kube\config"
|
||||||
|
|
||||||
|
# Localhost-IP durch die echte Master-IP ersetzen
|
||||||
|
(Get-Content "$HOME\.kube\config") -replace '127.0.0.1', '192.168.3.156' | Set-Content "$HOME\.kube\config"
|
||||||
|
|
||||||
|
# Testen
|
||||||
|
kubectl get nodes
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -729,4 +729,31 @@ Delete:
|
|||||||
httproute.yaml: This is for the "Gateway API." Unless you have a specific Gateway controller installed, standard ingress.yaml is what you'll use.
|
httproute.yaml: This is for the "Gateway API." Unless you have a specific Gateway controller installed, standard ingress.yaml is what you'll use.
|
||||||
|
|
||||||
NOTES.txt: Usually just contains generic text.
|
NOTES.txt: Usually just contains generic text.
|
||||||
````
|
```
|
||||||
|
|
||||||
|
#### What ToDo after deleting the primary node that bootstrapped the cluster.
|
||||||
|
```
|
||||||
|
ssh to other k3s control plane
|
||||||
|
|
||||||
|
sudo systemctl status k3s
|
||||||
|
sudo systemctl stop k3s
|
||||||
|
sudo k3s server --cluster-reset
|
||||||
|
sudo systemctl start k3s
|
||||||
|
sudo kubectl get nodes
|
||||||
|
|
||||||
|
sudo nano /etc/systemd/system/k3s.service
|
||||||
|
|
||||||
|
from
|
||||||
|
ExecStart=/usr/local/bin/k3s server --server https://192.168.3.91:6443 --tls-san 192.168.3.156
|
||||||
|
|
||||||
|
to
|
||||||
|
ExecStart=/usr/local/bin/k3s server --cluster-init --tls-san 192.168.3.156
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Delete stale nodes
|
||||||
|
```
|
||||||
|
sudo kubectl delete node invixel-vm1
|
||||||
|
sudo kubectl delete node invixel-vm6
|
||||||
|
sudo kubectl delete node invixel-vm7
|
||||||
|
sudo kubectl delete node invixel-ubuntu1
|
||||||
|
```
|
||||||
@ -30,7 +30,7 @@ kubectl exec -it nats-box -n infra -- nats -s nats://nats-cluster.infra.svc.clus
|
|||||||
kubectl exec -it nats-box -n infra -- nats -s nats://nats-cluster:4222 stream info request_stream
|
kubectl exec -it nats-box -n infra -- nats -s nats://nats-cluster:4222 stream info request_stream
|
||||||
kubectl exec -it nats-cluster-0 -n infra -c nats -- netstat -tuln
|
kubectl exec -it nats-cluster-0 -n infra -c nats -- netstat -tuln
|
||||||
kubectl get pod nats-cluster-0 -n infra -o wide
|
kubectl get pod nats-cluster-0 -n infra -o wide
|
||||||
|
kubectl get svc --all-namespaces
|
||||||
kubectl get endpoints nats-cluster -n infra
|
kubectl get endpoints nats-cluster -n infra
|
||||||
|
|
||||||
|
|
||||||
@ -132,6 +132,10 @@ kubectl run net-test --image=alpine --restart=Never -n infra -- nslookup google.
|
|||||||
kubectl exec -it yb-tserver-0 -n db -- ysqlsh -h
|
kubectl exec -it yb-tserver-0 -n db -- ysqlsh -h
|
||||||
kubectl port-forward svc/cosmo-router 3002:3002 -n apps
|
kubectl port-forward svc/cosmo-router 3002:3002 -n apps
|
||||||
kubectl port-forward svc/postgres-ha-rw 5436:5432 -n db
|
kubectl port-forward svc/postgres-ha-rw 5436:5432 -n db
|
||||||
|
|
||||||
|
|
||||||
|
# (Best Practice für Zero-Downtime): Evakuiere die alten Nodes zuerst manuell (Drain): Für die alten dbworker und nworker ausführen:
|
||||||
|
kubectl drain <name-des-alten-nodes> --ignore-daemonsets --delete-emptydir-data
|
||||||
```
|
```
|
||||||
#### The Difference: NATS Core vs. JetStream
|
#### The Difference: NATS Core vs. JetStream
|
||||||
- NATS Core (nats sub): This is "Fire and Forget." If no one is listening when Benthos sends the message, the message disappears forever.
|
- NATS Core (nats sub): This is "Fire and Forget." If no one is listening when Benthos sends the message, the message disappears forever.
|
||||||
|
|||||||
@ -92,3 +92,832 @@ Reason to use: This is the most "production-ready" version of what you are doing
|
|||||||
|Ingress|Layer 7|A ""Receptionist"" directing visitors based on the name on the |envelope.|
|
|Ingress|Layer 7|A ""Receptionist"" directing visitors based on the name on the |envelope.|
|
||||||
|LoadBalancer|Layer 4|"A ""Direct Pipeline"" to a specific door (port).|
|
|LoadBalancer|Layer 4|"A ""Direct Pipeline"" to a specific door (port).|
|
||||||
|Tailscale / VPN|Layer 3|"A ""Secret Tunnel"" that puts your whole house next to the office.|
|
|Tailscale / VPN|Layer 3|"A ""Secret Tunnel"" that puts your whole house next to the office.|
|
||||||
|
|
||||||
|
#### Finding IP
|
||||||
|
public: curl ifconfig.me
|
||||||
|
|
||||||
|
|Target|Command|Requirement|
|
||||||
|
|----|----|-----|
|
||||||
|
|Inside Container| hostname -I /ip addr show eth0|LXC Container|
|
||||||
|
|LXC Local IP|pct exec ID ip a|Access to Proxmox Host|
|
||||||
|
|Proxmox Local IP|hostname -I|Access to Proxmox Host|
|
||||||
|
|Proxmox Public IP|curl ifconfig.me|Internet Access|
|
||||||
|
|
||||||
|
|
||||||
|
#### Ipaymu Integration need static IP
|
||||||
|
|
||||||
|
[Cloudflare Worker] ──1. Request──► [Your Nginx Server] ──2. Request──► [iPaymu API]
|
||||||
|
│ │
|
||||||
|
[Cloudflare Worker] ◄─4. Response── [Your Nginx Server] ◄─3. Response─── [iPaymu API]
|
||||||
|
Problem: we deploy using cloudflared worker which has dynamic ips
|
||||||
|
|
||||||
|
- Find IP of the server
|
||||||
|
- curl -4 ifconfig.me
|
||||||
|
- curl -4 icanhazip.com
|
||||||
|
- curl -4 ipinfo.io/ip
|
||||||
|
- ip -4 addr show
|
||||||
|
|
||||||
|
1. Option 1: Cloudflare Tunnel (cloudflared) — Easiest & Most Secure
|
||||||
|
A. cloudflared inside an LXC Container (Separate from K3s)
|
||||||
|
An LXC container on the same LAN/Proxmox host can run cloudflared and route incoming traffic directly to your K3s cluster.
|
||||||
|
|
||||||
|
[Cloudflare Worker]
|
||||||
|
│ (Public HTTPS)
|
||||||
|
▼
|
||||||
|
[Cloudflare Edge Network]
|
||||||
|
│ (Outbound Tunnel)
|
||||||
|
[LXC Container running cloudflared]
|
||||||
|
│ (Local LAN / Bridge Traffic)
|
||||||
|
[K3s Cluster Node (Static IP)] ──► [Nginx Proxy Pod] ──► [iPaymu API]
|
||||||
|
|
||||||
|
How to configure the LXC Tunnel:
|
||||||
|
When you set up the Public Hostname rule in the Cloudflare Zero Trust Dashboard (or config.yaml), point the origin URL directly to your K3s Node IP and port:
|
||||||
|
|
||||||
|
- Service Type: HTTP
|
||||||
|
|
||||||
|
- URL: <K3S_NODE_IP>:<NODE_PORT_OR_INGRESS_PORT>
|
||||||
|
(e.g., [http://192.168.1.50:80](http://192.168.1.50:80) if calling Traefik/Ingress, or [http://192.168.1.50:30080](http://192.168.1.50:30080) if using a NodePort service).
|
||||||
|
|
||||||
|
Pros & Cons:
|
||||||
|
- Pros: Keeps your K3s cluster lightweight; if K3s reboots, the tunnel daemon stays alive in the LXC container.
|
||||||
|
- Cons: Requires exposing a NodePort or LoadBalancer (Traefik) on your K3s node so the LXC container can hit it on the local network.
|
||||||
|
|
||||||
|
B. cloudflared as a Pod inside K3s
|
||||||
|
Deploying cloudflared directly inside K3s as a Deployment is the most common Kubernetes-native method.
|
||||||
|
[Cloudflare Worker]
|
||||||
|
│
|
||||||
|
▼ (HTTPS over public internet)
|
||||||
|
[https://proxy.yourdomain.com] <-- Cloudflare Edge
|
||||||
|
│
|
||||||
|
▼ (Secure outbound Tunnel)
|
||||||
|
[cloudflared Pod in K3s / Swarm]
|
||||||
|
│
|
||||||
|
▼ (Internal Cluster Traffic)
|
||||||
|
[Nginx Proxy Pod (Static-IP Node)]
|
||||||
|
│
|
||||||
|
▼ (Outbound call from Static IPv4)
|
||||||
|
[iPaymu API]
|
||||||
|
|
||||||
|
2. Option 2: Traefik Ingress Controller + NodePort / Public IP
|
||||||
|
If your static-IP cluster node already has a Public Static IPv4 Address directly bound to its network interface:
|
||||||
|
|
||||||
|
[Cloudflare Worker]
|
||||||
|
│
|
||||||
|
▼ (HTTPS to Public Static IP)
|
||||||
|
[Public IP:443] ──► [Traefik Ingress] ──► [Nginx Proxy Service] ──► [iPaymu]
|
||||||
|
|
||||||
|
How to configure the K3s Tunnel:
|
||||||
|
Inside the Cloudflare dashboard, you point the public hostname to K3s's internal DNS service name:
|
||||||
|
|
||||||
|
- Service Type: HTTP
|
||||||
|
- URL: ipaymu-proxy-service.default.svc.cluster.local:80
|
||||||
|
|
||||||
|
Pros & Cons:
|
||||||
|
- Pros: Highly secure—you do not need to open any ports or expose NodePort services on your LAN. Traffic moves entirely within K3s cluster networking.
|
||||||
|
- Cons: Runs inside the Kubernetes runtime lifecycle.
|
||||||
|
|
||||||
|
3. Option 3: Port Mapping directly on the Static Node (hostPort)
|
||||||
|
If you don't want to deal with Traefik or Ingress controllers, you can expose your Nginx proxy directly on a port (e.g., 8080 or 8443) of your static-IP node using hostPort in K3s or ports mapping in Swarm.
|
||||||
|
|
||||||
|
K3s Manifest Snippet:
|
||||||
|
containers:
|
||||||
|
- name: nginx
|
||||||
|
image: nginx:alpine
|
||||||
|
ports:
|
||||||
|
- containerPort: 80
|
||||||
|
hostPort: 8080 # Exposes port 8080 directly on the physical host
|
||||||
|
|
||||||
|
|
||||||
|
Vour Cloudflare Worker calls http://<YOUR_STATIC_NODE_IP>:8080/ipaymu/api/v2/payment.
|
||||||
|
|
||||||
|
- How the Request Flow Works End-to-End
|
||||||
|
Once exposed via Option 1, 2, or 3, here is the full round-trip execution path:
|
||||||
|
```
|
||||||
|
// 1. Cloudflare Worker executes this code:
|
||||||
|
const response = await fetch("https://proxy.yourdomain.com/ipaymu/api/v2/payment", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"X-Proxy-Secret": "YOUR_INTERNAL_SECRET",
|
||||||
|
"va": "1176000000000000",
|
||||||
|
"signature": "CALCULATED_HMAC_SIGNATURE",
|
||||||
|
"Content-Type": "application/json"
|
||||||
|
},
|
||||||
|
body: JSON.stringify(paymentPayload)
|
||||||
|
});
|
||||||
|
|
||||||
|
// 2. Request reaches https://proxy.yourdomain.com (Your Cluster via Tunnel/Ingress)
|
||||||
|
// 3. Ingress routes it internally to your Nginx proxy pod
|
||||||
|
// 4. Nginx verifies "X-Proxy-Secret", then forwards the request to https://api.ipaymu.com
|
||||||
|
// 5. iPaymu receives the call, sees your Node's STATIC IP, and accepts it
|
||||||
|
// 6. iPaymu returns payment data -> Nginx -> Ingress -> Cloudflare Worker -> Client
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
#### Step by step using ONLY Traefik in K3s
|
||||||
|
```
|
||||||
|
Step 1: Create Traefik Middlewares for Security and Stripping
|
||||||
|
Instead of Nginx if ($http_x_proxy_secret) directives, you define Traefik Middleware resources:
|
||||||
|
|
||||||
|
# 1. Security Header Check Middleware
|
||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: Middleware
|
||||||
|
metadata:
|
||||||
|
name: ipaymu-auth-check
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
headers:
|
||||||
|
customRequestHeaders:
|
||||||
|
# Ensures requests forwarded to iPaymu clean up internal proxy secrets
|
||||||
|
X-Proxy-Secret: ""
|
||||||
|
---
|
||||||
|
# 2. Path Strip Middleware (Removes /ipaymu prefix before forwarding)
|
||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: Middleware
|
||||||
|
metadata:
|
||||||
|
name: ipaymu-strip-prefix
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
stripPrefix:
|
||||||
|
prefixes:
|
||||||
|
- /ipaymu
|
||||||
|
|
||||||
|
Step 2: Define an ExternalName Service for iPaymu
|
||||||
|
To tell Traefik to forward traffic out to iPaymu's external domain, create a Kubernetes ExternalName service:
|
||||||
|
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: ipaymu-external-api
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
type: ExternalName
|
||||||
|
externalName: api.ipaymu.com
|
||||||
|
ports:
|
||||||
|
- port: 443
|
||||||
|
targetPort: 443
|
||||||
|
protocol: TCP
|
||||||
|
|
||||||
|
Step 3: Route Traffic with Traefik IngressRoute
|
||||||
|
Now, create a Traefik IngressRoute pinned to your static IP node:
|
||||||
|
|
||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: IngressRoute
|
||||||
|
metadata:
|
||||||
|
name: ipaymu-proxy-route
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
entryPoints:
|
||||||
|
- web
|
||||||
|
- websecure
|
||||||
|
routes:
|
||||||
|
- match: Host(`proxy.yourdomain.com`) && PathPrefix(`/ipaymu`)
|
||||||
|
kind: Rule
|
||||||
|
middlewares:
|
||||||
|
- name: ipaymu-strip-prefix
|
||||||
|
services:
|
||||||
|
- name: ipaymu-external-api
|
||||||
|
port: 443
|
||||||
|
scheme: https
|
||||||
|
|
||||||
|
When should you STILL use Nginx?
|
||||||
|
While Traefik replaces Nginx for 90% of use cases, keeping a tiny Nginx pod is still useful if:
|
||||||
|
|
||||||
|
Complex Logic: You want custom Nginx Lua scripts or complex header manipulation (e.g., computing dynamic HMAC signatures or custom retry logic on upstream failures).
|
||||||
|
|
||||||
|
Familiarity: You are already comfortable writing .conf files and don't want to learn Traefik-specific Kubernetes CRD syntax (IngressRoute / Middleware).
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Step by step using traeffik and nginx in k3s
|
||||||
|
```
|
||||||
|
1. Label your target node (Crucial Step)
|
||||||
|
Label the K3s node that holds your static public IPv4 address so K3s knows where to schedule the proxy pods:
|
||||||
|
|
||||||
|
kubectl label nodes <your-static-node-name> egress-ip=static-ipaymu
|
||||||
|
|
||||||
|
2. k3s-ipaymu-proxy.yaml
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: ipaymu-proxy-config
|
||||||
|
namespace: default
|
||||||
|
data:
|
||||||
|
default.conf: |
|
||||||
|
upstream ipaymu_backend {
|
||||||
|
server api.ipaymu.com:443;
|
||||||
|
keepalive 32;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
|
||||||
|
location /ipaymu/ {
|
||||||
|
if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET_KEY") {
|
||||||
|
return 403;
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy_pass https://ipaymu_backend/;
|
||||||
|
proxy_set_header Host api.ipaymu.com;
|
||||||
|
|
||||||
|
# SSL Handshake settings
|
||||||
|
proxy_ssl_server_name on;
|
||||||
|
proxy_ssl_protocols TLSv1.2 TLSv1.3;
|
||||||
|
|
||||||
|
# HTTP Keepalive
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: ipaymu-proxy
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
replicas: 2 # Scale up replicas as needed
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: ipaymu-proxy
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: ipaymu-proxy
|
||||||
|
spec:
|
||||||
|
# Pin all replicas to the node with the static public IP
|
||||||
|
nodeSelector:
|
||||||
|
egress-ip: static-ipaymu
|
||||||
|
containers:
|
||||||
|
- name: nginx
|
||||||
|
image: nginx:alpine
|
||||||
|
ports:
|
||||||
|
- containerPort: 80
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: "200m"
|
||||||
|
memory: "128Mi"
|
||||||
|
requests:
|
||||||
|
cpu: "50m"
|
||||||
|
memory: "32Mi"
|
||||||
|
volumeMounts:
|
||||||
|
- name: nginx-config
|
||||||
|
mountPath: /etc/nginx/conf.d/default.conf
|
||||||
|
subPath: default.conf
|
||||||
|
volumes:
|
||||||
|
- name: nginx-config
|
||||||
|
configMap:
|
||||||
|
name: ipaymu-proxy-config
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: ipaymu-proxy-service
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
ports:
|
||||||
|
- port: 80
|
||||||
|
targetPort: 80
|
||||||
|
protocol: TCP
|
||||||
|
selector:
|
||||||
|
app: ipaymu-proxy
|
||||||
|
|
||||||
|
3. Deploy : kubectl apply -f k3s-ipaymu-proxy.yaml
|
||||||
|
4. How to call it inside K3s
|
||||||
|
Any other pod inside your K3s cluster can now make payment requests directly using internal Kubernetes DNS:
|
||||||
|
[http://ipaymu-proxy-service.default.svc.cluster.local/ipaymu/api/v2/payment](http://ipaymu-proxy-service.default.svc.cluster.local/ipaymu/api/v2/payment)
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Step by step using traeffik and cloudflared in k3s
|
||||||
|
```
|
||||||
|
Step 2.1: Enable allowExternalNameServices in Traefik
|
||||||
|
By default, K3s disables Traefik from routing to external domains via ExternalName. Create a HelmChartConfig manifest on your K3s server node to enable it:
|
||||||
|
|
||||||
|
Create file /var/lib/rancher/k3s/server/manifests/traefik-config.yaml:
|
||||||
|
|
||||||
|
apiVersion: helm.cattle.io/v1
|
||||||
|
kind: HelmChartConfig
|
||||||
|
metadata:
|
||||||
|
name: traefik
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
valuesContent: |-
|
||||||
|
providers:
|
||||||
|
kubernetesCRD:
|
||||||
|
allowExternalNameServices: true
|
||||||
|
|
||||||
|
K3s automatically applies this change in under 30 seconds.
|
||||||
|
|
||||||
|
Step 2.2: Label your Static-IP Node
|
||||||
|
Label the node with your static public IPv4 address so K3s knows where to schedule the tunnel:
|
||||||
|
|
||||||
|
kubectl label nodes <your-node-name> egress-ip=static-ipaymu
|
||||||
|
|
||||||
|
Step 2.3: Deploy the iPaymu Traefik Route
|
||||||
|
Save this to ipaymu-traefik.yaml and run kubectl apply -f ipaymu-traefik.yaml:
|
||||||
|
|
||||||
|
# 1. External Name Service pointing to iPaymu
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: ipaymu-external-service
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
type: ExternalName
|
||||||
|
externalName: api.ipaymu.com
|
||||||
|
ports:
|
||||||
|
- name: https
|
||||||
|
port: 443
|
||||||
|
targetPort: 443
|
||||||
|
protocol: TCP
|
||||||
|
---
|
||||||
|
# 2. TLS Transport for SNI support
|
||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: ServersTransport
|
||||||
|
metadata:
|
||||||
|
name: ipaymu-transport
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
serverName: api.ipaymu.com
|
||||||
|
---
|
||||||
|
# 3. Path Stripper Middleware (/ipaymu/api/v2/payment -> /api/v2/payment)
|
||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: Middleware
|
||||||
|
metadata:
|
||||||
|
name: ipaymu-strip-prefix
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
stripPrefix:
|
||||||
|
prefixes:
|
||||||
|
- /ipaymu
|
||||||
|
---
|
||||||
|
# 4. Traefik IngressRoute
|
||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: IngressRoute
|
||||||
|
metadata:
|
||||||
|
name: ipaymu-proxy-route
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
entryPoints:
|
||||||
|
- web
|
||||||
|
- websecure
|
||||||
|
routes:
|
||||||
|
- match: Host(`proxy.yourdomain.com`) && PathPrefix(`/ipaymu`)
|
||||||
|
kind: Rule
|
||||||
|
middlewares:
|
||||||
|
- name: ipaymu-strip-prefix
|
||||||
|
services:
|
||||||
|
- name: ipaymu-external-service
|
||||||
|
port: 443
|
||||||
|
scheme: https
|
||||||
|
serversTransport: ipaymu-transport
|
||||||
|
|
||||||
|
Step 2.4:
|
||||||
|
- Get Your Tunnel Secret Token
|
||||||
|
1. Log into your Cloudflare Zero Trust Dashboard.
|
||||||
|
2. Navigate to Networks → Tunnels → Create a Tunnel.
|
||||||
|
3. Choose Cloudflared as the connector and name it (e.g., k3s-ipaymu-tunnel).
|
||||||
|
4. On the installation page, select Docker/Kubernetes. Look for the command and copy the long base64 token string after --token.
|
||||||
|
|
||||||
|
- Deploy Cloudflare Tunnel inside K3s
|
||||||
|
Save this to cloudflared.yaml (replace YOUR_TOKEN_HERE with your Cloudflare Zero Trust tunnel token) and run kubectl apply -f cloudflared.yaml:
|
||||||
|
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: cloudflared-token
|
||||||
|
namespace: default
|
||||||
|
type: Opaque
|
||||||
|
stringData:
|
||||||
|
TUNNEL_TOKEN: "YOUR_TOKEN_HERE"
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: cloudflared
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: cloudflared
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: cloudflared
|
||||||
|
spec:
|
||||||
|
nodeSelector:
|
||||||
|
egress-ip: static-ipaymu
|
||||||
|
containers:
|
||||||
|
- name: cloudflared
|
||||||
|
image: cloudflare/cloudflared:latest
|
||||||
|
args:
|
||||||
|
- tunnel
|
||||||
|
- --no-autoupdate
|
||||||
|
- run
|
||||||
|
env:
|
||||||
|
- name: TUNNEL_TOKEN
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: cloudflared-token
|
||||||
|
key: TUNNEL_TOKEN
|
||||||
|
|
||||||
|
|
||||||
|
kubectl apply -f cloudflared.yaml
|
||||||
|
|
||||||
|
Step 2.5: Cloudflare Dashboard Routing
|
||||||
|
In Cloudflare Zero Trust Dashboard (→ Networks → Tunnels → Public Hostname):
|
||||||
|
|
||||||
|
Public Hostname: proxy.yourdomain.com
|
||||||
|
|
||||||
|
Path: /ipaymu
|
||||||
|
|
||||||
|
Type: HTTP
|
||||||
|
|
||||||
|
URL: traefik.kube-system.svc.cluster.local:80
|
||||||
|
```
|
||||||
|
#### Step by step using nginx and cloudflared in docker-compose
|
||||||
|
```
|
||||||
|
Step 3.1: Create Directory Structure
|
||||||
|
On your target server, create a project directory:
|
||||||
|
mkdir -p ipaymu-proxy
|
||||||
|
cd ipaymu-proxy
|
||||||
|
|
||||||
|
ipaymu-proxy/
|
||||||
|
├── docker-compose.yml
|
||||||
|
└── nginx.conf
|
||||||
|
|
||||||
|
Step 3.2: Create nginx.conf
|
||||||
|
events { worker_connections 1024; }
|
||||||
|
|
||||||
|
http {
|
||||||
|
upstream ipaymu_api {
|
||||||
|
server api.ipaymu.com:443;
|
||||||
|
keepalive 16;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name _;
|
||||||
|
|
||||||
|
location /ipaymu/ {
|
||||||
|
# Check secret key header sent by your Cloudflare Worker
|
||||||
|
if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET_KEY") {
|
||||||
|
return 403;
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy_pass https://ipaymu_api/;
|
||||||
|
proxy_set_header Host api.ipaymu.com;
|
||||||
|
|
||||||
|
# Ensure TLS handshake works with iPaymu
|
||||||
|
proxy_ssl_server_name on;
|
||||||
|
proxy_ssl_protocols TLSv1.2 TLSv1.3;
|
||||||
|
|
||||||
|
# HTTP Keepalive optimizations
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Step 3.3: Create docker-compose.yml
|
||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
ipaymu-proxy:
|
||||||
|
image: nginx:alpine
|
||||||
|
container_name: ipaymu_proxy
|
||||||
|
restart: always
|
||||||
|
ports:
|
||||||
|
- "80:80" # Or expose via reverse proxy / Cloudflare tunnel
|
||||||
|
volumes:
|
||||||
|
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "nginx", "-t"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
or with nginx:
|
||||||
|
|
||||||
|
#docker-compose.yml
|
||||||
|
version: '3.8'
|
||||||
|
services:
|
||||||
|
ipaymu-nginx:
|
||||||
|
image: nginx:alpine
|
||||||
|
container_name: ipaymu_nginx
|
||||||
|
restart: always
|
||||||
|
volumes:
|
||||||
|
- ./nginx.conf:/etc/nginx/nginx.conf:ro
|
||||||
|
networks:
|
||||||
|
- proxy-net
|
||||||
|
|
||||||
|
cloudflared:
|
||||||
|
image: cloudflare/cloudflared:latest
|
||||||
|
container_name: ipaymu_tunnel
|
||||||
|
restart: always
|
||||||
|
command: tunnel --no-autoupdate run
|
||||||
|
environment:
|
||||||
|
- TUNNEL_TOKEN=YOUR_CLOUDFLARE_TUNNEL_TOKEN_HERE
|
||||||
|
networks:
|
||||||
|
- proxy-net
|
||||||
|
|
||||||
|
networks:
|
||||||
|
proxy-net:
|
||||||
|
driver: bridge
|
||||||
|
|
||||||
|
Step 3.4: Deploy Docker Compose: docker compose up -d
|
||||||
|
Step 3.5: Cloudflare Dashboard RoutingIn Cloudflare Zero Trust Dashboard ($\rightarrow$ Networks $\rightarrow$ Tunnels $\rightarrow$ Public Hostname):
|
||||||
|
Public Hostname: proxy.yourdomain.com
|
||||||
|
Path: /ipaymuType: HTTP
|
||||||
|
URL: ipaymu_nginx:80 (Connects directly using Docker's internal container DNS).
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Step by step using docker swarm
|
||||||
|
|
||||||
|
```
|
||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
ipaymu-proxy:
|
||||||
|
image: nginx:alpine
|
||||||
|
ports:
|
||||||
|
- "80:80"
|
||||||
|
- "443:443"
|
||||||
|
configs:
|
||||||
|
- source: nginx_config
|
||||||
|
target: /etc/nginx/conf.d/default.conf
|
||||||
|
deploy:
|
||||||
|
mode: replicated
|
||||||
|
replicas: 1
|
||||||
|
placement:
|
||||||
|
constraints:
|
||||||
|
# Pin execution to the specific node with the static public IP
|
||||||
|
- node.hostname == node-with-static-ip
|
||||||
|
|
||||||
|
configs:
|
||||||
|
nginx_config:
|
||||||
|
inline: |
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name ipaymu-proxy.yourdomain.com;
|
||||||
|
|
||||||
|
location /ipaymu/ {
|
||||||
|
if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET") {
|
||||||
|
return 403;
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy_pass https://api.ipaymu.com/;
|
||||||
|
proxy_set_header Host api.ipaymu.com;
|
||||||
|
proxy_ssl_server_name on;
|
||||||
|
|
||||||
|
proxy_ssl_protocols TLSv1.2 TLSv1.3;
|
||||||
|
|
||||||
|
# Pass client details
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Cloudflare Worker]
|
||||||
|
│
|
||||||
|
├─── /ipaymu/v2/payment ─────────► [Nginx] ──► https://my.ipaymu.com/api/v2/payment
|
||||||
|
│
|
||||||
|
└─── /ipaymu-sandbox/v2/payment ─► [Nginx] ──► https://sandbox.ipaymu.com/api/v2/payment
|
||||||
|
|
||||||
|
1.a Create the dedicated folder in /opt/
|
||||||
|
mkdir -p /opt/ipaymu-swarm-proxy
|
||||||
|
cd /opt/ipaymu-swarm-proxy
|
||||||
|
|
||||||
|
|
||||||
|
b create the nginx configuration file nginx.conf:
|
||||||
|
```
|
||||||
|
events {
|
||||||
|
worker_connections 1024;
|
||||||
|
}
|
||||||
|
|
||||||
|
http {
|
||||||
|
# Upstream targets using persistent HTTP keepalives
|
||||||
|
upstream ipaymu_live {
|
||||||
|
server my.ipaymu.com:443;
|
||||||
|
keepalive 16;
|
||||||
|
}
|
||||||
|
|
||||||
|
upstream ipaymu_sandbox {
|
||||||
|
server sandbox.ipaymu.com:443;
|
||||||
|
keepalive 16;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name _;
|
||||||
|
|
||||||
|
# -------------------------------------------------------------
|
||||||
|
# 1. Production / Live Endpoint
|
||||||
|
# -------------------------------------------------------------
|
||||||
|
location /ipaymu/v2/ {
|
||||||
|
# Security Header Check
|
||||||
|
if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET_KEY") {
|
||||||
|
return 403;
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy_pass https://ipaymu_live/api/v2/;
|
||||||
|
proxy_set_header Host my.ipaymu.com;
|
||||||
|
|
||||||
|
# Ensure TLS/SNI Handshake matches my.ipaymu.com
|
||||||
|
proxy_ssl_server_name on;
|
||||||
|
proxy_ssl_name my.ipaymu.com;
|
||||||
|
proxy_ssl_protocols TLSv1.2 TLSv1.3;
|
||||||
|
|
||||||
|
# Connection optimizations
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
proxy_pass_request_headers on;
|
||||||
|
}
|
||||||
|
|
||||||
|
# -------------------------------------------------------------
|
||||||
|
# 2. Development / Sandbox Endpoint
|
||||||
|
# -------------------------------------------------------------
|
||||||
|
location /ipaymu-sandbox/v2/ {
|
||||||
|
# Security Header Check
|
||||||
|
if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET_KEY") {
|
||||||
|
return 403;
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy_pass https://ipaymu_sandbox/api/v2/;
|
||||||
|
proxy_set_header Host sandbox.ipaymu.com;
|
||||||
|
|
||||||
|
# Ensure TLS/SNI Handshake matches sandbox.ipaymu.com
|
||||||
|
proxy_ssl_server_name on;
|
||||||
|
proxy_ssl_name sandbox.ipaymu.com;
|
||||||
|
proxy_ssl_protocols TLSv1.2 TLSv1.3;
|
||||||
|
|
||||||
|
# Connection optimizations
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
proxy_pass_request_headers on;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Change ipaymu
|
||||||
|
```
|
||||||
|
# erstelle einen starker schlüssel YOUR_INTERNAL_SECRET_KEY:
|
||||||
|
openssl rand -hex 32
|
||||||
|
|
||||||
|
# Setze diesen Schlüssel in deinen Nginx-Code (nginx.conf):
|
||||||
|
if ($http_x_proxy_secret != "0fb1d9712f16bb64e0dff4f911af02e8a2280722c80097a845e248c7a49781d7") {
|
||||||
|
return 403;
|
||||||
|
}
|
||||||
|
|
||||||
|
Setze denselben Schlüssel in deinen Cloudflare Worker / App-Code:
|
||||||
|
|
||||||
|
const headers = {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"X-Proxy-Secret": "0fb1d9712f16bb64e0dff4f911af02e8a2280722c80097a845e248c7a49781d7", // Gleicher Schlüssel!
|
||||||
|
// ...
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
CONTAINER_ID=$(docker ps -q -f name=ipaymu_nginx-proxy | head -n1)
|
||||||
|
docker exec -it $CONTAINER_ID apk add --no-cache curl
|
||||||
|
docker exec -it $CONTAINER_ID curl -4 ifconfig.me
|
||||||
|
|
||||||
|
curl -i -X POST https://proxy.yourdomain.com/ipaymu/v2/payment \
|
||||||
|
-H "X-Proxy-Secret: YOUR_INTERNAL_SECRET_KEY" \
|
||||||
|
-H "Content-Type: application/json"
|
||||||
|
```
|
||||||
|
1. Das Label auf den gewählten Node setzen
|
||||||
|
Sobald du weißt, welcher Node die statische IP hat (z. B. node-01), vergibst du das Label egress=static-ipaymu:
|
||||||
|
```
|
||||||
|
docker node update --label-add egress=static-ipaymu node-01
|
||||||
|
|
||||||
|
docker node update --label-add egress=static-ipaymu <node-name-or-id>
|
||||||
|
docker node inspect node-01 --format '{{ json .Spec.Labels }}'
|
||||||
|
```
|
||||||
|
|
||||||
|
4. Deploy And check
|
||||||
|
docker run cloudflare/cloudflared:latest tunnel --no-autoupdate run --token eyJhIjoiNjFhMTZkYzg3MmM4ZjlmZTQ4Mzc0ZWQ5NzVmNTcwNmIiLCJ0IjoiNTA5ZmRlZjQtZjEzYy00ZTQ1LTllZDQtYjZhMjExOTFlY2Y3IiwicyI6Ik5UWmtZVFU1T1dZdE5XVmtZUzAwTldZeExXSm1Oek10TlRRd1pqUTBNR1kxWW1RMSJ9
|
||||||
|
|
||||||
|
- Option 1 Using Docker Swarm env ( does not work )
|
||||||
|
```
|
||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
nginx-proxy:
|
||||||
|
image: nginx:alpine
|
||||||
|
configs:
|
||||||
|
- source: nginx_config_v1
|
||||||
|
target: /etc/nginx/nginx.conf
|
||||||
|
networks:
|
||||||
|
- payment-network
|
||||||
|
deploy:
|
||||||
|
mode: replicated
|
||||||
|
replicas: 2
|
||||||
|
placement:
|
||||||
|
constraints:
|
||||||
|
# Pin to the node holding your static IP for iPaymu whitelisting
|
||||||
|
- node.labels.egress == static-ipaymu
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
|
||||||
|
cloudflared:
|
||||||
|
image: cloudflare/cloudflared:latest
|
||||||
|
command: tunnel --no-autoupdate run
|
||||||
|
environment:
|
||||||
|
- TUNNEL_TOKEN=${CLOUDFLARE_TUNNEL_TOKEN}
|
||||||
|
networks:
|
||||||
|
- payment-network
|
||||||
|
deploy:
|
||||||
|
mode: replicated
|
||||||
|
replicas: 2
|
||||||
|
placement:
|
||||||
|
constraints:
|
||||||
|
- node.labels.egress == static-ipaymu
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
|
||||||
|
configs:
|
||||||
|
nginx_config_v1:
|
||||||
|
file: ./nginx.conf
|
||||||
|
|
||||||
|
networks:
|
||||||
|
payment-network:
|
||||||
|
driver: overlay
|
||||||
|
attachable: true
|
||||||
|
```
|
||||||
|
create .env file in the same directory:
|
||||||
|
sudo nano /opt/ipaymu-swarm-proxy/.env
|
||||||
|
CLOUDFLARE_TUNNEL_TOKEN=eyJhIjoiNjFhMTZkYzg3MmM4ZjlmZTQ4Mzc0ZWQ5NzVmNTcwNmIiLCJ0IjoiNTA5ZmRlZjQtZjEzYy00ZTQ1LTllZDQtYjZhMjExOTFlY2Y3IiwicyI6Ik5UWmtZVFU1T1dZdE5XVmtZUzAwTldZeExXSm1Oek10TlRRd1pqUTBNR1kxWW1RMSJ9
|
||||||
|
|
||||||
|
docker run cloudflare/cloudflared:latest tunnel --no-autoupdate run --token eyJhIjoiNjFhMTZkYzg3MmM4ZjlmZTQ4Mzc0ZWQ5NzVmNTcwNmIiLCJ0IjoiNTA5ZmRlZjQtZjEzYy00ZTQ1LTllZDQtYjZhMjExOTFlY2Y3IiwicyI6Ik5UWmtZVFU1T1dZdE5XVmtZUzAwTldZeExXSm1Oek10TlRRd1pqUTBNR1kxWW1RMSJ9
|
||||||
|
|
||||||
|
docker stack deploy -c ipaymu-swarm.yml ipaymu
|
||||||
|
or
|
||||||
|
export $(cat /opt/ipaymu-swarm-proxy/.env | xargs) && sudo -E docker stack deploy -c ipaymu-swarm.yml ipaymu
|
||||||
|
|
||||||
|
or
|
||||||
|
|
||||||
|
set -a && source /opt/ipaymu-swarm-proxy/.env && set +a && sudo -E docker stack deploy -c ipaymu-swarm.yml ipaymu
|
||||||
|
|
||||||
|
or
|
||||||
|
|
||||||
|
CLOUDFLARE_TUNNEL_TOKEN="eyJhIjoiNjFhMTZkYzg3MmM4ZjlmZTQ4Mzc0ZWQ5NzVmNTcwNmIiLCJ0IjoiNTA5ZmRlZjQtZjEzYy00ZTQ1LTllZDQtYjZhMjExOTFlY2Y3IiwicyI6Ik5UWmtZVFU1T1dZdE5XVmtZUzAwTldZeExXSm1Oek10TlRRd1pqUTBNR1kxWW1RMSJ9" sudo -E docker stack deploy -c ipaymu-swarm.yml ipaymu
|
||||||
|
|
||||||
|
# Option 2 Docker Swarm Secret
|
||||||
|
Create a Docker Secret on your Manager node:
|
||||||
|
printf "your_actual_token_here" | docker secret create cloudflared_tunnel_token -
|
||||||
|
|
||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
cloudflared:
|
||||||
|
image: cloudflare/cloudflared:latest
|
||||||
|
# Tells cloudflared to read the token file created by Docker Secrets
|
||||||
|
command: tunnel --no-autoupdate run --token-file /run/secrets/cloudflared_token
|
||||||
|
secrets:
|
||||||
|
- source: cloudflared_tunnel_token
|
||||||
|
target: cloudflared_token
|
||||||
|
networks:
|
||||||
|
- ipaymu-overlay
|
||||||
|
deploy:
|
||||||
|
mode: replicated
|
||||||
|
replicas: 2
|
||||||
|
placement:
|
||||||
|
constraints:
|
||||||
|
- node.labels.egress == static-ipaymu
|
||||||
|
|
||||||
|
secrets:
|
||||||
|
cloudflared_tunnel_token:
|
||||||
|
external: true
|
||||||
|
|
||||||
|
Summary Recommendation
|
||||||
|
For simple setups: Use Method 1 (.env file).
|
||||||
|
For strict security requirements: Use Method 4 (Docker Secrets).
|
||||||
|
|
||||||
|
docker stack services ipaymu
|
||||||
|
docker service logs ipaymu_cloudflared --tail 20
|
||||||
|
|
||||||
|
Now you need to label the node like in docker-stack.yml:
|
||||||
|
docker node update --label-add egress=static-ipaymu <node-hostname-or-id>
|
||||||
|
|
||||||
|
docker node update --label-add egress=static-ipaymu invixel-debian1
|
||||||
|
docker node update --label-add egress=static-ipaymu invixel-vm5
|
||||||
|
|
||||||
|
How to verify the exit IP
|
||||||
|
After deploying to K3s or Swarm, exec into your Nginx container/pod and test what public IP iPaymu will see:
|
||||||
|
|
||||||
|
docker exec -it <container_id> curl -4 ifconfig.me
|
||||||
|
|
||||||
|
kubectl exec -it deployment/ipaymu-proxy -- curl -4 ifconfig.me
|
||||||
|
|
||||||
|
|
||||||
|
curl -i -X POST https://bali-car-dev.easy-rent.com/ipaymu/v2/payment \
|
||||||
|
-H "X-Proxy-Secret: 0fb1d9712f16bb64e0dff4f911af02e8a2280722c80097a845e248c7a49781d7" \
|
||||||
|
-H "Content-Type: application/json"
|
||||||
|
```
|
||||||
Loading…
x
Reference in New Issue
Block a user