initial setup - most importants
This commit is contained in:
wendi gunawan
parent
4119e44d83
commit
b458d5805f
@ -12,26 +12,64 @@
|
||||
; 192.168.3.93
|
||||
; 192.168.3.92
|
||||
; 192.168.3.94
|
||||
;[master]
|
||||
;192.168.3.91 ansible_user=master-1
|
||||
;192.168.3.120 ansible-user=master
|
||||
;192.168.3.97 ansible-user=master-7
|
||||
[master]
|
||||
192.168.3.91 ansible_user=master-1
|
||||
invixel-ubuntu4 ansible_host=192.168.3.156 ansible_user=master
|
||||
invixel-vm1 ansible_host=192.168.3.91 ansible_user=master-1
|
||||
invixel-vm7 ansible_host=192.168.3.97 ansible_user=master-7
|
||||
;ssh-keygen -t ed25519 -C "josh@admin" -> generiere ssh schlüssel paar einfach zweimal leer Enter drücken
|
||||
;sudo visudo : in the server you want to add
|
||||
;add master ALL=(ALL) NOPASSWD:ALL
|
||||
; oder echo "master ALL=(ALL) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/master -> invixel-ubuntu2
|
||||
; for debian: echo "master ALL=(ALL) NOPASSWD:ALL" | tee /etc/sudoers.d/master
|
||||
; --> alternativ ohne Änderung auf Visudo mit -K ask nach Root Password
|
||||
; ansible-playbook -i hosts.ini k3s-remote-control.yaml -K
|
||||
; for debian: chmod 0440 /etc/sudoers.d/master
|
||||
;ssh-copy-id -i ~/.ssh/id_ed25519 master@192.168.3.156 -> kopiere auf die Server
|
||||
|
||||
# different user name
|
||||
[dbworkers]
|
||||
192.168.3.92 ansible_user=master-2
|
||||
192.168.3.93 ansible_user=master-3
|
||||
192.168.3.94 ansible_user=master-4
|
||||
[pure_dbworkers]
|
||||
invixel-ubuntu2 ansible_host=192.168.3.155 ansible_user=master
|
||||
;192.168.3.122 ansible_user=master
|
||||
invixel-vm5 ansible_host=192.168.3.95 ansible_user=master-5
|
||||
invixel-vm6 ansible_host=192.168.3.96 ansible_user=master-6
|
||||
invixel-ubuntu3 ansible_host=192.168.3.153 ansible_user=master
|
||||
invixel-ubuntu5 ansible_host=192.168.3.154 ansible_user=master
|
||||
|
||||
[workers]
|
||||
192.168.3.95 ansible_user=master-5
|
||||
192.168.3.96 ansible_user=master-6
|
||||
invixel-debian1 ansible_host=192.168.3.151 ansible_user=master
|
||||
invixel-ubuntu5 ansible_host=192.168.3.154 ansible_user=master
|
||||
invixel-ubuntu3 ansible_host=192.168.3.153 ansible_user=master
|
||||
|
||||
# Hier bündeln wir die echten DB-Worker inklusive dem Master vm7
|
||||
[dbworkers:children]
|
||||
pure_dbworkers
|
||||
|
||||
[dbworkers:hosts]
|
||||
invixel-vm1 # Ansible weiß jetzt, dass vm1 hier dazugehört und nutzt die IP/User von oben!
|
||||
invixel-vm7
|
||||
# This group combines both for easy targeting
|
||||
[k8s_nodes:children]
|
||||
master
|
||||
workers
|
||||
|
||||
|
||||
[k3s_agents:children]
|
||||
pure_dbworkers
|
||||
workers
|
||||
# [k8s_nodes:vars]
|
||||
# # Ensure Ansible uses the correct Python on the remote nodes
|
||||
# ansible_python_interpreter=/usr/bin/python3
|
||||
# # If you use the same SSH key for all:
|
||||
# ansible_ssh_private_key_file=~/.ssh/id_rsa
|
||||
|
||||
# Das Endergebnis im Cluster
|
||||
# Sobald dein Playbook durchgelaufen ist und du auf deinem Laptop kubectl get nodes eingibst, sieht deine Topologie so aus:
|
||||
|
||||
# 3 Master (HA-Control-Plane mit etcd): invixel-vm1, invixel-vm7 und invixel-ubuntu4. Alle drei können auch Pods ausführen.
|
||||
|
||||
# 3 dedizierte DB-Worker: .92, .93, .94 (plus invixel-vm7 als Master-DB-Hybrid).
|
||||
|
||||
# 5 dedizierte normale Worker: .95, .96, .153, .154, .155 (plus invixel-ubuntu4 als Master-Worker-Hybrid).
|
||||
@ -9,22 +9,52 @@
|
||||
# ansible.builtin.lineinfile:
|
||||
# path: "~/.bashrc"
|
||||
# line: 'export ANSIBLE_CONFIG="/mnt/f/Invixel/k8s-infra-cluster/ansible.cfg"'
|
||||
# state: present
|
||||
#
|
||||
# Run this on master node 192.168.3.91
|
||||
# sudo mkdir -p /etc/rancher/k3s
|
||||
# sudo nano /etc/rancher/k3s/config.yaml
|
||||
# state: present
|
||||
# bind-address: "192.168.3.91"
|
||||
# advertise-address: "192.168.3.91"
|
||||
# tls-san:
|
||||
# - "192.168.3.91"
|
||||
# cluster-init: true
|
||||
|
||||
# Reset on invixel-vm1 192.168.3.91
|
||||
#sudo systemctl stop k3s
|
||||
#sudo k3s server --cluster-reset
|
||||
#sudo systemctl start k3s
|
||||
|
||||
|
||||
|
||||
# sudo systemctl start k3s
|
||||
|
||||
- name: Setup Laptop to Control K3s Cluster
|
||||
hosts: master
|
||||
hosts: master[0]
|
||||
become: yes
|
||||
|
||||
vars:
|
||||
# Replace this with your Master's actual LAN or Tailscale IP
|
||||
master_public_ip: "192.168.3.91"
|
||||
master_public_ip: "192.168.3.156"
|
||||
local_kube_path: "~/.kube/config"
|
||||
|
||||
tasks:
|
||||
- name: Ensure K3s is running with TLS SAN for the Public IP
|
||||
# - name: Ensure K3s is running with etcd HA mode (--cluster-init)
|
||||
# # Ensure K3s is running with TLS SAN for the Public IP
|
||||
# # curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--tls-san {{ master_public_ip }}" sh -
|
||||
# ansible.builtin.shell: |
|
||||
# # Auf dem ERSTEN, existierenden Master ausführen:
|
||||
# curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--tls-san {{ master_public_ip }} --cluster-init" sh -
|
||||
# register: k3s_install
|
||||
|
||||
|
||||
# Run this only one time
|
||||
- name: Ensure K3s is running with etcd HA mode (--cluster-init)
|
||||
ansible.builtin.shell: |
|
||||
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--tls-san {{ master_public_ip }}" sh -
|
||||
register: k3s_install
|
||||
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--bind-address={{ master_public_ip }} --advertise-address={{ master_public_ip }} --tls-san {{ master_public_ip }} --cluster-init" sh -
|
||||
# Verhindert mehrfaches Ausführen, wenn K3s bereits existiert:
|
||||
args:
|
||||
creates: /usr/local/bin/k3s
|
||||
- name: Wait for k3s.yaml to be fully generated
|
||||
ansible.builtin.wait_for:
|
||||
path: /etc/rancher/k3s/k3s.yaml
|
||||
@ -45,6 +75,20 @@
|
||||
# dest: "{{ local_kube_path }}"
|
||||
# mode: '0600'
|
||||
|
||||
# For Phase 2
|
||||
- name: Fetch the cluster join token from the first master
|
||||
ansible.builtin.slurp:
|
||||
src: /var/lib/rancher/k3s/server/node-token
|
||||
register: master_token_encoded
|
||||
|
||||
- name: Share the token with the rest of the playbook run
|
||||
ansible.builtin.set_fact:
|
||||
cluster_token: "{{ master_token_encoded.content | b64decode | trim }}"
|
||||
delegate_to: localhost
|
||||
delegate_facts: true
|
||||
# For Phase 2
|
||||
|
||||
|
||||
- name: Read k3s.yaml from master
|
||||
ansible.builtin.slurp:
|
||||
src: /etc/rancher/k3s/k3s.yaml
|
||||
@ -58,4 +102,76 @@
|
||||
dest: "{{ local_kube_path }}"
|
||||
mode: '0600'
|
||||
|
||||
# ==============================================================================
|
||||
# PHASE 2: WEITERE CONTROL PLANES HINZUFÜGEN (JOIN)
|
||||
# ==============================================================================
|
||||
- name: Join Additional Control Planes to the Cluster
|
||||
hosts: master[1:] # Führt dies auf ALLEN ANDEREN Servern in der Master-Gruppe aus (Server 2, 3, etc.)
|
||||
become: yes
|
||||
vars:
|
||||
first_master_ip: "192.168.3.156" # Die IP des ersten Masters, an den sich alle anmelde
|
||||
tasks:
|
||||
- name: Join as additional HA Control Plane
|
||||
ansible.builtin.shell: |
|
||||
# Auf dem NEUEN, zweiten Server ausführen:
|
||||
curl -sfL https://get.k3s.io | K3S_TOKEN="{{ hostvars['localhost']['cluster_token'] }}" INSTALL_K3S_EXEC="--server https://{{ first_master_ip }}:6443 --tls-san {{ ansible_host }}" sh -
|
||||
# Hinweis: {{ ansible_host }} nimmt automatisch die IP des jeweiligen Servers aus deiner hosts.ini
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# PHASE 3: ALLE WORKER NODES (DB- & NORMALE WORKER) SAUBER HINZUFÜGEN
|
||||
# ==============================================================================
|
||||
- name: Join All Worker Nodes to the Cluster
|
||||
hosts: k3s_agents # Benutze die Sammelgruppe aus deiner hosts.ini
|
||||
become: yes
|
||||
vars:
|
||||
first_master_ip: "192.168.3.156"
|
||||
tasks:
|
||||
- name: Check if an old K3s installation exists
|
||||
ansible.builtin.stat:
|
||||
path: /usr/local/bin/k3s-agent-uninstall.sh
|
||||
register: old_k3s_agent
|
||||
|
||||
- name: Clean old cluster configuration if present
|
||||
ansible.builtin.shell: /usr/local/bin/k3s-agent-uninstall.sh
|
||||
when: old_k3s_agent.stat.exists
|
||||
|
||||
- name: Join as K3s Agent (Fresh or Re-Join)
|
||||
ansible.builtin.shell: |
|
||||
curl -sfL https://get.k3s.io | K3S_URL="https://{{ first_master_ip }}:6443" K3S_TOKEN="{{ hostvars['localhost']['cluster_token'] }}" sh -
|
||||
|
||||
# ==============================================================================
|
||||
# PHASE 4: KUBERNETES ROLES AUTOMATISCH ZUWEISEN
|
||||
# ==============================================================================
|
||||
- name: Assign Kubernetes Roles to Nodes
|
||||
hosts: master[0]
|
||||
become: yes
|
||||
tasks:
|
||||
- name: Label DB Workers
|
||||
ansible.builtin.command:
|
||||
# cmd: "kubectl label node {{ item }} node-role.kubernetes.io/dbworker= --overwrite"
|
||||
# sudo kubectl label node invixel-ubuntu2 node-role.kubernetes.io/nworker=
|
||||
# sudo kubectl label node invixel-ubuntu3 node-role.kubernetes.io/nworker=
|
||||
# sudo kubectl label node invixel-ubuntu-5 node-role.kubernetes.io/nworker=
|
||||
cmd: >
|
||||
kubectl label node
|
||||
{{ hostvars[item]['ansible_hostname'] | default(hostvars[item]['inventory_hostname']) }}
|
||||
node-role.kubernetes.io/dbworker= --overwrite
|
||||
loop: "{{ groups['pure_dbworkers'] }}"
|
||||
ignore_errors: yes
|
||||
|
||||
- name: Label Normal Workers
|
||||
ansible.builtin.command:
|
||||
# cmd: "kubectl label node {{ item }} node-role.kubernetes.io/nworker= --overwrite"
|
||||
cmd: >
|
||||
kubectl label node
|
||||
{{ hostvars[item]['ansible_hostname'] | default(hostvars[item]['inventory_hostname']) }}
|
||||
node-role.kubernetes.io/nworker= --overwrite
|
||||
# Verwendet die Namen aus deiner [workers] Gruppe in der hosts.ini
|
||||
loop: "{{ groups['workers'] }}"
|
||||
ignore_errors: yes
|
||||
|
||||
|
||||
# export ANSIBLE_CONFIG=./ansible.cfg
|
||||
|
||||
# ansible-playbook -i hosts.ini k3s-remote-control.yml
|
||||
49
manuals.md
49
manuals.md
@ -1,5 +1,18 @@
|
||||
### Getting Started
|
||||
#### Join new nodes
|
||||
```
|
||||
# retrieve token on control-plane node
|
||||
sudo cat /var/lib/rancher/k3s/server/node-token
|
||||
K10094c77f9748b705ce6fd128b916dd96793160853577fdcba986c57de9b6f4a3c::server:b5743ed19b035f78f59607b2fa3bda2e
|
||||
|
||||
# join via
|
||||
curl -sfL https://get.k3s.io | K3S_URL=https://<your-server-ip>:6443 K3S_TOKEN=<NodeToken> sh -
|
||||
|
||||
K3S_URL: The address of your control plane node
|
||||
|
||||
curl -sfL https://get.k3s.io | K3S_URL=https://192.168.3.91:6443 K3S_TOKEN=K10094c77f9748b705ce6fd128b916dd96793160853577fdcba986c57de9b6f4a3c::server:b5743ed19b035f78f59607b2fa3bda2e sh -
|
||||
|
||||
```
|
||||
#### 1. Using ansible
|
||||
```
|
||||
a. ansible-playbook -i hosts.ini ansible-prepare-k3s.yml
|
||||
@ -127,6 +140,18 @@ kubectl get secret postgres-ha-app -n db -o json | jq 'del(.metadata.namespace,.
|
||||
kubectl get secret postgres-ha-superuser -n db -o json | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid)' | kubectl apply -n infra -f -
|
||||
|
||||
```
|
||||
#### Avoiding sudo - Unable to read Error while executing kubectl command option
|
||||
```
|
||||
# Option 1: Change permissions (allows your user to read)
|
||||
sudo chmod 644 /etc/rancher/k3s/k3s.yaml
|
||||
|
||||
# Option 2: Copy the config to your user's directory
|
||||
mkdir -p ~/.kube
|
||||
sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
|
||||
sudo chown $(id -u):$(id -g) ~/.kube/config
|
||||
chmod 600 ~/.kube/config
|
||||
```
|
||||
|
||||
#### Debug
|
||||
```
|
||||
kubectl get all -A
|
||||
@ -177,12 +202,34 @@ sudo chmod 644 /etc/rancher/k3s/k3s.yaml
|
||||
Run in PS
|
||||
kustomize build . --enable-helm | kubectl apply --server-side --force-conflicts -f -
|
||||
|
||||
4. pods exist in PS but not in wsl
|
||||
4. pods exist in PS but not in wsl -- Join/Query from Laptop to k3s servers
|
||||
|
||||
# On WSL
|
||||
mkdir -p ~/.kube
|
||||
sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
|
||||
sudo chown $(id -u):$(id -g) ~/.kube/config
|
||||
export KUBECONFIG=~/.kube/config
|
||||
|
||||
change IP manually from 127.0.0.1 to 192.168.3.91
|
||||
|
||||
or
|
||||
|
||||
mkdir -p ~/.kube
|
||||
scp master-1@192.168.3.91:/etc/rancher/k3s/k3s.yaml ~/.kube/config
|
||||
sed -i 's/127.0.0.1/192.168.3.91/g' ~/.kube/config
|
||||
chmod 600 ~/.kube/config
|
||||
|
||||
# Testen
|
||||
kubectl get nodes
|
||||
|
||||
# on PS
|
||||
ssh master@192.168.3.156 "sudo cat /etc/rancher/k3s/k3s.yaml" > "$HOME\.kube\config"
|
||||
|
||||
# Localhost-IP durch die echte Master-IP ersetzen
|
||||
(Get-Content "$HOME\.kube\config") -replace '127.0.0.1', '192.168.3.156' | Set-Content "$HOME\.kube\config"
|
||||
|
||||
# Testen
|
||||
kubectl get nodes
|
||||
```
|
||||
|
||||
|
||||
|
||||
@ -729,4 +729,31 @@ Delete:
|
||||
httproute.yaml: This is for the "Gateway API." Unless you have a specific Gateway controller installed, standard ingress.yaml is what you'll use.
|
||||
|
||||
NOTES.txt: Usually just contains generic text.
|
||||
````
|
||||
```
|
||||
|
||||
#### What ToDo after deleting the primary node that bootstrapped the cluster.
|
||||
```
|
||||
ssh to other k3s control plane
|
||||
|
||||
sudo systemctl status k3s
|
||||
sudo systemctl stop k3s
|
||||
sudo k3s server --cluster-reset
|
||||
sudo systemctl start k3s
|
||||
sudo kubectl get nodes
|
||||
|
||||
sudo nano /etc/systemd/system/k3s.service
|
||||
|
||||
from
|
||||
ExecStart=/usr/local/bin/k3s server --server https://192.168.3.91:6443 --tls-san 192.168.3.156
|
||||
|
||||
to
|
||||
ExecStart=/usr/local/bin/k3s server --cluster-init --tls-san 192.168.3.156
|
||||
```
|
||||
|
||||
#### Delete stale nodes
|
||||
```
|
||||
sudo kubectl delete node invixel-vm1
|
||||
sudo kubectl delete node invixel-vm6
|
||||
sudo kubectl delete node invixel-vm7
|
||||
sudo kubectl delete node invixel-ubuntu1
|
||||
```
|
||||
@ -30,7 +30,7 @@ kubectl exec -it nats-box -n infra -- nats -s nats://nats-cluster.infra.svc.clus
|
||||
kubectl exec -it nats-box -n infra -- nats -s nats://nats-cluster:4222 stream info request_stream
|
||||
kubectl exec -it nats-cluster-0 -n infra -c nats -- netstat -tuln
|
||||
kubectl get pod nats-cluster-0 -n infra -o wide
|
||||
|
||||
kubectl get svc --all-namespaces
|
||||
kubectl get endpoints nats-cluster -n infra
|
||||
|
||||
|
||||
@ -132,6 +132,10 @@ kubectl run net-test --image=alpine --restart=Never -n infra -- nslookup google.
|
||||
kubectl exec -it yb-tserver-0 -n db -- ysqlsh -h
|
||||
kubectl port-forward svc/cosmo-router 3002:3002 -n apps
|
||||
kubectl port-forward svc/postgres-ha-rw 5436:5432 -n db
|
||||
|
||||
|
||||
# (Best Practice für Zero-Downtime): Evakuiere die alten Nodes zuerst manuell (Drain): Für die alten dbworker und nworker ausführen:
|
||||
kubectl drain <name-des-alten-nodes> --ignore-daemonsets --delete-emptydir-data
|
||||
```
|
||||
#### The Difference: NATS Core vs. JetStream
|
||||
- NATS Core (nats sub): This is "Fire and Forget." If no one is listening when Benthos sends the message, the message disappears forever.
|
||||
|
||||
@ -92,3 +92,832 @@ Reason to use: This is the most "production-ready" version of what you are doing
|
||||
|Ingress|Layer 7|A ""Receptionist"" directing visitors based on the name on the |envelope.|
|
||||
|LoadBalancer|Layer 4|"A ""Direct Pipeline"" to a specific door (port).|
|
||||
|Tailscale / VPN|Layer 3|"A ""Secret Tunnel"" that puts your whole house next to the office.|
|
||||
|
||||
#### Finding IP
|
||||
public: curl ifconfig.me
|
||||
|
||||
|Target|Command|Requirement|
|
||||
|----|----|-----|
|
||||
|Inside Container| hostname -I /ip addr show eth0|LXC Container|
|
||||
|LXC Local IP|pct exec ID ip a|Access to Proxmox Host|
|
||||
|Proxmox Local IP|hostname -I|Access to Proxmox Host|
|
||||
|Proxmox Public IP|curl ifconfig.me|Internet Access|
|
||||
|
||||
|
||||
#### Ipaymu Integration need static IP
|
||||
|
||||
[Cloudflare Worker] ──1. Request──► [Your Nginx Server] ──2. Request──► [iPaymu API]
|
||||
│ │
|
||||
[Cloudflare Worker] ◄─4. Response── [Your Nginx Server] ◄─3. Response─── [iPaymu API]
|
||||
Problem: we deploy using cloudflared worker which has dynamic ips
|
||||
|
||||
- Find IP of the server
|
||||
- curl -4 ifconfig.me
|
||||
- curl -4 icanhazip.com
|
||||
- curl -4 ipinfo.io/ip
|
||||
- ip -4 addr show
|
||||
|
||||
1. Option 1: Cloudflare Tunnel (cloudflared) — Easiest & Most Secure
|
||||
A. cloudflared inside an LXC Container (Separate from K3s)
|
||||
An LXC container on the same LAN/Proxmox host can run cloudflared and route incoming traffic directly to your K3s cluster.
|
||||
|
||||
[Cloudflare Worker]
|
||||
│ (Public HTTPS)
|
||||
▼
|
||||
[Cloudflare Edge Network]
|
||||
│ (Outbound Tunnel)
|
||||
[LXC Container running cloudflared]
|
||||
│ (Local LAN / Bridge Traffic)
|
||||
[K3s Cluster Node (Static IP)] ──► [Nginx Proxy Pod] ──► [iPaymu API]
|
||||
|
||||
How to configure the LXC Tunnel:
|
||||
When you set up the Public Hostname rule in the Cloudflare Zero Trust Dashboard (or config.yaml), point the origin URL directly to your K3s Node IP and port:
|
||||
|
||||
- Service Type: HTTP
|
||||
|
||||
- URL: <K3S_NODE_IP>:<NODE_PORT_OR_INGRESS_PORT>
|
||||
(e.g., [http://192.168.1.50:80](http://192.168.1.50:80) if calling Traefik/Ingress, or [http://192.168.1.50:30080](http://192.168.1.50:30080) if using a NodePort service).
|
||||
|
||||
Pros & Cons:
|
||||
- Pros: Keeps your K3s cluster lightweight; if K3s reboots, the tunnel daemon stays alive in the LXC container.
|
||||
- Cons: Requires exposing a NodePort or LoadBalancer (Traefik) on your K3s node so the LXC container can hit it on the local network.
|
||||
|
||||
B. cloudflared as a Pod inside K3s
|
||||
Deploying cloudflared directly inside K3s as a Deployment is the most common Kubernetes-native method.
|
||||
[Cloudflare Worker]
|
||||
│
|
||||
▼ (HTTPS over public internet)
|
||||
[https://proxy.yourdomain.com] <-- Cloudflare Edge
|
||||
│
|
||||
▼ (Secure outbound Tunnel)
|
||||
[cloudflared Pod in K3s / Swarm]
|
||||
│
|
||||
▼ (Internal Cluster Traffic)
|
||||
[Nginx Proxy Pod (Static-IP Node)]
|
||||
│
|
||||
▼ (Outbound call from Static IPv4)
|
||||
[iPaymu API]
|
||||
|
||||
2. Option 2: Traefik Ingress Controller + NodePort / Public IP
|
||||
If your static-IP cluster node already has a Public Static IPv4 Address directly bound to its network interface:
|
||||
|
||||
[Cloudflare Worker]
|
||||
│
|
||||
▼ (HTTPS to Public Static IP)
|
||||
[Public IP:443] ──► [Traefik Ingress] ──► [Nginx Proxy Service] ──► [iPaymu]
|
||||
|
||||
How to configure the K3s Tunnel:
|
||||
Inside the Cloudflare dashboard, you point the public hostname to K3s's internal DNS service name:
|
||||
|
||||
- Service Type: HTTP
|
||||
- URL: ipaymu-proxy-service.default.svc.cluster.local:80
|
||||
|
||||
Pros & Cons:
|
||||
- Pros: Highly secure—you do not need to open any ports or expose NodePort services on your LAN. Traffic moves entirely within K3s cluster networking.
|
||||
- Cons: Runs inside the Kubernetes runtime lifecycle.
|
||||
|
||||
3. Option 3: Port Mapping directly on the Static Node (hostPort)
|
||||
If you don't want to deal with Traefik or Ingress controllers, you can expose your Nginx proxy directly on a port (e.g., 8080 or 8443) of your static-IP node using hostPort in K3s or ports mapping in Swarm.
|
||||
|
||||
K3s Manifest Snippet:
|
||||
containers:
|
||||
- name: nginx
|
||||
image: nginx:alpine
|
||||
ports:
|
||||
- containerPort: 80
|
||||
hostPort: 8080 # Exposes port 8080 directly on the physical host
|
||||
|
||||
|
||||
Vour Cloudflare Worker calls http://<YOUR_STATIC_NODE_IP>:8080/ipaymu/api/v2/payment.
|
||||
|
||||
- How the Request Flow Works End-to-End
|
||||
Once exposed via Option 1, 2, or 3, here is the full round-trip execution path:
|
||||
```
|
||||
// 1. Cloudflare Worker executes this code:
|
||||
const response = await fetch("https://proxy.yourdomain.com/ipaymu/api/v2/payment", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"X-Proxy-Secret": "YOUR_INTERNAL_SECRET",
|
||||
"va": "1176000000000000",
|
||||
"signature": "CALCULATED_HMAC_SIGNATURE",
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
body: JSON.stringify(paymentPayload)
|
||||
});
|
||||
|
||||
// 2. Request reaches https://proxy.yourdomain.com (Your Cluster via Tunnel/Ingress)
|
||||
// 3. Ingress routes it internally to your Nginx proxy pod
|
||||
// 4. Nginx verifies "X-Proxy-Secret", then forwards the request to https://api.ipaymu.com
|
||||
// 5. iPaymu receives the call, sees your Node's STATIC IP, and accepts it
|
||||
// 6. iPaymu returns payment data -> Nginx -> Ingress -> Cloudflare Worker -> Client
|
||||
|
||||
```
|
||||
|
||||
|
||||
#### Step by step using ONLY Traefik in K3s
|
||||
```
|
||||
Step 1: Create Traefik Middlewares for Security and Stripping
|
||||
Instead of Nginx if ($http_x_proxy_secret) directives, you define Traefik Middleware resources:
|
||||
|
||||
# 1. Security Header Check Middleware
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: Middleware
|
||||
metadata:
|
||||
name: ipaymu-auth-check
|
||||
namespace: default
|
||||
spec:
|
||||
headers:
|
||||
customRequestHeaders:
|
||||
# Ensures requests forwarded to iPaymu clean up internal proxy secrets
|
||||
X-Proxy-Secret: ""
|
||||
---
|
||||
# 2. Path Strip Middleware (Removes /ipaymu prefix before forwarding)
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: Middleware
|
||||
metadata:
|
||||
name: ipaymu-strip-prefix
|
||||
namespace: default
|
||||
spec:
|
||||
stripPrefix:
|
||||
prefixes:
|
||||
- /ipaymu
|
||||
|
||||
Step 2: Define an ExternalName Service for iPaymu
|
||||
To tell Traefik to forward traffic out to iPaymu's external domain, create a Kubernetes ExternalName service:
|
||||
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ipaymu-external-api
|
||||
namespace: default
|
||||
spec:
|
||||
type: ExternalName
|
||||
externalName: api.ipaymu.com
|
||||
ports:
|
||||
- port: 443
|
||||
targetPort: 443
|
||||
protocol: TCP
|
||||
|
||||
Step 3: Route Traffic with Traefik IngressRoute
|
||||
Now, create a Traefik IngressRoute pinned to your static IP node:
|
||||
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: ipaymu-proxy-route
|
||||
namespace: default
|
||||
spec:
|
||||
entryPoints:
|
||||
- web
|
||||
- websecure
|
||||
routes:
|
||||
- match: Host(`proxy.yourdomain.com`) && PathPrefix(`/ipaymu`)
|
||||
kind: Rule
|
||||
middlewares:
|
||||
- name: ipaymu-strip-prefix
|
||||
services:
|
||||
- name: ipaymu-external-api
|
||||
port: 443
|
||||
scheme: https
|
||||
|
||||
When should you STILL use Nginx?
|
||||
While Traefik replaces Nginx for 90% of use cases, keeping a tiny Nginx pod is still useful if:
|
||||
|
||||
Complex Logic: You want custom Nginx Lua scripts or complex header manipulation (e.g., computing dynamic HMAC signatures or custom retry logic on upstream failures).
|
||||
|
||||
Familiarity: You are already comfortable writing .conf files and don't want to learn Traefik-specific Kubernetes CRD syntax (IngressRoute / Middleware).
|
||||
```
|
||||
|
||||
#### Step by step using traeffik and nginx in k3s
|
||||
```
|
||||
1. Label your target node (Crucial Step)
|
||||
Label the K3s node that holds your static public IPv4 address so K3s knows where to schedule the proxy pods:
|
||||
|
||||
kubectl label nodes <your-static-node-name> egress-ip=static-ipaymu
|
||||
|
||||
2. k3s-ipaymu-proxy.yaml
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: ipaymu-proxy-config
|
||||
namespace: default
|
||||
data:
|
||||
default.conf: |
|
||||
upstream ipaymu_backend {
|
||||
server api.ipaymu.com:443;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
|
||||
location /ipaymu/ {
|
||||
if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET_KEY") {
|
||||
return 403;
|
||||
}
|
||||
|
||||
proxy_pass https://ipaymu_backend/;
|
||||
proxy_set_header Host api.ipaymu.com;
|
||||
|
||||
# SSL Handshake settings
|
||||
proxy_ssl_server_name on;
|
||||
proxy_ssl_protocols TLSv1.2 TLSv1.3;
|
||||
|
||||
# HTTP Keepalive
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
}
|
||||
}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: ipaymu-proxy
|
||||
namespace: default
|
||||
spec:
|
||||
replicas: 2 # Scale up replicas as needed
|
||||
selector:
|
||||
matchLabels:
|
||||
app: ipaymu-proxy
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ipaymu-proxy
|
||||
spec:
|
||||
# Pin all replicas to the node with the static public IP
|
||||
nodeSelector:
|
||||
egress-ip: static-ipaymu
|
||||
containers:
|
||||
- name: nginx
|
||||
image: nginx:alpine
|
||||
ports:
|
||||
- containerPort: 80
|
||||
resources:
|
||||
limits:
|
||||
cpu: "200m"
|
||||
memory: "128Mi"
|
||||
requests:
|
||||
cpu: "50m"
|
||||
memory: "32Mi"
|
||||
volumeMounts:
|
||||
- name: nginx-config
|
||||
mountPath: /etc/nginx/conf.d/default.conf
|
||||
subPath: default.conf
|
||||
volumes:
|
||||
- name: nginx-config
|
||||
configMap:
|
||||
name: ipaymu-proxy-config
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ipaymu-proxy-service
|
||||
namespace: default
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 80
|
||||
protocol: TCP
|
||||
selector:
|
||||
app: ipaymu-proxy
|
||||
|
||||
3. Deploy : kubectl apply -f k3s-ipaymu-proxy.yaml
|
||||
4. How to call it inside K3s
|
||||
Any other pod inside your K3s cluster can now make payment requests directly using internal Kubernetes DNS:
|
||||
[http://ipaymu-proxy-service.default.svc.cluster.local/ipaymu/api/v2/payment](http://ipaymu-proxy-service.default.svc.cluster.local/ipaymu/api/v2/payment)
|
||||
```
|
||||
|
||||
#### Step by step using traeffik and cloudflared in k3s
|
||||
```
|
||||
Step 2.1: Enable allowExternalNameServices in Traefik
|
||||
By default, K3s disables Traefik from routing to external domains via ExternalName. Create a HelmChartConfig manifest on your K3s server node to enable it:
|
||||
|
||||
Create file /var/lib/rancher/k3s/server/manifests/traefik-config.yaml:
|
||||
|
||||
apiVersion: helm.cattle.io/v1
|
||||
kind: HelmChartConfig
|
||||
metadata:
|
||||
name: traefik
|
||||
namespace: kube-system
|
||||
spec:
|
||||
valuesContent: |-
|
||||
providers:
|
||||
kubernetesCRD:
|
||||
allowExternalNameServices: true
|
||||
|
||||
K3s automatically applies this change in under 30 seconds.
|
||||
|
||||
Step 2.2: Label your Static-IP Node
|
||||
Label the node with your static public IPv4 address so K3s knows where to schedule the tunnel:
|
||||
|
||||
kubectl label nodes <your-node-name> egress-ip=static-ipaymu
|
||||
|
||||
Step 2.3: Deploy the iPaymu Traefik Route
|
||||
Save this to ipaymu-traefik.yaml and run kubectl apply -f ipaymu-traefik.yaml:
|
||||
|
||||
# 1. External Name Service pointing to iPaymu
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ipaymu-external-service
|
||||
namespace: default
|
||||
spec:
|
||||
type: ExternalName
|
||||
externalName: api.ipaymu.com
|
||||
ports:
|
||||
- name: https
|
||||
port: 443
|
||||
targetPort: 443
|
||||
protocol: TCP
|
||||
---
|
||||
# 2. TLS Transport for SNI support
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: ServersTransport
|
||||
metadata:
|
||||
name: ipaymu-transport
|
||||
namespace: default
|
||||
spec:
|
||||
serverName: api.ipaymu.com
|
||||
---
|
||||
# 3. Path Stripper Middleware (/ipaymu/api/v2/payment -> /api/v2/payment)
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: Middleware
|
||||
metadata:
|
||||
name: ipaymu-strip-prefix
|
||||
namespace: default
|
||||
spec:
|
||||
stripPrefix:
|
||||
prefixes:
|
||||
- /ipaymu
|
||||
---
|
||||
# 4. Traefik IngressRoute
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: ipaymu-proxy-route
|
||||
namespace: default
|
||||
spec:
|
||||
entryPoints:
|
||||
- web
|
||||
- websecure
|
||||
routes:
|
||||
- match: Host(`proxy.yourdomain.com`) && PathPrefix(`/ipaymu`)
|
||||
kind: Rule
|
||||
middlewares:
|
||||
- name: ipaymu-strip-prefix
|
||||
services:
|
||||
- name: ipaymu-external-service
|
||||
port: 443
|
||||
scheme: https
|
||||
serversTransport: ipaymu-transport
|
||||
|
||||
Step 2.4:
|
||||
- Get Your Tunnel Secret Token
|
||||
1. Log into your Cloudflare Zero Trust Dashboard.
|
||||
2. Navigate to Networks → Tunnels → Create a Tunnel.
|
||||
3. Choose Cloudflared as the connector and name it (e.g., k3s-ipaymu-tunnel).
|
||||
4. On the installation page, select Docker/Kubernetes. Look for the command and copy the long base64 token string after --token.
|
||||
|
||||
- Deploy Cloudflare Tunnel inside K3s
|
||||
Save this to cloudflared.yaml (replace YOUR_TOKEN_HERE with your Cloudflare Zero Trust tunnel token) and run kubectl apply -f cloudflared.yaml:
|
||||
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: cloudflared-token
|
||||
namespace: default
|
||||
type: Opaque
|
||||
stringData:
|
||||
TUNNEL_TOKEN: "YOUR_TOKEN_HERE"
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: cloudflared
|
||||
namespace: default
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: cloudflared
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: cloudflared
|
||||
spec:
|
||||
nodeSelector:
|
||||
egress-ip: static-ipaymu
|
||||
containers:
|
||||
- name: cloudflared
|
||||
image: cloudflare/cloudflared:latest
|
||||
args:
|
||||
- tunnel
|
||||
- --no-autoupdate
|
||||
- run
|
||||
env:
|
||||
- name: TUNNEL_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: cloudflared-token
|
||||
key: TUNNEL_TOKEN
|
||||
|
||||
|
||||
kubectl apply -f cloudflared.yaml
|
||||
|
||||
Step 2.5: Cloudflare Dashboard Routing
|
||||
In Cloudflare Zero Trust Dashboard (→ Networks → Tunnels → Public Hostname):
|
||||
|
||||
Public Hostname: proxy.yourdomain.com
|
||||
|
||||
Path: /ipaymu
|
||||
|
||||
Type: HTTP
|
||||
|
||||
URL: traefik.kube-system.svc.cluster.local:80
|
||||
```
|
||||
#### Step by step using nginx and cloudflared in docker-compose
|
||||
```
|
||||
Step 3.1: Create Directory Structure
|
||||
On your target server, create a project directory:
|
||||
mkdir -p ipaymu-proxy
|
||||
cd ipaymu-proxy
|
||||
|
||||
ipaymu-proxy/
|
||||
├── docker-compose.yml
|
||||
└── nginx.conf
|
||||
|
||||
Step 3.2: Create nginx.conf
|
||||
events { worker_connections 1024; }
|
||||
|
||||
http {
|
||||
upstream ipaymu_api {
|
||||
server api.ipaymu.com:443;
|
||||
keepalive 16;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
location /ipaymu/ {
|
||||
# Check secret key header sent by your Cloudflare Worker
|
||||
if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET_KEY") {
|
||||
return 403;
|
||||
}
|
||||
|
||||
proxy_pass https://ipaymu_api/;
|
||||
proxy_set_header Host api.ipaymu.com;
|
||||
|
||||
# Ensure TLS handshake works with iPaymu
|
||||
proxy_ssl_server_name on;
|
||||
proxy_ssl_protocols TLSv1.2 TLSv1.3;
|
||||
|
||||
# HTTP Keepalive optimizations
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
Step 3.3: Create docker-compose.yml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
ipaymu-proxy:
|
||||
image: nginx:alpine
|
||||
container_name: ipaymu_proxy
|
||||
restart: always
|
||||
ports:
|
||||
- "80:80" # Or expose via reverse proxy / Cloudflare tunnel
|
||||
volumes:
|
||||
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
|
||||
healthcheck:
|
||||
test: ["CMD", "nginx", "-t"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
or with nginx:
|
||||
|
||||
#docker-compose.yml
|
||||
version: '3.8'
|
||||
services:
|
||||
ipaymu-nginx:
|
||||
image: nginx:alpine
|
||||
container_name: ipaymu_nginx
|
||||
restart: always
|
||||
volumes:
|
||||
- ./nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
networks:
|
||||
- proxy-net
|
||||
|
||||
cloudflared:
|
||||
image: cloudflare/cloudflared:latest
|
||||
container_name: ipaymu_tunnel
|
||||
restart: always
|
||||
command: tunnel --no-autoupdate run
|
||||
environment:
|
||||
- TUNNEL_TOKEN=YOUR_CLOUDFLARE_TUNNEL_TOKEN_HERE
|
||||
networks:
|
||||
- proxy-net
|
||||
|
||||
networks:
|
||||
proxy-net:
|
||||
driver: bridge
|
||||
|
||||
Step 3.4: Deploy Docker Compose: docker compose up -d
|
||||
Step 3.5: Cloudflare Dashboard RoutingIn Cloudflare Zero Trust Dashboard ($\rightarrow$ Networks $\rightarrow$ Tunnels $\rightarrow$ Public Hostname):
|
||||
Public Hostname: proxy.yourdomain.com
|
||||
Path: /ipaymuType: HTTP
|
||||
URL: ipaymu_nginx:80 (Connects directly using Docker's internal container DNS).
|
||||
|
||||
```
|
||||
|
||||
#### Step by step using docker swarm
|
||||
|
||||
```
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
ipaymu-proxy:
|
||||
image: nginx:alpine
|
||||
ports:
|
||||
- "80:80"
|
||||
- "443:443"
|
||||
configs:
|
||||
- source: nginx_config
|
||||
target: /etc/nginx/conf.d/default.conf
|
||||
deploy:
|
||||
mode: replicated
|
||||
replicas: 1
|
||||
placement:
|
||||
constraints:
|
||||
# Pin execution to the specific node with the static public IP
|
||||
- node.hostname == node-with-static-ip
|
||||
|
||||
configs:
|
||||
nginx_config:
|
||||
inline: |
|
||||
server {
|
||||
listen 80;
|
||||
server_name ipaymu-proxy.yourdomain.com;
|
||||
|
||||
location /ipaymu/ {
|
||||
if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET") {
|
||||
return 403;
|
||||
}
|
||||
|
||||
proxy_pass https://api.ipaymu.com/;
|
||||
proxy_set_header Host api.ipaymu.com;
|
||||
proxy_ssl_server_name on;
|
||||
|
||||
proxy_ssl_protocols TLSv1.2 TLSv1.3;
|
||||
|
||||
# Pass client details
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
}
|
||||
|
||||
[Cloudflare Worker]
|
||||
│
|
||||
├─── /ipaymu/v2/payment ─────────► [Nginx] ──► https://my.ipaymu.com/api/v2/payment
|
||||
│
|
||||
└─── /ipaymu-sandbox/v2/payment ─► [Nginx] ──► https://sandbox.ipaymu.com/api/v2/payment
|
||||
|
||||
1.a Create the dedicated folder in /opt/
|
||||
mkdir -p /opt/ipaymu-swarm-proxy
|
||||
cd /opt/ipaymu-swarm-proxy
|
||||
|
||||
|
||||
b create the nginx configuration file nginx.conf:
|
||||
```
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
# Upstream targets using persistent HTTP keepalives
|
||||
upstream ipaymu_live {
|
||||
server my.ipaymu.com:443;
|
||||
keepalive 16;
|
||||
}
|
||||
|
||||
upstream ipaymu_sandbox {
|
||||
server sandbox.ipaymu.com:443;
|
||||
keepalive 16;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
# -------------------------------------------------------------
|
||||
# 1. Production / Live Endpoint
|
||||
# -------------------------------------------------------------
|
||||
location /ipaymu/v2/ {
|
||||
# Security Header Check
|
||||
if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET_KEY") {
|
||||
return 403;
|
||||
}
|
||||
|
||||
proxy_pass https://ipaymu_live/api/v2/;
|
||||
proxy_set_header Host my.ipaymu.com;
|
||||
|
||||
# Ensure TLS/SNI Handshake matches my.ipaymu.com
|
||||
proxy_ssl_server_name on;
|
||||
proxy_ssl_name my.ipaymu.com;
|
||||
proxy_ssl_protocols TLSv1.2 TLSv1.3;
|
||||
|
||||
# Connection optimizations
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_pass_request_headers on;
|
||||
}
|
||||
|
||||
# -------------------------------------------------------------
|
||||
# 2. Development / Sandbox Endpoint
|
||||
# -------------------------------------------------------------
|
||||
location /ipaymu-sandbox/v2/ {
|
||||
# Security Header Check
|
||||
if ($http_x_proxy_secret != "YOUR_INTERNAL_SECRET_KEY") {
|
||||
return 403;
|
||||
}
|
||||
|
||||
proxy_pass https://ipaymu_sandbox/api/v2/;
|
||||
proxy_set_header Host sandbox.ipaymu.com;
|
||||
|
||||
# Ensure TLS/SNI Handshake matches sandbox.ipaymu.com
|
||||
proxy_ssl_server_name on;
|
||||
proxy_ssl_name sandbox.ipaymu.com;
|
||||
proxy_ssl_protocols TLSv1.2 TLSv1.3;
|
||||
|
||||
# Connection optimizations
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_pass_request_headers on;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
2. Change ipaymu
|
||||
```
|
||||
# erstelle einen starker schlüssel YOUR_INTERNAL_SECRET_KEY:
|
||||
openssl rand -hex 32
|
||||
|
||||
# Setze diesen Schlüssel in deinen Nginx-Code (nginx.conf):
|
||||
if ($http_x_proxy_secret != "0fb1d9712f16bb64e0dff4f911af02e8a2280722c80097a845e248c7a49781d7") {
|
||||
return 403;
|
||||
}
|
||||
|
||||
Setze denselben Schlüssel in deinen Cloudflare Worker / App-Code:
|
||||
|
||||
const headers = {
|
||||
"Content-Type": "application/json",
|
||||
"X-Proxy-Secret": "0fb1d9712f16bb64e0dff4f911af02e8a2280722c80097a845e248c7a49781d7", // Gleicher Schlüssel!
|
||||
// ...
|
||||
};
|
||||
|
||||
|
||||
CONTAINER_ID=$(docker ps -q -f name=ipaymu_nginx-proxy | head -n1)
|
||||
docker exec -it $CONTAINER_ID apk add --no-cache curl
|
||||
docker exec -it $CONTAINER_ID curl -4 ifconfig.me
|
||||
|
||||
curl -i -X POST https://proxy.yourdomain.com/ipaymu/v2/payment \
|
||||
-H "X-Proxy-Secret: YOUR_INTERNAL_SECRET_KEY" \
|
||||
-H "Content-Type: application/json"
|
||||
```
|
||||
1. Das Label auf den gewählten Node setzen
|
||||
Sobald du weißt, welcher Node die statische IP hat (z. B. node-01), vergibst du das Label egress=static-ipaymu:
|
||||
```
|
||||
docker node update --label-add egress=static-ipaymu node-01
|
||||
|
||||
docker node update --label-add egress=static-ipaymu <node-name-or-id>
|
||||
docker node inspect node-01 --format '{{ json .Spec.Labels }}'
|
||||
```
|
||||
|
||||
4. Deploy And check
|
||||
docker run cloudflare/cloudflared:latest tunnel --no-autoupdate run --token eyJhIjoiNjFhMTZkYzg3MmM4ZjlmZTQ4Mzc0ZWQ5NzVmNTcwNmIiLCJ0IjoiNTA5ZmRlZjQtZjEzYy00ZTQ1LTllZDQtYjZhMjExOTFlY2Y3IiwicyI6Ik5UWmtZVFU1T1dZdE5XVmtZUzAwTldZeExXSm1Oek10TlRRd1pqUTBNR1kxWW1RMSJ9
|
||||
|
||||
- Option 1 Using Docker Swarm env ( does not work )
|
||||
```
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
nginx-proxy:
|
||||
image: nginx:alpine
|
||||
configs:
|
||||
- source: nginx_config_v1
|
||||
target: /etc/nginx/nginx.conf
|
||||
networks:
|
||||
- payment-network
|
||||
deploy:
|
||||
mode: replicated
|
||||
replicas: 2
|
||||
placement:
|
||||
constraints:
|
||||
# Pin to the node holding your static IP for iPaymu whitelisting
|
||||
- node.labels.egress == static-ipaymu
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
|
||||
cloudflared:
|
||||
image: cloudflare/cloudflared:latest
|
||||
command: tunnel --no-autoupdate run
|
||||
environment:
|
||||
- TUNNEL_TOKEN=${CLOUDFLARE_TUNNEL_TOKEN}
|
||||
networks:
|
||||
- payment-network
|
||||
deploy:
|
||||
mode: replicated
|
||||
replicas: 2
|
||||
placement:
|
||||
constraints:
|
||||
- node.labels.egress == static-ipaymu
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
|
||||
configs:
|
||||
nginx_config_v1:
|
||||
file: ./nginx.conf
|
||||
|
||||
networks:
|
||||
payment-network:
|
||||
driver: overlay
|
||||
attachable: true
|
||||
```
|
||||
create .env file in the same directory:
|
||||
sudo nano /opt/ipaymu-swarm-proxy/.env
|
||||
CLOUDFLARE_TUNNEL_TOKEN=eyJhIjoiNjFhMTZkYzg3MmM4ZjlmZTQ4Mzc0ZWQ5NzVmNTcwNmIiLCJ0IjoiNTA5ZmRlZjQtZjEzYy00ZTQ1LTllZDQtYjZhMjExOTFlY2Y3IiwicyI6Ik5UWmtZVFU1T1dZdE5XVmtZUzAwTldZeExXSm1Oek10TlRRd1pqUTBNR1kxWW1RMSJ9
|
||||
|
||||
docker run cloudflare/cloudflared:latest tunnel --no-autoupdate run --token eyJhIjoiNjFhMTZkYzg3MmM4ZjlmZTQ4Mzc0ZWQ5NzVmNTcwNmIiLCJ0IjoiNTA5ZmRlZjQtZjEzYy00ZTQ1LTllZDQtYjZhMjExOTFlY2Y3IiwicyI6Ik5UWmtZVFU1T1dZdE5XVmtZUzAwTldZeExXSm1Oek10TlRRd1pqUTBNR1kxWW1RMSJ9
|
||||
|
||||
docker stack deploy -c ipaymu-swarm.yml ipaymu
|
||||
or
|
||||
export $(cat /opt/ipaymu-swarm-proxy/.env | xargs) && sudo -E docker stack deploy -c ipaymu-swarm.yml ipaymu
|
||||
|
||||
or
|
||||
|
||||
set -a && source /opt/ipaymu-swarm-proxy/.env && set +a && sudo -E docker stack deploy -c ipaymu-swarm.yml ipaymu
|
||||
|
||||
or
|
||||
|
||||
CLOUDFLARE_TUNNEL_TOKEN="eyJhIjoiNjFhMTZkYzg3MmM4ZjlmZTQ4Mzc0ZWQ5NzVmNTcwNmIiLCJ0IjoiNTA5ZmRlZjQtZjEzYy00ZTQ1LTllZDQtYjZhMjExOTFlY2Y3IiwicyI6Ik5UWmtZVFU1T1dZdE5XVmtZUzAwTldZeExXSm1Oek10TlRRd1pqUTBNR1kxWW1RMSJ9" sudo -E docker stack deploy -c ipaymu-swarm.yml ipaymu
|
||||
|
||||
# Option 2 Docker Swarm Secret
|
||||
Create a Docker Secret on your Manager node:
|
||||
printf "your_actual_token_here" | docker secret create cloudflared_tunnel_token -
|
||||
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
cloudflared:
|
||||
image: cloudflare/cloudflared:latest
|
||||
# Tells cloudflared to read the token file created by Docker Secrets
|
||||
command: tunnel --no-autoupdate run --token-file /run/secrets/cloudflared_token
|
||||
secrets:
|
||||
- source: cloudflared_tunnel_token
|
||||
target: cloudflared_token
|
||||
networks:
|
||||
- ipaymu-overlay
|
||||
deploy:
|
||||
mode: replicated
|
||||
replicas: 2
|
||||
placement:
|
||||
constraints:
|
||||
- node.labels.egress == static-ipaymu
|
||||
|
||||
secrets:
|
||||
cloudflared_tunnel_token:
|
||||
external: true
|
||||
|
||||
Summary Recommendation
|
||||
For simple setups: Use Method 1 (.env file).
|
||||
For strict security requirements: Use Method 4 (Docker Secrets).
|
||||
|
||||
docker stack services ipaymu
|
||||
docker service logs ipaymu_cloudflared --tail 20
|
||||
|
||||
Now you need to label the node like in docker-stack.yml:
|
||||
docker node update --label-add egress=static-ipaymu <node-hostname-or-id>
|
||||
|
||||
docker node update --label-add egress=static-ipaymu invixel-debian1
|
||||
docker node update --label-add egress=static-ipaymu invixel-vm5
|
||||
|
||||
How to verify the exit IP
|
||||
After deploying to K3s or Swarm, exec into your Nginx container/pod and test what public IP iPaymu will see:
|
||||
|
||||
docker exec -it <container_id> curl -4 ifconfig.me
|
||||
|
||||
kubectl exec -it deployment/ipaymu-proxy -- curl -4 ifconfig.me
|
||||
|
||||
|
||||
curl -i -X POST https://bali-car-dev.easy-rent.com/ipaymu/v2/payment \
|
||||
-H "X-Proxy-Secret: 0fb1d9712f16bb64e0dff4f911af02e8a2280722c80097a845e248c7a49781d7" \
|
||||
-H "Content-Type: application/json"
|
||||
```
|
||||
Loading…
x
Reference in New Issue
Block a user