77 lines
2.1 KiB
YAML
77 lines
2.1 KiB
YAML
# templates/sql_ingest.yaml
|
|
name: yugabyte_insert
|
|
type: output
|
|
|
|
fields:
|
|
- name: table_name
|
|
type: string
|
|
- name: dsn
|
|
type: string
|
|
- name: columns
|
|
type: unknown
|
|
- name: value_mapping
|
|
type: string
|
|
- name: conflict_columns # NEW: Make it flexible!
|
|
type: string
|
|
default: "content_hash"
|
|
|
|
mapping: |
|
|
# root.processors = [
|
|
# # LAYER 2: Local Bloom-style Gate (RAM)
|
|
# {
|
|
# "dedupe": {
|
|
# "cache": "local_bloom_gate",
|
|
# # "key": "${! content().hash(\"sha256\").encode(\"hex\") }",
|
|
# # without concatenating with this.table name inserting multiple tables not possible
|
|
# "key": "${! meta(\"conflict_col\") }-" + this.table_name ,
|
|
# "drop_on_err": false
|
|
# }
|
|
# },
|
|
# # VISIBILITY: Log what survived the Bloom Filter
|
|
# {
|
|
# "log": {
|
|
# "level": "INFO",
|
|
# "message": "Passed Bloom Filter: transaction ${! meta(\"conflict_col\") }"
|
|
# }
|
|
# },
|
|
# # LAYER 3 Redis Global Gate (Network)
|
|
# {
|
|
# "dedupe": {
|
|
# "cache": "dedupe_cache",
|
|
# # "key": "${! content().hash(\"sha256\").encode(\"hex\") }",
|
|
# "key": "${! meta(\"conflict_col\") }-" + this.table_name ,
|
|
# "drop_on_err": false
|
|
# }
|
|
# },
|
|
# # NEW: Add Global Logging/Error Handling for ALL inserts using this template
|
|
# {
|
|
# "catch": [
|
|
# {
|
|
# "log": {
|
|
# "level": "ERROR",
|
|
# "message": "SQL Error on " + this.table_name + ": ${! error() }"
|
|
# }
|
|
# }
|
|
# ]
|
|
# },
|
|
# # B. Success Logging
|
|
# {
|
|
# "log": {
|
|
# "level": "INFO",
|
|
# "message": "Attempting insert for " + this.table_name + " ID: ${! json(\"data.order.id\") }"
|
|
# }
|
|
# }
|
|
|
|
# ]
|
|
|
|
root.sql_insert = {
|
|
"driver": "postgres",
|
|
"dsn": this.dsn,
|
|
"table": this.table_name,
|
|
"columns": this.columns,
|
|
# FIX: We use 'this.value_mapping' directly so the stream's logic is used
|
|
"args_mapping": this.value_mapping,
|
|
"suffix": "ON CONFLICT ( "+ this.conflict_columns + ") DO NOTHING"
|
|
#"suffix": "ON CONFLICT (content_hash) DO NOTHING"
|
|
}
|