# templates/sql_ingest.yaml name: yugabyte_insert type: output fields: - name: table_name type: string - name: dsn type: string - name: columns type: unknown - name: value_mapping type: string - name: conflict_columns # NEW: Make it flexible! type: string default: "content_hash" mapping: | # root.processors = [ # # LAYER 2: Local Bloom-style Gate (RAM) # { # "dedupe": { # "cache": "local_bloom_gate", # # "key": "${! content().hash(\"sha256\").encode(\"hex\") }", # # without concatenating with this.table name inserting multiple tables not possible # "key": "${! meta(\"conflict_col\") }-" + this.table_name , # "drop_on_err": false # } # }, # # VISIBILITY: Log what survived the Bloom Filter # { # "log": { # "level": "INFO", # "message": "Passed Bloom Filter: transaction ${! meta(\"conflict_col\") }" # } # }, # # LAYER 3 Redis Global Gate (Network) # { # "dedupe": { # "cache": "dedupe_cache", # # "key": "${! content().hash(\"sha256\").encode(\"hex\") }", # "key": "${! meta(\"conflict_col\") }-" + this.table_name , # "drop_on_err": false # } # }, # # NEW: Add Global Logging/Error Handling for ALL inserts using this template # { # "catch": [ # { # "log": { # "level": "ERROR", # "message": "SQL Error on " + this.table_name + ": ${! error() }" # } # } # ] # }, # # B. Success Logging # { # "log": { # "level": "INFO", # "message": "Attempting insert for " + this.table_name + " ID: ${! json(\"data.order.id\") }" # } # } # ] root.sql_insert = { "driver": "postgres", "dsn": this.dsn, "table": this.table_name, "columns": this.columns, # FIX: We use 'this.value_mapping' directly so the stream's logic is used "args_mapping": this.value_mapping, "suffix": "ON CONFLICT ( "+ this.conflict_columns + ") DO NOTHING" #"suffix": "ON CONFLICT (content_hash) DO NOTHING" }