Sanitize NUL bytes in operation and effect details before jsonb insert

internal/db2/history/details.go

@@ -0,0 +1,83 @@
+package history
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+)
+
+// jsonNullEscape is how encoding/json renders a NUL character (U+0000): the six
+// ASCII bytes backslash, 'u', '0', '0', '0', '0'. Postgres jsonb cannot store a
+// NUL, because it decodes escapes into text and text cannot hold one (a plain
+// json column would accept it), so a marshaled document containing one cannot be
+// written to a jsonb column as-is. It is used only as a cheap pre-check.
+var jsonNullEscape = []byte{0x5c, 'u', '0', '0', '0', '0'}
+
+// sanitizeJSONBDetails makes a marshaled JSON document safe to store in a jsonb
+// column by removing NUL characters from every string it contains.
+//
+// Strings derived from ledger data are not guaranteed to be free of bytes that
+// jsonb cannot represent: asset codes are opaque 4/12-byte arrays and Soroban
+// ScVal strings are arbitrary bytes. A NUL would otherwise cause the insert to
+// fail.
+//
+// The common case — no NUL — returns the input untouched. Otherwise the document
+// is decoded, NUL is stripped from its string values (and any object keys), and
+// it is re-encoded, leaving all JSON parsing and escaping to encoding/json
+// rather than editing the serialized bytes by hand. UseNumber keeps numeric
+// values exact across the round trip.
+func sanitizeJSONBDetails(details []byte) []byte {
+	if !bytes.Contains(details, jsonNullEscape) {
+		return details
+	}
+
+	decoder := json.NewDecoder(bytes.NewReader(details))
+	decoder.UseNumber()
+	var decoded interface{}
+	if err := decoder.Decode(&decoded); err != nil {
+		return details
+	}
+
+	sanitized, err := json.Marshal(stripNULFromJSON(decoded))
+	if err != nil {
+		return details
+	}
+	return sanitized
+}
+
+// stripNULFromJSON removes NUL characters from every string within a value
+// decoded from JSON. Such a value only ever holds string, json.Number, bool,
+// nil, []interface{} and map[string]interface{} — never a struct — so recursion
+// over these cases reaches every string.
+func stripNULFromJSON(value interface{}) interface{} {
+	switch v := value.(type) {
+	case string:
+		return stripNUL(v)
+	case []interface{}:
+		for i := range v {
+			v[i] = stripNULFromJSON(v[i])
+		}
+		return v
+	case map[string]interface{}:
+		sanitized := make(map[string]interface{}, len(v))
+		for key, val := range v {
+			sanitized[stripNUL(key)] = stripNULFromJSON(val)
+		}
+		return sanitized
+	default:
+		return v
+	}
+}
+
+// stripNUL removes every NUL character (U+0000) from s.
+func stripNUL(s string) string {
+	if !strings.ContainsRune(s, 0) {
+		return s
+	}
+	return strings.Map(func(r rune) rune {
+		if r == 0 {
+			return -1
+		}
+		return r
+	}, s)
+}

internal/db2/history/details_test.go

@@ -0,0 +1,176 @@
+package history
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/guregu/null"
+
+	"github.com/stellar/go-stellar-sdk/toid"
+	"github.com/stellar/go-stellar-sdk/xdr"
+	"github.com/stellar/stellar-horizon/internal/db2"
+	"github.com/stellar/stellar-horizon/internal/test"
+)
+
+const nulByteTestIssuer = "GA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVSGZ"
+
+// literalNulEscape is the six literal characters backslash, 'u', '0', '0', '0',
+// '0' — the text that spells a NUL escape, but is NOT a NUL. It is built from
+// bytes to avoid any real escape sequence in the source.
+var literalNulEscape = string([]byte{0x5c, 'u', '0', '0', '0', '0'})
+
+// nulByteTestDetails returns marshaled details whose asset code contains an
+// interior NUL — a value that can occur in ledger-derived data. encoding/json
+// renders the NUL as jsonNullEscape, which a jsonb column cannot store, so this
+// exercises the sanitizeJSONBDetails path. sanitizedAsset is what the asset must
+// read as once the NUL has been stripped.
+func nulByteTestDetails(t *testing.T) (marshaled []byte, sanitizedAsset string) {
+	t.Helper()
+	poisonedCode := string([]byte{'A', 0x00, 'B'}) // 'A' NUL 'B'
+	details, err := json.Marshal(map[string]string{
+		"from":  "asset",
+		"asset": poisonedCode + ":" + nulByteTestIssuer,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Contains(details, jsonNullEscape) {
+		t.Fatalf("expected marshaled details to contain the NUL escape, got %s", details)
+	}
+	return details, "AB:" + nulByteTestIssuer
+}
+
+func TestSanitizeJSONBDetails(t *testing.T) {
+	mustMarshal := func(v map[string]string) []byte {
+		b, err := json.Marshal(v)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return b
+	}
+
+	poisoned, wantAsset := nulByteTestDetails(t)
+
+	got := sanitizeJSONBDetails(poisoned)
+
+	if bytes.Contains(got, jsonNullEscape) {
+		t.Errorf("sanitized details still contain the NUL escape: %s", got)
+	}
+	if strings.ContainsRune(string(got), rune(0)) {
+		t.Errorf("sanitized details still contain a raw NUL: %q", string(got))
+	}
+	if !json.Valid(got) {
+		t.Errorf("sanitized details are not valid JSON: %s", got)
+	}
+	var out map[string]string
+	if err := json.Unmarshal(got, &out); err != nil {
+		t.Fatalf("cannot unmarshal sanitized details: %v", err)
+	}
+	if out["asset"] != wantAsset {
+		t.Errorf("asset = %q, want %q", out["asset"], wantAsset)
+	}
+
+	// A document with no NUL escape must be returned unchanged.
+	clean := []byte(`{"asset": "AB:` + nulByteTestIssuer + `", "from": "asset"}`)
+	if got := sanitizeJSONBDetails(clean); !bytes.Equal(got, clean) {
+		t.Errorf("clean input was modified: got %s want %s", got, clean)
+	}
+
+	// A value that legitimately contains the literal escape text (a backslash
+	// followed by u0000, not a NUL) marshals with a doubled leading backslash. It
+	// must be preserved, not mistaken for a NUL escape and truncated into invalid
+	// JSON.
+	withLiteral := mustMarshal(map[string]string{"asset": literalNulEscape})
+	sanitized := sanitizeJSONBDetails(withLiteral)
+	if !json.Valid(sanitized) {
+		t.Errorf("literal-escape input produced invalid JSON: %s", sanitized)
+	}
+	var lit map[string]string
+	if err := json.Unmarshal(sanitized, &lit); err != nil {
+		t.Fatalf("cannot unmarshal sanitized literal-escape details: %v", err)
+	}
+	if lit["asset"] != literalNulEscape {
+		t.Errorf("literal escape value corrupted: got %q want %q", lit["asset"], literalNulEscape)
+	}
+
+	// A value with both a real NUL and the literal escape text: only the NUL is
+	// removed, the literal text survives.
+	mixed := mustMarshal(map[string]string{
+		"asset": string([]byte{'A', 0x00}) + literalNulEscape,
+	})
+	sanitizedMixed := sanitizeJSONBDetails(mixed)
+	if !json.Valid(sanitizedMixed) {
+		t.Errorf("mixed input produced invalid JSON: %s", sanitizedMixed)
+	}
+	var mix map[string]string
+	if err := json.Unmarshal(sanitizedMixed, &mix); err != nil {
+		t.Fatalf("cannot unmarshal sanitized mixed details: %v", err)
+	}
+	if want := "A" + literalNulEscape; mix["asset"] != want {
+		t.Errorf("mixed value: got %q want %q", mix["asset"], want)
+	}
+}
+
+// TestSanitizeJSONBDetailsWritesToDB is the end-to-end regression guard: a
+// NUL-bearing details document must insert into the real jsonb columns
+// (history_operations.details and history_effects.details) instead of failing.
+// Both sinks share one database fixture to keep this DB-backed test cheap.
+func TestSanitizeJSONBDetailsWritesToDB(t *testing.T) {
+	tt := test.Start(t)
+	defer tt.Finish()
+	test.ResetHorizonDB(t, tt.HorizonDB)
+	q := &Q{tt.HorizonSession()}
+
+	details, wantAsset := nulByteTestDetails(t)
+	sequence := int32(56)
+	account := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY"
+
+	// history_operations.details: without sanitizeJSONBDetails this insert fails
+	// because jsonb cannot store a NUL.
+	tt.Require.NoError(q.Begin(tt.Ctx))
+	opBuilder := q.NewOperationBatchInsertBuilder()
+	opID := toid.New(sequence, 1, 1).ToInt64()
+	tt.Require.NoError(opBuilder.Add(
+		opID,
+		toid.New(sequence, 1, 0).ToInt64(),
+		1,
+		xdr.OperationTypeInvokeHostFunction,
+		details,
+		account,
+		null.String{},
+		false,
+	))
+	tt.Require.NoError(opBuilder.Exec(tt.Ctx, q))
+	tt.Require.NoError(q.Commit())
+
+	var storedOp string
+	tt.Require.NoError(q.GetRaw(tt.Ctx, &storedOp,
+		"SELECT details FROM history_operations WHERE id = $1", opID))
+	tt.Assert.False(strings.ContainsRune(storedOp, rune(0)), "operation details must not contain a NUL")
+	tt.Assert.Contains(storedOp, wantAsset)
+
+	// history_effects.details: the second jsonb sink.
+	tt.Require.NoError(q.Begin(tt.Ctx))
+	accountLoader := NewAccountLoader(ConcurrentInserts)
+	effectBuilder := q.NewEffectBatchInsertBuilder()
+	tt.Require.NoError(effectBuilder.Add(
+		accountLoader.GetFuture(account),
+		null.String{},
+		toid.New(sequence, 1, 1).ToInt64(),
+		1,
+		EffectType(3),
+		details,
+	))
+	tt.Require.NoError(accountLoader.Exec(tt.Ctx, q))
+	tt.Require.NoError(effectBuilder.Exec(tt.Ctx, q))
+	tt.Require.NoError(q.Commit())
+
+	effects, err := q.Effects(tt.Ctx, db2.PageQuery{Cursor: "0-0", Order: "asc", Limit: 200}, 0)
+	tt.Require.NoError(err)
+	tt.Require.Len(effects, 1)
+	storedEffect := effects[0].DetailsString.String
+	tt.Assert.False(strings.ContainsRune(storedEffect, rune(0)), "effect details must not contain a NUL")
+	tt.Assert.Contains(storedEffect, wantAsset)
+}

internal/db2/history/effect_batch_insert_builder.go

@@ -51,7 +51,7 @@ func (i *effectBatchInsertBuilder) Add(
 		"history_operation_id": operationID,
 		"order":                order,
 		"type":                 effectType,
-		"details":              string(details),
+		"details":              string(sanitizeJSONBDetails(details)),
 	})
 }
 

internal/db2/history/operation_batch_insert_builder.go

@@ -55,7 +55,7 @@ func (i *operationBatchInsertBuilder) Add(
 		"transaction_id":       transactionID,
 		"application_order":    applicationOrder,
 		"type":                 operationType,
-		"details":              string(details),
+		"details":              string(sanitizeJSONBDetails(details)),
 		"source_account":       sourceAccount,
 		"source_account_muxed": sourceAccountMuxed,
 		"is_payment":           nil,