//////////////////////////////////////////////////////////////////
//
// Copyright (c) 2025-2026 YottaDB LLC and/or its subsidiaries.
// All rights reserved.
//
//        This source code contains the intellectual property
//        of its copyright holder(s), and is made available
//        under a license.  If you do not know the terms of
//        the license, please stop and do not read further.
//
//////////////////////////////////////////////////////////////////

// Define Node type for access YottaDB database

package yottadb

import (
        "reflect"
        "runtime"
        "strconv"
        "strings"
        "sync/atomic"
        "testing"
        "time"
        "unsafe"

        "github.com/outrigdev/goid"
        "lang.yottadb.com/go/yottadb/v2/ydberr"
)

/* #include "libyottadb.h"
#include "yottadb.h"

// Work around Go compiler issue to be fixed in Go 1.25: https://go-review.googlesource.com/c/go/+/642235
extern size_t _GoStringLen(_GoString_ s);
extern const char *_GoStringPtr(_GoString_ s);

// Fill ydb_buffer_t with a Go string.
// Returns 1 on success or 0 if the string had to be truncated
int fill_buffer(ydb_buffer_t *buf, _GoString_ val) {
        unsigned int len = _GoStringLen(val);
        buf->len_used = len <= buf->len_alloc? len: buf->len_alloc;
        memcpy(buf->buf_addr, _GoStringPtr(val), buf->len_used);
        return len <= buf->len_alloc;
}

// Fill ydb_buffer_t from a Go []byte slice.
// Returns 1 on success or 0 if the string had to be truncated
int fill_buffer_bytes(ydb_buffer_t *buf, char *data, int len) {
        buf->len_used = len <= buf->len_alloc? len: buf->len_alloc;
        memcpy(buf->buf_addr, data, buf->len_used);
        return len <= buf->len_alloc;
}

// C routine to get address of ydb_lock_st() since CGo doesn't let you take the address of a variadic parameter-list function.
// #cgo nocallback getfunc_ydb_lock_st
// #cgo noescape getfunc_ydb_lock_st
void *getfunc_ydb_lock_st(void) {
        return (void *)&ydb_lock_st;
}
*/
import "C"

// ---- Conn connection object

// Amount to overallocate string spaces for potential future use with larger strings.
// This could be set to C.YDB_MAX_STR to avoid any reallocation, but that would make all new connections large.
// This must be at least large enough to store a number converted to a string (see setAnyValue)
const overalloc = 1024 // Initial size (may be enlarged).

// Conn represents a goroutine-specific 'connection' object for calling the YottaDB API.
// You must use a different connection for each goroutine.
type Conn struct {
        // Conn type wraps C.conn in a Go struct so Go can add methods to it.

        // Pointer to C.conn rather than the item itself so we can malloc it and point to it from C without Go moving it.
        cconn *C.conn
        // tptoken is a place to store tptoken for thread-safe ydb_*_st() function calls
        // It was originally made to be atomic and a pointer, so that Conn.CloneConn() could share pointers to it until
        // it was realized that created a bug. But keep it atomic since atomicity has no compiled overhead
        // with Uint64 on a 64-bit machine (confirmed empirically).
        tptoken atomic.Uint64
        // timeoutAction is the action to take on transaction timeout. See [conn.TimeoutAction]()
        timeoutAction int
        goroutineID   uint64 // allows a runtime assert that this Conn is being accessed by the correct goroutine
}

// _newConn is a subset of NewConn without initCheck and value space -- used by signals.init()
func _newConn() *Conn {
        var conn Conn
        conn.cconn = (*C.conn)(calloc(C.sizeof_conn)) // must use our calloc, not malloc: see calloc doc
        conn.tptoken = atomic.Uint64{}
        conn.tptoken.Store(C.YDB_NOTTP)
        conn.timeoutAction = TransactionTimeout
        conn.goroutineID = goid.Get()
        // Create space for err
        conn.cconn.errstr.buf_addr = (*C.char)(C.malloc(C.YDB_MAX_ERRORMSG))
        conn.cconn.errstr.len_alloc = C.YDB_MAX_ERRORMSG
        conn.cconn.errstr.len_used = 0

        runtime.AddCleanup(&conn, func(cn *C.conn) {
                C.free(unsafe.Pointer(cn.value.buf_addr))
                C.free(unsafe.Pointer(cn.errstr.buf_addr))
                C.free(unsafe.Pointer(cn.vplist))
                C.free(unsafe.Pointer(cn.paramBlock))
                C.free(unsafe.Pointer(cn))
        }, conn.cconn)
        return &conn
}

// NewConn creates a new database connection.
// Each goroutine must have its own connection.
// To prevent deadlocks, this function panics if another Conn has already been created in this goroutine.
func NewConn() *Conn {
        // Go workaround to make examples work properly
        if testing.Testing() {
                // This is very hacky, but lets all Go Example*() functions run.
                // The problem is that although Go runs tests in separate goroutines,
                // it runs all examples in the same goroutine.
                // So each example has to use the same Conn, but at the same time
                // each example has to call NewConn() in order to be a verbatim user-runnable example.
                // So if testing and the caller is Example*(), we call forceNewConn() instead of NewConn()
                // to ignore the fact that it's getting more than only one per goroutine.
                // In an ideal world, Go testing would let us hook each example before running it to freshen things up.
                programCounter, _, _, ok := runtime.Caller(1)
                f := runtime.FuncForPC(programCounter)
                names := strings.Split(f.Name(), ".")
                if ok && f != nil && strings.HasPrefix(names[len(names)-1], "Example") {
                        return forceNewConn()
                }
        }
        // Note that it would be possible to use the goroutine ID to allow use of multiple Conns with transactions,
        // But using the goroutine ID is not recommended by Go, so it is deemed acceptable only to use it for catching
        // unintentional programmer errors, as is done by this panic.
        val, ok := dbHandle.routineHasConn.Swap(goid.Get(), true)
        hasConn := ok && val.(bool)
        if hasConn {
                panic(errorf(ydberr.MultipleConns, "tried to run NewConn() in goroutine %d that already has a Conn", goid.Get()))
        }
        return forceNewConnWithoutRegistering()
}

// forceNewConn is the same as [NewConn] but does not prevent multiple Conns per goroutine.
// It is private, for internal use and testing.
// It is ok to create more than one Conn per connection provided only one is used while a transactions is active.
// Since the Conn contains the transaction token, starting a transaction with one Conn and then performing
// database functions with another conn while that first transaction is still processing, will cause a deadlock.
func forceNewConn() *Conn {
        dbHandle.routineHasConn.Store(goid.Get(), true)
        return forceNewConnWithoutRegistering()
}

// forceNewConnWithoutRegistering creates a new Conn but does not register that this thread has a Conn.
func forceNewConnWithoutRegistering() *Conn {
        initCheck()
        conn := _newConn()
        // Create initial space for value used by various C API calls and returns
        conn.cconn.value.buf_addr = (*C.char)(C.malloc(overalloc))
        conn.cconn.value.len_alloc = C.uint(overalloc)
        conn.cconn.value.len_used = 0
        return conn
}

var hasFastGoID bool

// Setup code to determine whether goid supports fast fetching of the goroutineID.
// Hopefully that module will expose this information itself in its next version:
// I have offered a pull request at https://github.com/outrigdev/goid/issues/2
// and hopefully it will not be Go-version-dependent in future: https://github.com/outrigdev/goid/issues/1
func init() {
        version := runtime.Version()
        hasFastGoID = strings.HasPrefix(version, "go1.23.") || strings.HasPrefix(version, "go1.24.") || strings.HasPrefix(version, "go1.25.")
        hasFastGoID = hasFastGoID && (runtime.GOARCH == "arm64" || runtime.GOARCH == "amd64")
}

// prepAPI initializes anything necessary before C API calls.
// This sets the error_output string to the empty string in case the API call fails -- at least then we don't get an obsolete string reported.
func (conn *Conn) prepAPI() {
        if hasFastGoID && conn.goroutineID != goid.Get() {
                panic(errorf(ydberr.InvalidGoroutineID, "yottadb function invoked in goroutine %d from a Conn instance belonging to goroutine %d", goid.Get(), conn.goroutineID))
        }
        conn.cconn.errstr.len_used = 0 // ensure error string is empty before API call
}

// ensureValueSize reallocates value.buf_addr if necessary to fit a string of size.
func (conn *Conn) ensureValueSize(cap int) {
        if cap > C.YDB_MAX_STR {
                panic(errorf(ydberr.InvalidStringLength, "Invalid string length %d: max %d", cap, C.YDB_MAX_STR))
        }
        value := &conn.cconn.value
        if cap > int(value.len_alloc) {
                cap += overalloc // allocate some extra for potential future use
                addr := (*C.char)(C.realloc(unsafe.Pointer(value.buf_addr), C.size_t(cap)))
                if addr == nil {
                        panic(errorf(ydberr.OutOfMemory, "out of memory when allocating %d bytes for string data transfer to YottaDB", cap))
                }
                value.buf_addr = addr
                value.len_alloc = C.uint(cap)
        }
}

// setValue stores val into the ydb_buffer of Conn.cconn.value.
func (conn *Conn) setValue(val string) *C.ydb_buffer_t {
        cconn := conn.cconn
        conn.ensureValueSize(len(val))
        C.fill_buffer(&cconn.value, val)
        return &cconn.value
}

// setValueBytes stores val into the ydb_buffer of Conn.cconn.value.
func (conn *Conn) setValueBytes(val []byte) *C.ydb_buffer_t {
        cconn := conn.cconn
        conn.ensureValueSize(len(val))
        C.fill_buffer_bytes(&cconn.value, (*C.char)(unsafe.Pointer(unsafe.SliceData(val))), C.int(len(val)))
        return &cconn.value
}

// anyToString converts a number, []byte slices, or string to a string, like Sprint(val) but faster.
//   - val may be a string, []byte slice, integer type, or float; numeric types are converted to a string using the appropriate strconv function.
func anyToString(val any) string {
        switch n := val.(type) {
        // Go evaluates these cases in order, so put common ones first
        case string:
                return n
        case int:
                return strconv.FormatInt(int64(n), 10)
        case float64:
                return strconv.FormatFloat(n, 'G', -1, 64)
        case int64:
                return strconv.FormatInt(n, 10)
        case int32:
                return strconv.FormatInt(int64(n), 10)
        case uint:
                return strconv.FormatUint(uint64(n), 10)
        case uint32:
                return strconv.FormatUint(uint64(n), 10)
        case uint64:
                return strconv.FormatUint(n, 10)
        case float32:
                return strconv.FormatFloat(float64(n), 'G', -1, 32)
        case []byte:
                return string(n)
        default:
                panic(errorf(ydberr.InvalidValueType, "subscript (%v) must be a string, number, or []byte slice but is %s", val, reflect.TypeOf(val)))
        }
}

// setAnyValue is the same as setValue but accepts any type.
// It is akin to setValue(Sprint(val)) but faster.
//   - val may be a string, []byte slice, integer type, bool, or float; numeric types are converted to a string using the appropriate strconv function.
//     Type bool is converted to 0 or 1
//
// This function could use [anyToString] but it is faster when it doesn't because it can store []byte arrays directly into YDB buffer without conversion.
func (conn *Conn) setAnyValue(val any) {
        var str string
        switch n := val.(type) {
        // Go evaluates these cases in order, so put common ones first
        case string:
                // ensure enough space is allocated (not needed for number cases
                conn.setValue(n)
                return
        case int:
                str = strconv.FormatInt(int64(n), 10)
        case float64:
                str = strconv.FormatFloat(n, 'G', -1, 64)
        case int64:
                str = strconv.FormatInt(n, 10)
        case int32:
                str = strconv.FormatInt(int64(n), 10)
        case uint:
                str = strconv.FormatUint(uint64(n), 10)
        case uint32:
                str = strconv.FormatUint(uint64(n), 10)
        case uint64:
                str = strconv.FormatUint(n, 10)
        case float32:
                str = strconv.FormatFloat(float64(n), 'G', -1, 32)
        case bool:
                if n {
                        str = "1"
                } else {
                        str = "0"
                }
        case []byte:
                conn.setValueBytes(n)
                return
        default:
                panic(errorf(ydberr.InvalidValueType, "value (%v) must be a string, number, or []byte slice but is %s", val, reflect.TypeOf(val)))
        }
        // The following is equivalent to setValue() but without the size check which is unnecessary since NewConn allocates at least overalloc size
        C.fill_buffer(&conn.cconn.value, str)
        runtime.KeepAlive(conn) // ensure conn sticks around until we've finished copying data into it's C allocation
}

func (conn *Conn) getValue() string {
        cconn := conn.cconn
        r := C.GoStringN(cconn.value.buf_addr, C.int(cconn.value.len_used))
        runtime.KeepAlive(conn) // ensure conn sticks around until we've finished copying data from it's C allocation
        return r
}

// Zwr2Str takes the given ZWRITE-formatted string and converts it to return as a normal ASCII string.
//   - If the input string does not fit within the maximum YottaDB string size, return a *Error with Code=ydberr.InvalidStringLength
//   - If zstr is not in valid zwrite format, return the empty string and a *Error with Code=ydberr.InvalidZwriteFormat.
//   - Otherwise, return the decoded string.
//   - Note that the length of a string in zwrite format is always greater than or equal to the string in its original, unencoded format.
//
// Panics on other errors because they are are all panic-worthy (e.g. invalid variable names).
func (conn *Conn) Zwr2Str(zstr string) (string, error) {
        cconn := conn.cconn
        // Don't rely on setValue (below) to check length because it panics, whereas this function is supposed to return errors
        if len(zstr) > C.YDB_MAX_STR {
                return "", errorf(ydberr.InvalidStringLength, "Invalid string length %d: max %d", len(zstr), C.YDB_MAX_STR)
        }
        cbuf := conn.setValue(zstr)
        conn.prepAPI()
        status := C.ydb_zwr2str_st(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, cbuf, cbuf)
        if status == ydberr.INVSTRLEN {
                // NOTE: this code will never run in the current design because setValue() above always allocates enough space
                // Allocate more space and retry the call
                conn.ensureValueSize(int(cconn.value.len_used))
                conn.prepAPI()
                status = C.ydb_zwr2str_st(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, cbuf, cbuf)
        }
        if status != YDB_OK {
                // This error shouldn't happen so hard to test code coverage
                return "", conn.lastError(status)
        }
        val := conn.getValue()
        if val == "" && zstr != "" {
                s := zstr
                if len(s) > 80 {
                        s = s[:80] + "..."
                }
                return "", errorf(ydberr.InvalidZwriteFormat, "string has invalid ZWRITE-format: %s", s)
        }
        return val, nil
}

// Str2Zwr takes the given Go string and converts it to return a ZWRITE-formatted string
//   - If the input string does not fit within the maximum YottaDB string size, return a *Error with Code=ydberr.InvalidStringLength
//   - If the output string does not fit within the maximum YottaDB string size, return a *Error with Code=ydberr.INVSTRLEN
//   - Otherwise, return the ZWRITE-formatted string.
//   - Note that the length of a string in zwrite format is always greater than or equal to the string in its original, unencoded format.
//
// Panics on other errors because they are are all panic-worthy (e.g. invalid variable names).
func (conn *Conn) Str2Zwr(str string) (string, error) {
        cconn := conn.cconn
        // Don't rely on setValue (below) to check length because it panics, whereas this function is supposed to return errors
        if len(str) > C.YDB_MAX_STR {
                return "", errorf(ydberr.InvalidStringLength, "Invalid string length %d: max %d", len(str), C.YDB_MAX_STR)
        }
        cbuf := conn.setValue(str)
        conn.prepAPI()
        status := C.ydb_str2zwr_st(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, cbuf, cbuf)
        if status == ydberr.INVSTRLEN {
                // Allocate more space and retry the call
                conn.ensureValueSize(int(cconn.value.len_used))
                cbuf := conn.setValue(str)
                conn.prepAPI()
                status = C.ydb_str2zwr_st(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, cbuf, cbuf)
        }
        if status != YDB_OK {
                return "", conn.lastError(status)
        }
        return conn.getValue(), nil
}

// Check whether an entire string is printable ASCII to avoid unnecessarily calling YDB Str2Zwr().
func printableASCII(s string) bool {
        for i := 0; i < len(s); i++ {
                if s[i] < ' ' || s[i] > '~' {
                        return false
                }
        }
        return true
}

// Quote adds quotes around strings but not around numbers (just as YottaDB would display them).
//   - The input value is treated as a string if it cannot be converted, unchanged, to and from float64
//     using [strconv.ParseFloat](value, 64) and [strconv.FormatFloat](number, 'f', -1, 64)
//   - If the string contains unprintable ASCII characters it is converted to YottaDB ZWRITE format using [Conn.Str2Zwr].
//   - This is exported so that the user can validate against the same conversion that is used by YDBGo.
func (conn *Conn) Quote(value string) string {
        num, err := strconv.ParseFloat(value, 64)
        // Treat as number only if it can be converted back to the same number -- in which case M would treat it as a number
        if err == nil && value == strconv.FormatFloat(num, 'f', -1, 64) {
                return value
        }
        if printableASCII(value) {
                return "\"" + value + "\""
        } else {
                zwr, err := conn.Str2Zwr(value)
                if err != nil {
                        panic(err)
                }
                return zwr
        }
}

// KillLocalsExcept kills all M 'locals' except for the ones listed by name in exclusions.
//   - To kill a specific variable use [Node.Kill]()
func (conn *Conn) KillLocalsExcept(exclusions ...string) {
        var status C.int
        cconn := conn.cconn

        names := stringArrayToAnyArray(exclusions)
        if len(names) == 0 {
                conn.prepAPI()
                status = C.ydb_delete_excl_st(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, 0, nil)
        } else {
                // use a Node type just as a handy way to store exclusions strings as a ydb_buffer_t array
                namelist := conn._Node(names[0], names[1:])
                conn.prepAPI()
                status = C.ydb_delete_excl_st(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, C.int(len(names)), namelist.cnode.buffers)
                runtime.KeepAlive(namelist) // ensure namelist sticks around until we've finished copying data from it's C allocation
        }
        if status != YDB_OK {
                panic(conn.lastError(status))
        }
}

// KillAllLocals kills all M 'locals'.
// It is for clearer source code as it simply calls KillLocalsExcept() without listing any exceptions.
//   - To kill a specific variable use [Node.Kill]()
func (conn *Conn) KillAllLocals() {
        conn.KillLocalsExcept()
}

// Lock releases all existing locks and attempt to acquire locks matching all supplied nodes, waiting up to timeout for availability.
//   - Equivalent to the M `LOCK` command. See [Node.Lock]() and [Node.Unlock]() methods for single-lock usage.
//   - A timeout of zero means try only once.
//   - Return true if lock was acquired; otherwise false.
//   - Panics with error TIME2LONG if the timeout exceeds YDB_MAX_TIME_NSEC or on other panic-worthy errors (e.g. invalid variable names).
func (conn *Conn) Lock(timeout time.Duration, nodes ...*Node) bool {
        timeoutNsec := C.ulonglong(timeout.Nanoseconds())
        cconn := conn.cconn

        // Add each parameter to the vararg list
        conn.vpStart() // restart parameter list
        conn.vpAddParam64(conn.tptoken.Load())
        conn.vpAddParam(uintptr(unsafe.Pointer(&cconn.errstr)))
        conn.vpAddParam64(uint64(timeoutNsec))
        conn.vpAddParam(uintptr(len(nodes)))
        for _, node := range nodes {
                cnode := node.cnode
                conn.vpAddParam(uintptr(unsafe.Pointer(cnode.buffers)))
                conn.vpAddParam(uintptr(cnode.len - 1))
                conn.vpAddParam(uintptr(unsafe.Pointer(bufferIndex(cnode.buffers, 1))))
        }

        // vplist now contains the parameter list we want to send to ydb_lock_st(). But CGo doesn't permit us
        // to call or even create a function pointer to ydb_lock_st(). So get it with getfunc_ydb_lock_st().
        status := conn.vpCall(C.getfunc_ydb_lock_st()) // call ydb_lock_st()
        runtime.KeepAlive(nodes)                       // ensure nodes sticks around until we've finished copying data from their C allocations
        if status != YDB_OK && status != C.YDB_LOCK_TIMEOUT {
                panic(conn.lastError(status))
        }
        return status == YDB_OK
}

// This function is just to test CGo speed in the benchmarks
func callCGo() {
        C.getfunc_ydb_lock_st()
}

//////////////////////////////////////////////////////////////////
//
// Copyright (c) 2025-2026 YottaDB LLC and/or its subsidiaries.
// All rights reserved.
//
//        This source code contains the intellectual property
//        of its copyright holder(s), and is made available
//        under a license.  If you do not know the terms of
//        the license, please stop and do not read further.
//
//////////////////////////////////////////////////////////////////

// ---- Error types for YottaDB errors, YDBGo Init errors, and YDBGo Signal errors

package yottadb

import (
        "errors"
        "fmt"
        "runtime"
        "strconv"
        "strings"

        "lang.yottadb.com/go/yottadb/v2/ydberr"
)

// #include "libyottadb.h"
import "C"

// MaxStacktrace is the maximum number of stack frames stored in an Error instance
var MaxStacktrace = 4096

// Error type holds YDBGo and YottaDB errors including a numeric error code.
// YDBGo error strategy is as follows. Database setup functions like [Init] and [Conn.Import] (and its returned functions) return errors.
// However, [Node] functions panic on errors because Node errors are caused by either programmer blunders (like invalid variable name)
// or system-level events (like out of memory). This approach greatly simplifies the use of Node methods.
//
// If a particular panic needs to be captured this can be done with recover as YDBGo ensures that all its errors and panics
// are of type yottadb.Error to facilitate capture of the specific cause using the embedded code:
//   - All YottaDB errors are formated in $ZSTATUS format message and the YottaDB numeric error code with negative value,
//     defined in ydberr/errorscodes.go.
//   - All YDBGo errors likewise have a message string, but have a positive error code, defined in ydberr.ydberr.go.
//
// The [yottadb.Error] type implements the [Error.Is] method to check the entire error chain, matching only against the error code.
// However, the API function [yottadb.ErrorIs] wraps this more conveniently to check any error type (even non-YottaDB errors)
// for a match against a given YottaDB error code: yottadb.ErrorIs(err, ydberr.<ErrorCode>).
type Error struct {
        Message string  // The error string - generally from $ZSTATUS when available
        Code    int     // The error value (e.g. ydberr.INVSTRLEN, etc)
        chain   []error // Lists any errors wrapped by this one
        stack   string  // stack trace of error, used by Error.Format
}

// Error is a type method of [yottadb.Error] to return the error message string.
func (err *Error) Error() string {
        return err.Message // The error code's name is already included in the message from YottaDB, so don't add the code
}

// ErrorCode returns the error code of err if it is an instance of yottadb.Error; otherwise returns ydberr.NotYDBError.
// Unlike err.Code this works even if err is not an error instance.
func ErrorCode(err any) int {
        if e, ok := err.(*Error); ok {
                return e.Code
        }
        return ydberr.NotYDBError
}

// Unwrap allows yottadb.Error to wrap other underlying errors. See [errors.Unwrap].
func (err *Error) Unwrap() []error {
        return err.chain
}

// Is lets [errors.Is]() search an error or chain of wrapped errors for a yottadb.Error with a matching ydberr code.
// See [ErrorIs]() for a more practical way to use this capability.
// Only the error code is matched, not the message: this supports matching errors even when YottaDB messages vary.
func (err *Error) Is(target error) bool {
        t, ok := target.(*Error)
        return ok && err.Code == t.Code
}

// ErrorIs uses [errors.Is]() to search an error or chain of wrapped errors for a yottadb.Error with a matching ydberr code.
// Only the error code is matched, not the message, to support YottaDB error messages that vary.
// For example, to test for YottaDB INVSTRLEN error:
//
//        if yottadb.ErrorIs(err, ydberr.INVSTRLEN) {
//
// is a short equivalent of:
//
//        if errors.Is(err, &Error{Code: ydberr.INVSTRLEN}) {
//
// It differs from a simple type test using yottadb.Error.Code in that it searches for a match in the entire chain of wrapped errors
func ErrorIs(err any, code int) bool {
        err2, ok := err.(error)
        return ok && errors.Is(err2, &Error{Code: code})
}

// newError returns error code and message as a [yottadb.Error] error type.
// Any errors supplied in wrapErrors are wrapped in the returned error and their messages appended (after a colon) to the given message.
// For YDBGo error strategy see [yottadb.Error]
func newError(code int, message string, wrapErrors ...error) error {
        for _, err := range wrapErrors {
                newMessage := err.Error()
                if newMessage != "" {
                        message = message + ": " + newMessage
                }
        }
        stack := make([]byte, MaxStacktrace)
        n := runtime.Stack(stack[:], false)
        return &Error{Code: code, Message: message, chain: wrapErrors, stack: string(stack[:n])}
}

// errorf same as fmt.Errorf except that it returns error type yottadb.Error with specified code; and doesn't handle %w.
func errorf(code int, format string, args ...any) error {
        return newError(code, fmt.Sprintf(format, args...))
}

// ---- Error Functions dependent on yottadb.Conn to fetch message strings from YottaDB

// getErrorString returns a copy of conn.cconn.errstr as a Go string.
func (conn *Conn) getErrorString() string {
        // len_used should never be greater than len_alloc since all errors should fit into errstr, but just in case, take the min
        errstr := conn.cconn.errstr
        r := C.GoStringN(errstr.buf_addr, C.int(min(errstr.len_used, errstr.len_alloc)))
        runtime.KeepAlive(conn) // ensure conn sticks around until we've finished copying data from it's C allocation
        return r
}

// lastError returns, given error code, the ydb error message stored by the previous YottaDB call as an error type or nil if there was no error.
// If you don't know the code, call lastCode()
func (conn *Conn) lastError(code C.int) error {
        if code == C.YDB_OK {
                return nil
        }
        // The next two cases duplicate code in recoverMessage but are performance-critical so are checked here:
        if code == YDB_TP_RESTART {
                return newError(int(code), "YDB_TP_RESTART")
        }
        if code == YDB_TP_ROLLBACK {
                return newError(int(code), "YDB_TP_ROLLBACK")
        }
        msg := conn.getErrorString()
        if msg == "" { // See if msg is still empty (we set it to empty before calling the API in conn.prepAPI()
                // This code gets run, for example, if ydb_exit() is called before a YDB function is invoked
                // causing the YDB function to exit without filling conn.cconn.errstr
                return newError(int(code), conn.recoverMessage(code))
        }

        fields := strings.SplitN(msg, ",", 3)
        if len(fields) < 3 {
                // If msg is improperly formatted, panic it verbatim with code YDBMessageInvalid (this should never happen with messages from YottaDB)
                panic(errorf(ydberr.YDBMessageInvalid, "contact YottaDB support: improperly formatted YottaDB error message: %s", msg))
        }
        // code := fields[0]
        entryref, text := fields[1], fields[2]
        // Hide entryref since it is always the same "(SimpleThreadAPI)" unless returning an error from within an M call
        if entryref != "(SimpleThreadAPI)" {
                text = text + " at M entryref " + entryref
        }
        return newError(int(code), text)
}

// lastCode extracts the ydb error status code from the message stored by the previous YottaDB call.
func (conn *Conn) lastCode() C.int {
        msg := conn.getErrorString()
        if msg == "" {
                // if msg is empty there was no error because we set it to empty before calling the API in conn.prepAPI()
                return C.int(YDB_OK)
        }

        // Extract the error code from msg
        index := strings.Index(msg, ",")
        if index == -1 {
                // If msg is improperly formatted, panic it verbatim with an YDBMessageInvalid code (this should never happen with messages from YottaDB)
                panic(errorf(ydberr.YDBMessageInvalid, "contact YottaDB support: could not parse YottaDB error message: %s", msg))
        }
        code, err := strconv.ParseInt(msg[:index], 10, 64)
        if err != nil {
                // If msg has no number, panic it verbatim with an YDBMessageInvalid code (this should never happen with messages from YottaDB)
                panic(errorf(ydberr.YDBMessageInvalid, "contact YottaDB support: could not recover error code from YottaDB error message: %s", msg))
        }
        return C.int(-code)
}

// recoverMessage tries to get the error message (albeit without argument substitution) from the supplied status code in cases where the message has been lost.
// Note: lastCode() may not be called after recoverMessage() because recoverMessage clobbers cconn.errstr when it calls ydb_message_t
func (conn *Conn) recoverMessage(status C.int) string {
        cconn := conn.cconn
        // Check special cases first.
        switch status {
        // Identify certain return codes that are not identified by ydb_message_t().
        // I have only observed YDB_TP_RESTART being returned, but include the others just in case.
        case YDB_TP_RESTART:
                return "YDB_TP_RESTART"
        case YDB_TP_ROLLBACK:
                return "YDB_TP_ROLLBACK"
        case YDB_NOTOK:
                return "YDB_NOTOK"
        case YDB_LOCK_TIMEOUT:
                return "YDB_LOCK_TIMEOUT"
        case YDB_DEFER_HANDLER:
                return "YDB_DEFER_HANDLER"
        case ydberr.THREADEDAPINOTALLOWED:
                // This error will prevent ydb_message_t() from working below, so instead return a hard-coded error message.
                return "%YDB-E-THREADEDAPINOTALLOWED, Process cannot switch to using threaded Simple API while already using Simple API"
        case ydberr.CALLINAFTERXIT:
                // The engine is shut down so calling ydb_message_t will fail if we attempt it so just hard-code this error return value.
                return "%YDB-E-CALLINAFTERXIT, After a ydb_exit(), a process cannot create a valid YottaDB context"
        }
        // note: ydb_message_t() only looks at the absolute value of status so no need to negate it
        conn.prepAPI()
        rc := C.ydb_message_t(C.uint64_t(conn.tptoken.Load()), nil, status, &cconn.errstr)
        if rc != YDB_OK {
                // Handle UNKNOWNSYSERR specially with a friendly message as it is the most likely error when ydb_message_t can't find the message
                if rc == ydberr.UNKNOWNSYSERR {
                        panic(errorf(int(rc), "%%YDB-E-UNKNOWNSYSERR, [%d (%#x) returned by ydb_* C API] does not correspond to a known YottaDB error code", status, status))
                }
                // Do not call lastError if there is no message because it will infinitely recurse back to here to get the message
                // Pretty hard to work out how to coverage-test this error
                panic(errorf(ydberr.YDBMessageRecoveryFailure, "ydb_message_t() returned YottaDB error code %d (%#x) when trying to get the message for error %d (%#x)", rc, rc, status, status))
        }
        return conn.getErrorString()
}

//////////////////////////////////////////////////////////////////
//
// Copyright (c) 2025-2026 YottaDB LLC and/or its subsidiaries.
// All rights reserved.
//
//        This source code contains the intellectual property
//        of its copyright holder(s), and is made available
//        under a license.  If you do not know the terms of
//        the license, please stop and do not read further.
//
//////////////////////////////////////////////////////////////////

// Initialize and Shutdown YottaDB

package yottadb

import (
        "fmt"
        "log"
        "runtime/debug"
        "strconv"
        "strings"
        "sync"
        "sync/atomic"
        "time"

        "lang.yottadb.com/go/yottadb/v2/ydberr"
)

// #include "libyottadb.h"
// extern void *ydb_signal_exit_callback(void);
import "C"

// DB is the type returned by Init() which must be passed to Shutdown().
// It is used as a clue to the user that they must not forget to Shutdown()
type DB struct {
        YDBRelease     float64  // release version of the installed YottaDB
        routineHasConn sync.Map // atomic map[uint64]bool to tell whether a goroutine already has a Conn; one bool for each goroutine ID (uint64)
}

// dbHandle is the global handle used to access database metadata
// Make this a single instance per app so that we can use it, e.g. for routineHasConn to store goroutineIDs for the whole process
var dbHandle = &DB{}

// Init and Exit globals
var wgexit sync.WaitGroup
var inInit sync.Mutex        // Mutex for access to init AND exit
var wgSigInit sync.WaitGroup // Used to make sure signals are setup before Init() exits
var initCount atomic.Int64   // Increment when Init() called and decrement when Shutdown() called; shutdown when it reaches 0

// MustInit calls [Init]() and panics on errors. It is purely to shorten example code.
func MustInit() *DB {
        db, err := Init()
        if err != nil {
                panic(err)
        }
        return db
}

// getZYRelease function pointer allows TestInit to monkey-patch it for testing of release parsing logic in Init()
var getZYRelease func(conn *Conn) string = func(conn *Conn) string { return conn.Node("$ZYRELEASE").Get() }

// Init initializes the YottaDB engine and sets up signal handling.
// Init may be called multiple times (e.g. by different goroutines) but [Shutdown]() must be called exactly once
// for each time Init() was called. See [Shutdown] for more detail on the fallout from incorrect usage.
// Although Init could have been made to happen automatically, this more explicit approach clarifies that
// Shutdown() MUST be called before process exit.
//   - Be sure to read the cautions at [Shutdown].
//   - Init returns a value of type DB that must be passed to the [Shutdown] function.
//   - Returns yottadb.Error with Code=ydberr.Init on failure, which will also wrap any errors from YottaDB in the error chain.
//
// Users should defer [Shutdown] from their main routine before using other database functions; for example:
//
//        db, err := yottadb.Init()
//        if err != nil {
//          panic(err)
//        }
//        defer yottadb.Shutdown(db)
//          ... user code to use the database ...
func Init() (*DB, error) {
        // This is an atypical method of doing simple API initialization compared to
        // other language APIs, where you can just make any API call, and initialization is automatic.
        // But the Go wrapper needs to do its initialization differently to setup signal handling differently.
        // Usually, YottaDB sets up its signal handling, but to work well with Go, Go itself needs to do the
        // signal handling and forward it as needed to the YottaDB engine.

        inInit.Lock()
        defer inInit.Unlock()     // Release lock when we leave this routine
        if initCount.Add(1) > 1 { // Must increment this before calling NewConn() below or it will fail initCheck()
                return dbHandle, nil // already initialized
        }
        defer initCount.Add(-1) // decrement it again in case there is an error return (successful return below increments it an extra time to compensate)

        // Init YottaDB engine/runtime
        // In Go, instead of ydb_init(), use ydb_main_lang_init() which lets us pass an exit handler for YDB to call on fatal signals.
        // We must make the exit handler panic to ensure defers get called before exit.
        // YDB calls the exit handler after rundown (equivalent to ydb_exit()).
        printEntry("YottaDB Init()")

        // temporary conn used purely during Init() to fetch version info from YDB
        // use forceNewConnWithoutRegistering() so as not to set routineHasConn for this goroutine since it is only used in init
        conn := forceNewConnWithoutRegistering()
        ydbSigPanicCalled.Store(false) // Since running ydb_main_lang_init, ydb_exit() has not been called by a signal
        // Note: ydb_init returns positive rather than negative status unlike all other API functions, so negate it here.
        status := -C.ydb_main_lang_init(C.YDB_MAIN_LANG_GO, C.ydb_signal_exit_callback)
        if status != YDB_OK {
                dberror := newError(int(status), conn.recoverMessage(status))
                return nil, newError(ydberr.Init, "YottaDB initialization failed", dberror)
        }

        var releaseMajorStr, releaseMinorStr string
        releaseInfoString := getZYRelease(conn)

        // The returned output should have the YottaDB version as the 2nd token in the form rX.YY[Y] where:
        //   - 'r' is a fixed character
        //   - X is a numeric digit specifying the major version number
        //   - YY[Y] are basically the remaining digits and specify the minor release number.
        releaseInfoTokens := strings.Fields(releaseInfoString)
        releaseNumberStr := releaseInfoTokens[1] // Fetch second token
        if releaseNumberStr[:1] != "r" {         // Better start with 'r'
                return nil, errorf(ydberr.Init, "expected YottaDB version $ZYRELEASE value to start with 'r' but it returned: %s", releaseInfoString)
        }
        releaseNumberStr = releaseNumberStr[1:]          // Remove starting 'r' in the release number
        dotIndex := strings.Index(releaseNumberStr, ".") // Look for the decimal point that separates major/minor values
        if dotIndex >= 0 {                               // Decimal point found
                releaseMajorStr = string(releaseNumberStr[:dotIndex])
                releaseMinorStr = string(releaseNumberStr[dotIndex+1:])
        } else {
                releaseMajorStr = releaseNumberStr // Isolate the major version number
                releaseMinorStr = "00"
        }
        // Note it is possible for either the major or minor release values to have a single letter suffix that is primarily
        // for use in a development environment (no production releases have character suffixes). If we get an error, try
        // removing a char off the end and retry.
        // The possibility of major release having a letter suffix prevents the use of fmt.Scanf()
        _, err := strconv.Atoi(releaseMajorStr)
        if err != nil {
                releaseMajorStr = releaseMajorStr[:len(releaseMajorStr)-1]
                _, err = strconv.Atoi(releaseMajorStr)
                if err != nil {
                        return nil, errorf(ydberr.Init, "Failure trying to convert YottaDB version $ZYRELEASE (%s) major release number to integer", releaseInfoString)
                }
        }
        _, err = strconv.Atoi(releaseMinorStr)
        if err != nil { // Strip off last char and try again
                releaseMinorStr = releaseMinorStr[:len(releaseMinorStr)-1]
                _, err = strconv.Atoi(releaseMinorStr)
                if err != nil {
                        return nil, errorf(ydberr.Init, "Failure trying to convert YottaDB version $ZYRELEASE (%s) minor release number to integer", releaseInfoString)
                }
        }
        // Verify we are running with the minimum YottaDB version or later
        runningYDBRelease, err := strconv.ParseFloat(releaseMajorStr+"."+releaseMinorStr, 64)
        if err != nil {
                // Cannot coverage-test this as it should never occur
                panic(newError(ydberr.Init, fmt.Sprintf("YDBGo wrapper error validating YottaDB version (%s); contact YottaDB support", releaseMajorStr+"."+releaseMinorStr), err)) // shouldn't happen due to check above
        }
        minYDBRelease, err := strconv.ParseFloat(MinYDBRelease[1:], 64)
        if err != nil {
                panic(errorf(ydberr.Init, "source code constant MinYDBRelease (%s) is not formatted correctly as rX.YY", MinYDBRelease))
        }
        if minYDBRelease > runningYDBRelease {
                return nil, errorf(ydberr.Init, "Not running with at least minimum YottaDB release. Needed: %s  Have: r%s.%s",
                        MinYDBRelease, releaseMajorStr, releaseMinorStr)
        }

        // Start up a goroutine to handle signals for each signal we want to be notified of. This is so that if one signal is in process,
        // we can still catch a different signal and deliver it appropriately (probably to the same goroutine). For each signal,
        // bump our wait group counter so we don't proceed until all of these goroutines are initialized.
        // If you need to handle any more or fewer signals, alter YDBSignals at the top of this module.
        for _, sig := range YDBSignals {
                wgSigInit.Add(1) // Indicate this signal goroutine is not yet initialized
                value, _ := ydbSignalMap.Load(sig)
                info := value.(*sigInfo)
                go handleSignal(info)
        }
        // Now wait for the goroutine to initialize and get signals all set up. When that is done, we can return
        wgSigInit.Wait()
        dbHandle = &DB{runningYDBRelease, sync.Map{}}

        // Increment this once more in the case of success since there is a defer that decrements it for error cases
        initCount.Add(1)
        return dbHandle, nil
}

// initCheck Panics if Init() has not been called
func initCheck() {
        if initCount.Load() == 0 {
                panic(errorf(ydberr.Init, "Init() must be called first"))
        }
}

// Shutdown invokes YottaDB's rundown function ydb_exit() to shut down the database properly.
// It MUST be called prior to process termination by any application that calls [Init]().
// It is recommended to defer Shutdown() immediately after calling [Init]() in the main routine.
// You should also defer [ShutdownOnPanic]() from new goroutines to ensure shutdown occurs if they panic.
//
// This is necessary, particularly in Go, because Go does not call the C atexit() handler (unless building with certain test options),
// so YottaDB itself cannot automatically ensure correct rundown of the database.
//
// If Shutdown() is not called prior to process termination, steps must be taken to ensure database integrity, as documented
// in [Database Integrity] and unreleased locks may cause small subsequent delays (see [relevant LKE documentation]).
//
// Deferring Shutdown() has the side benefit of exiting silently on ydberr.CALLINAFTERXIT panics if they come from
// a Ctrl-C (SIGINT) panic that has already occurred in another goroutine.
//
// Notes:
//   - It is the main routine's responsibility to ensure that any goroutines have finished using the database before it calls
//     yottadb.Shutdown(). Otherwise they will receive ydberr.CALLINAFTERXIT errors from YDBGo.
//   - Avoid Go's [os.Exit]() function because it bypasses any defers (it is a low-level OS call).
//   - Shutdown() must be called exactly once for each time [Init]() was called, and shutdown will not occur until the last time.
//
// Returns [ydberr.ShutdownIncomplete] if it has to wait longer than MaxNormalExitWait for signal handling goroutines to exit.
// No other errors are returned. Panics if Shutdown is called more than Init.
//
// [Database Integrity]: https://docs.yottadb.com/MultiLangProgGuide/goprogram.html#database-integrity
// [relevant LKE documentation]: https://docs.yottadb.com/AdminOpsGuide/mlocks.html#introduction
//
// [Go Using Signals]: https://docs.yottadb.com/MultiLangProgGuide/goprogram.html#go-using-signals
// [exceptions]: https://github.com/golang/go/issues/20713#issuecomment-1518197679
func Shutdown(handle *DB) error {
        return _shutdown(handle, false)
}

// ShutdownHard shuts down immediately even if it has not yet been called as many times as Init.
// It is used before a fatal exit like panic or fatals signals.
// ShutdownHard may be called any number of times without ill effect (e.g. by different goroutines during an application shutdown).
func ShutdownHard(handle *DB) error {
        return _shutdown(handle, true)
}

// _shutdown is the core of Shutdown.
// If force is true, Shutdown now even if it has not yet been called as many times as Init.
func _shutdown(handle *DB, force bool) error {
        // Do-nothing hack purely to prevent goimport from removing runtime/debug from imports since it's required for the docstring above
        debug.SetPanicOnFault(debug.SetPanicOnFault(false))

        // Defer a func that exits silently (after shutting down) if it was a fatal signal that caused the shutdown,
        // (which would have happened  in another goroutine).
        defer func() {
                if err := recover(); err != nil {
                        // Quit rather than error if Ctrl-C signal caused the shutdown
                        quitAfterSIGINT(err)
                        // Otherwise re-panic
                        panic(err)
                }
        }()

        // use the same mutex as Init because we don't want either to run simultaneously
        inInit.Lock()         // One goroutine at a time through here else we can get DATA-RACE warnings accessing wgexit wait group
        defer inInit.Unlock() // Release lock when we leave this routine
        if force {
                if initCount.Load() == 0 {
                        // Skip coverage-test of the next line: it would need a separate goroutine that calls ShutdownHard() while Shutdown() is already running. Tricky timing.
                        return nil // already done
                }
                initCount.Store(1)
        }
        if initCount.Load() == 0 {
                panic(errorf(ydberr.Shutdown, "Shutdown() called more times than Init()"))
        }
        if !force && initCount.Add(-1) > 0 {
                // Don't shutdown if some other goroutine is still using the dbase
                return nil
        }

        if DebugMode.Load() >= 2 {
                log.Println("Exit(): YDB Engine shutdown started")
        }
        // When we run ydb_exit(), set up a timer that will pop if ydb_exit() gets stuck in a deadlock or whatever. We could
        // be running after some fatal error has occurred so things could potentially be fairly screwed up and ydb_exit() may
        // not be able to get the lock. We'll give it the given amount of time to finish before we give up and just exit.
        exitdone := make(chan struct{}, 1)
        wgexit.Add(1)
        go func() {
                _ = C.ydb_exit()
                wgexit.Done()
        }()
        wgexit.Add(1) // And run our signal goroutine cleanup in parallel
        go func() {
                shutdownSignalGoroutines()
                wgexit.Done()
        }()
        // And now, set up our channel notification for when those both ydb_exit() and signal goroutine shutdown finish
        go func() {
                wgexit.Wait()
                close(exitdone)
        }()

        // Wait for either ydb_exit to complete or the timeout to expire but how long we wait depends on how we are ending.
        // If a signal drove a panic, we have a much shorter wait as it is highly likely the YDB engine lock is held and
        // ydb_exit() won't be able to grab it causing a hang. The timeout is to prevent the hang from becoming permanent.
        // This is not a real issue because the signal handler would have driven the exit handler to clean things up already.
        // On the other hand, if this is a normal exit, we need to be able to wait a reasonably long time in case there is
        // a significant amount of data to flush.
        exitWait := MaxNormalExitWait
        if ydbSigPanicCalled.Load() {
                exitWait = MaxPanicExitWait
        }
        var errstr string
        select {
        case <-exitdone:
                // We don't really care at this point what the return code is as we're just trying to run things down the
                // best we can as this is the end of using the YottaDB engine in this process.
        case <-time.After(exitWait):
                if DebugMode.Load() >= 2 {
                        log.Println("Shutdown(): Wait for ydb_exit() expired")
                }
                if !ydbSigPanicCalled.Load() {
                        // If we panic'd due to a signal, we definitely have run the exit handler as it runs before the panic is
                        // driven so we can bypass this message in that case.
                        errstr = "YottaDB database rundown may have been bypassed due to timeout - run MUPIP JOURNAL ROLLBACK BACKWARD / MUPIP JOURNAL RECOVER BACKWARD / MUPIP RUNDOWN"
                        syslogEntry(errstr)
                }
        }
        if errstr != "" {
                return newError(ydberr.ShutdownIncomplete, errstr)
        }
        return nil
}

//////////////////////////////////////////////////////////////////
//
// Copyright (c) 2025 YottaDB LLC and/or its subsidiaries.
// All rights reserved.
//
//        This source code contains the intellectual property
//        of its copyright holder(s), and is made available
//        under a license.  If you do not know the terms of
//        the license, please stop and do not read further.
//
//////////////////////////////////////////////////////////////////

// Handle calls to M from Go

package yottadb

import (
        "bytes"
        "fmt"
        "log"
        "os"
        "reflect"
        "regexp"
        "runtime"
        "strconv"
        "strings"
        "sync"
        "unsafe"

        "lang.yottadb.com/go/yottadb/v2/ydberr"
)

/* #include "libyottadb.h"

// C routine to get address of ydb_cip_t() since CGo doesn't let you take the address of a variadic parameter-list function.
void *getfunc_ydb_cip_t(void) {
        return (void *)&ydb_cip_t;
}
*/
import "C"

// ---- Define type returned by Import for calling M routines

// MFunctions is returned by [Import] to represent an M-call table with some methods that allow the user to call its M routines.
type MFunctions struct {
        // Almost-private metadata for testing or specialised access to the imported call table. Public for specialised use.
        Table *CallTable
        Conn  *Conn
}

// Call calls an M routine rname with parameters args and returns string, int64 or float64.
// Since the programmer knows the return-type in advance from the M-call table, the return type may be safely
// forced to that type with an unchecked type assertion. For example:
//
//        x = m.Call("add", 1, 2).(int64)
//
// This version panics on errors. See [MFunctions.CallErr]() for a version that returns errors.
func (m *MFunctions) Call(rname string, args ...any) any {
        routine := m.getRoutine(rname)
        ret, err := m.Conn.callM(routine, args)
        if err != nil {
                panic(err)
        }
        return ret
}

// CallErr calls an M routine rname with parameters args and returns string, int64 or float64, and any error.
// This version returns any errors. See [MFunctions.Call]() for a version that panics.
func (m *MFunctions) CallErr(rname string, args ...any) (any, error) {
        routine := m.getRoutine(rname)
        ret, err := m.Conn.callM(routine, args)
        if err != nil {
                return nil, err
        }
        return ret, nil
}

// Wrap returns a function that calls an M routine rname that returns any.
// This can speed up calling M routines by avoiding the name lookup each invocation.
// This version creates functions that panic: compare [MFunctions.WrapErr] that returns errors instead.
func (m *MFunctions) Wrap(rname string) func(args ...any) any {
        routine := m.getRoutine(rname)
        return func(args ...any) any {
                ret, err := m.Conn.callM(routine, args)
                if err != nil {
                        panic(err)
                }
                return ret
        }
}

// WrapRetInt produces a Go convenience function that wraps an M routine that returns int.
// This can avoid messy type assertion and speed up calling M routines by avoiding the name lookup each invocation.
// Rather than int64 it returns int as the default type in Go so that regular arithmetic can be done on the result without casting.
// This version creates functions that panic: compare [MFunctions.WrapErr] that returns errors instead.
func (m *MFunctions) WrapRetInt(rname string) func(args ...any) int {
        routine := m.getRoutine(rname)
        return func(args ...any) int {
                ret, err := m.Conn.callM(routine, args)
                if err != nil {
                        panic(err)
                }
                return ret.(int)
        }
}

// WrapRetString produces a Go convenience function that wraps an M routine that returns string.
// This can avoid messy type assertion and speed up calling M routines by avoiding the name lookup each invocation.
// This version creates functions that panic: compare [MFunctions.WrapErr] that returns errors instead.
func (m *MFunctions) WrapRetString(rname string) func(args ...any) string {
        routine := m.getRoutine(rname)
        return func(args ...any) string {
                ret, err := m.Conn.callM(routine, args)
                if err != nil {
                        panic(err)
                }
                return ret.(string)
        }
}

// WrapRetFloat produces a Go convenience function that wraps an M routine that returns float64.
// This can avoid messy type assertion and speed up calling M routines by avoiding the name lookup each invocation.
// This version creates functions that panic: compare [MFunctions.WrapErr] that returns errors instead.
func (m *MFunctions) WrapRetFloat(rname string) func(args ...any) float64 {
        routine := m.getRoutine(rname)
        return func(args ...any) float64 {
                ret, err := m.Conn.callM(routine, args)
                if err != nil {
                        panic(err)
                }
                return ret.(float64)
        }
}

// WrapErr returns a function that calls an M routine rname that returns any.
// This can speed up calling M routines by avoiding the name lookup each invocation.
// This version creates functions that return errors: compare [MFunctions.Wrap] that panics instead.
func (m *MFunctions) WrapErr(rname string) func(args ...any) (any, error) {
        routine := m.getRoutine(rname)
        return func(args ...any) (any, error) {
                return m.Conn.callM(routine, args)
        }
}

func (m *MFunctions) getRoutine(name string) *RoutineData {
        routine, ok := m.Table.Routines[name]
        if !ok {
                panic(errorf(ydberr.MCallNotFound, "M routine '%s' not found in M-call table", name))
        }
        return routine
}

// ---- Internal types

type typeInfo struct {
        kind    reflect.Kind // for matching
        ydbType string
}

var typeMapper map[string]typeInfo = map[string]typeInfo{
        "":        {reflect.Invalid, "void"},
        "string":  {reflect.String, "ydb_buffer_t*"},
        "int":     {reflect.Int, "ydb_long_t*"}, // YottaDB (u)long type switches between 32- and 64-bit by platform, just like Go int
        "uint":    {reflect.Uint, "ydb_ulong_t*"},
        "int32":   {reflect.Int32, "ydb_int_t*"}, // YottaDB (u)int type is 32-bit only like Go int32
        "uint32":  {reflect.Uint32, "ydb_uint_t*"},
        "int64":   {reflect.Int64, "ydb_int64_t*"},
        "uint64":  {reflect.Uint64, "ydb_uint64_t*"},
        "float32": {reflect.Float32, "ydb_float_t*"},
        "float64": {reflect.Float64, "ydb_double_t*"},
}

var returnTypes map[string]struct{} = map[string]struct{}{
        "":        {},
        "string":  {},
        "int":     {},
        "int64":   {},
        "float64": {},
}

// CallTable stores internal metadata used for calling M and a table of Go functions by name loaded from the M call-in table.
type CallTable struct {
        handle   C.uintptr_t // handle used to access the call table
        Filename string
        YDBTable string // Table after pre-processing the M-call table into YDB format
        // List of M routine metadata structs: one for each routine imported.
        Routines map[string]*RoutineData
}

// typeSpec stores the specification for the return value and each parameter of an M routine call.
type typeSpec struct {
        pointer bool         // true if the parameter contains * and is thus passed by reference
        alloc   int          // preallocation size for this parameter
        typ     string       // type string
        kind    reflect.Kind // store kind of type
}

// routineCInfo is YottaDB's C descriptors for a routine.
// Note: these cannot be merged into RoutineData because a pointer to them must be passed to AddCleanup() as a single pointer.
type routineCInfo struct {
        nameDesc *C.ci_name_descriptor // descriptor for M routine with fastpath for calls after the first
}

// RoutineData stores info used to call an M routine.
type RoutineData struct {
        Name          string     // Go name for the M routine
        Entrypoint    string     // the point to be called in M code
        Types         []typeSpec // stores type specification for the routine's return value and each parameter
        preallocation int        // sum of all types.preallocation -- avoids having to calculate this in callM()
        // metadata:
        Table *CallTable    // call table used to find this routine name
        cinfo *routineCInfo // stores YottaDB's C descriptors for the routine
}

var callingM sync.Mutex // Mutex for access to ydb_ci_tab_switch() so table doesn't switch again before fetching ci_name_descriptor routine info

// parseType parses Go type of form [*]type[preallocation] (e.g. *string[100]) into a typeSpec struct.
// Return typeSpec struct
func parseType(typeStr string) (*typeSpec, error) {
        typeStr = regexp.MustCompile(`\s*`).ReplaceAllString(typeStr, "") // remove spaces, e.g., between <type> and '*'
        pattern := regexp.MustCompile(`^(\*?)(\w*)(\*?)([0-9\[\]]*)$`)
        m := pattern.FindStringSubmatch(typeStr)
        if m == nil {
                return nil, errorf(ydberr.MCallTypeUnhandled, "does not match a YottaDB call-in table type specification")
        }
        asterisk, typ, badAsterisk, allocStr := m[1], m[2], m[3], m[4]
        // Check that user didn't accidentally place * after the retType like he would in C
        if badAsterisk == "*" {
                return nil, errorf(ydberr.MCallBadAsterisk, "should not have asterisk at the end")
        }
        pointer := asterisk == "*"
        pattern = regexp.MustCompile(`\[(\d+)\]`)
        m = pattern.FindStringSubmatch(allocStr)
        preallocation := -1
        if m != nil {
                if n, ok := strconv.Atoi(m[1]); ok == nil {
                        preallocation = n
                }
        }
        if preallocation == -1 && typ == "string" && pointer {
                return nil, errorf(ydberr.MCallPreallocRequired, "[preallocation] must be supplied after *string type")
        }
        if preallocation != -1 && typ != "string" {
                return nil, errorf(ydberr.MCallPreallocInvalid, "[preallocation] should not be supplied for number type")
        }
        if preallocation != -1 && !pointer {
                return nil, errorf(ydberr.MCallPreallocInvalid, "[preallocation] should not be specified for non-pointer type because it is not an output from M")
        }
        if preallocation == -1 {
                preallocation = 0 // default to zero if not supplied
        }
        if _, ok := typeMapper[typ]; !ok {
                return nil, errorf(ydberr.MCallTypeUnknown, "invalid type")
        }
        kind := typeMapper[typ].kind
        return &typeSpec{pointer, preallocation, typ, kind}, nil
}

// ParsePrototype parses one line of the ydb call-in file format.
//   - line is the text in the current line of the call-in file
//
// Return *RoutineData containing routine name and Go function that wraps the M routine, and some metadata.
// Note that the returned RoutineData.table is not filled in (nil) as this function does not know the table.
// Blank or pure comment lines return nil.
func parsePrototype(line string) (*RoutineData, error) {
        // Remove comments and ignore blank lines
        line = regexp.MustCompile(`[/]/.*`).ReplaceAllString(line, "")
        if regexp.MustCompile(`^\s*$`).MatchString(line) {
                return nil, nil // return nil if blank line
        }

        // Example prototype line: test_Run: *string[100] %Run^test(*string, int64, int64)
        // Go playground to test with a regex akin to this is at: https://go.dev/play/p/8-e53CpcagC
        // Note: this allows * before or after retType so parseType() can produce a specific error later for incorrectly placing it after retType
        pattern := regexp.MustCompile(`\s*([^:\s]+)\s*:\s*(\*?\s*?[\w_]*\s*?\*?\s*?\[?\s*?[\d]*\s*?\]?)(\s*)([^(\s]+)\s*\(([^)]*)\)`)
        m := pattern.FindStringSubmatch(line)
        if m == nil {
                return nil, errorf(ydberr.MCallInvalidPrototype, "line does not match prototype format 'Go_name: [ret_type] M_entrypoint([*]type, [*]type, ...)'")
        }
        name, retType, space, entrypoint, params := m[1], m[2], m[3], m[4], m[5]

        // Fix up case where the user specified no return type and part of retType captured part of entrypoint
        if space == "" {
                entrypoint = retType + entrypoint // anything captured in retType is part of entrypoint
                retType = ""
        }

        // Check for a valid M entrypoint. It contain only %^@+ and alphanumeric characters.
        // Here I only check valid characters. YottaDB will produce an error in case of incorrect positioning of those characters
        if !regexp.MustCompile(`^[%^@+0-9a-zA-Z]+$`).MatchString(entrypoint) {
                return nil, errorf(ydberr.MCallEntrypointInvalid, "entrypoint (%s) to call M must contain only alphanumeric and %%^@+ characters", entrypoint)
        }

        // Create list of types for return value and then each parameter
        _retType := retType
        if _retType != "" && !strings.Contains(retType, "*") {
                // treat retType as if it were a pointer type since it has to receive back a value
                _retType = "*" + _retType
        }
        typ, err := parseType(_retType)
        if err != nil {
                err.(*Error).Message = fmt.Sprintf("return type (%s) %s", retType, err)
                return nil, err
        }
        if _, ok := returnTypes[typ.typ]; !ok {
                return nil, errorf(ydberr.MCallTypeUnknown, "invalid return type %s (must be string, int, int64, or float64)", retType)
        }
        if strings.Contains(retType, "*") {
                return nil, errorf(ydberr.MCallTypeMismatch, "return type (%s) must not be a pointer type", retType)
        }
        types := []typeSpec{*typ}
        if len(strings.TrimSpace(params)) > 0 { // because Go's Split function awkwardly yields a single string if the input is empty
                for i, typeStr := range regexp.MustCompile(`[,\)]`).Split(params, -1) {
                        typ, err = parseType(typeStr)
                        if err != nil {
                                err.(*Error).Message = fmt.Sprintf("parameter %d (%s) %s", i+1, typeStr, err)
                                return nil, err
                        }
                        if typ.typ == "" {
                                return nil, errorf(ydberr.MCallTypeMissing, "parameter %d is empty but should contain a type on", i+1)
                        }
                        types = append(types, *typ)
                }
        }

        // Iterate types again to calculate total preallocation.
        // Avoids having to calculate this each time callM is called.
        preallocation := 0
        for _, typ := range types {
                preallocation += typ.alloc // add user's explicit string preallocation
        }

        // Make sure we aren't trying to send too many parameters.
        // The -1 is because the return value doesn't count against YDB_MAX_PARMS.
        if len(types)-1 > int(C.YDB_MAX_PARMS) {
                return nil, errorf(ydberr.MCallTooManyParameters, "number of parameters %d exceeds YottaDB maximum of %d", len(types)-1, int(C.YDB_MAX_PARMS))
        }

        // Create routine struct
        nameDesc := (*C.ci_name_descriptor)(calloc(C.sizeof_ci_name_descriptor)) // must use our calloc, not malloc: see calloc doc
        nameDesc.rtn_name.address = C.CString(name)                              // Allocates new memory (released by AddCleanup above
        nameDesc.rtn_name.length = C.ulong(len(name))
        cinfo := routineCInfo{nameDesc}
        routine := RoutineData{name, entrypoint, types, preallocation, nil, &cinfo}
        // Queue the cleanup function to free it
        runtime.AddCleanup(&cinfo, func(nameDesc *C.ci_name_descriptor) {
                // free string data in namedesc first
                C.free(unsafe.Pointer(nameDesc.rtn_name.address))
                C.free(unsafe.Pointer(nameDesc))
        }, cinfo.nameDesc)

        return &routine, nil
}

// Import loads a call-in table for use by this connection only.
// The M routines listed in the call-in 'table' (specified below) are each wrapped in a Go function which may be subsequently
// called using the returned [MFunctions.Call](name) or referenced as a Go function using [MFunctions.Wrap](name).
//
// If 'table' string contains ":" it is considered to be the call-in table specification itself; otherwise it is treated as the filename of a call-in file to be opened and read.
//
// # M-call table format specification
//
// An M-call table specifies M routines which may be called by Go.
// It may be a string or a file (typically a file with extension .mcalls) and is case-sensitive.
// The format of an M-call table is a sequence of text lines where each line contains an M routine prototype specifications as follows:
//
//        Go_name: [ret_type] M_entrypoint(type, type, ...)
//
// Elements of that line are defined as follows:
//   - Go_name may be any go string.
//   - M_entrypoint is any valid [M entry reference].
//   - ret_type may be omitted if an M return value is not supplied. Otherwise it must be *string, *int, *int64, *float64 or omitted (for void return)
//   - any spaces adjacent to commas, asterisk and square brackets (,*[]) are ignored.
//
// Zero or more parameter type specifications are allowed and must be a Go type specifier: string, int, uint, int32, uint32, int64, uint64, float32, float64,
// or a pointer version of the same in Go type format (e.g. *int).
//   - If a pointer type is selected then the parameter is passed by reference so that the M routine can modify the parameter.
//   - Any *string types and string return values must be followed by a preallocation value in square brackets (e.g. *string[100]).
//
// This allows Go to preallocate enough space for the returned string. If necessary, YottaDB will truncate returned strings so they fit.
//
// Comments begin with // and continue to the end of the line. Blank lines or pure-comment lines are ignored.
//
// [M entry reference]: https://docs.yottadb.com/ProgrammersGuide/langfeat.html#entry-references
func (conn *Conn) Import(table string) (*MFunctions, error) {
        var tbl CallTable
        cconn := conn.cconn

        // Open and read M-call table so we can preprocess it into a YottaDB-format call-in table
        prototypes := []byte(table)
        if !strings.Contains(table, ":") {
                tbl.Filename = table
                var err error
                prototypes, err = os.ReadFile(table)
                if err != nil {
                        return nil, errorf(ydberr.ImportRead, "could not read call-in table file '%s': %s", table, err)
                }
        }

        // Process `prototypes` ci-table ourselves to get routine names and types.
        // Do this after ydb has processed the file to let ydb catch any errors in the table.
        var routines = make(map[string]*RoutineData)
        for i, line := range bytes.Split(prototypes, []byte{'\n'}) {
                routine, err := parsePrototype(string(line))
                if err != nil {
                        err := err.(*Error)
                        return nil, newError(err.Code, fmt.Sprintf("%s line %d: %s", err, i+1, bytes.TrimSpace(line)), newError(ydberr.ImportParse, "")) // wrap ImportError under err
                }
                if routine == nil {
                        continue
                }
                routine.Table = &tbl
                routines[routine.Name] = routine
        }
        tbl.Routines = routines

        // Create new prototype table in YottaDB-format for output to call-in file
        var b strings.Builder
        for _, routine := range routines {
                var ydbTypes []string
                for i, typ := range routine.Types {
                        ydbType := typeMapper[typ.typ].ydbType
                        if typ.typ == "string" && dbHandle.YDBRelease < 1.36 {
                                // Make ydb <1.36 always use 'ydb_string_t', since it doesn't have 'ydb_buffer_t'.
                                // It doesn't work as well for IO parameters because output length cannot be longer than input value,
                                // but at least it will maintain backward compatibility for any apps that previously used YDB <1.36
                                ydbType = "ydb_string_t*"
                        }
                        // Add * for pointer types unless the ydb type already inherently has a * (e.g. strings)
                        if typ.pointer && !strings.HasSuffix(ydbType, "*") {
                                // Coverage tests will not run this line because typeMapper currently always maps to pointer types
                                // to keep the logic simple. But leave this line here in case someone changes typeMapper in future.
                                ydbType = ydbType + "*"
                        }
                        // Add IO: or I: for all parameters (not for retval, though).
                        if i > 0 {
                                if typ.pointer {
                                        ydbType = "IO:" + ydbType
                                } else {
                                        ydbType = "I:" + ydbType
                                }
                        }
                        ydbTypes = append(ydbTypes, ydbType)
                }
                fmt.Fprintf(&b, "%s: %s %s(%s)\n", routine.Name, ydbTypes[0], routine.Entrypoint, strings.Join(ydbTypes[1:], ", "))
        }
        tbl.YDBTable = strings.Trim(b.String(), "\n")

        // Now create a ydb version of the call-in table without any preallocation specs (which YDB doesn't currently support)
        f, err := os.CreateTemp("", "YDBGo_callins_*.ci")
        if err != nil {
                // Not in coverage test because it should never fail since we've just written the file
                return nil, errorf(ydberr.ImportTemp, "could not open temporary call-in table file '%s': %s", f.Name(), err)
        }
        if DebugMode.Load() >= 1 { // In debug modes retain YDB-format temporary file for later inspection
                log.Printf("Temporary call-in table file is: %s\n", f.Name())
        } else {
                defer os.Remove(f.Name())
        }
        _, err = f.WriteString(tbl.YDBTable)
        f.Close()
        if err != nil {
                // Not in coverage test because it should never fail
                return nil, errorf(ydberr.ImportTemp, "could not write temporary call-in table file '%s': %s", f.Name(), err)
        }

        // Tell YottaDB to process the call-in table
        cstr := C.CString(f.Name())
        defer C.free(unsafe.Pointer(cstr))
        handle := (*C.uintptr_t)(C.malloc(C.sizeof_uintptr_t))
        defer C.free(unsafe.Pointer(handle))
        conn.prepAPI()
        status := C.ydb_ci_tab_open_t(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, cstr, handle)
        tbl.handle = *handle
        if status != YDB_OK {
                // Not in coverage test because it should never fail since Import creates a valid call-in table
                err := conn.lastError(status).(*Error)
                return nil, newError(err.Code, fmt.Sprintf("%s while processing call-in table:\n%s\n", err, tbl.YDBTable), newError(ydberr.ImportOpen, ""))
        }

        mfunctions := MFunctions{&tbl, conn}
        return &mfunctions, nil
}

const paramSize = max(C.sizeof_uintptr_t, C.sizeof_ydb_string_t, C.sizeof_ydb_buffer_t)

// paramAlloc allocates and returns parameter block if it hasn't already been allocated.
func (conn *Conn) paramAlloc() unsafe.Pointer {
        cconn := conn.cconn
        // Lazily allocate param block only if needed by callM
        if cconn.paramBlock != nil {
                return cconn.paramBlock
        }
        // Allocate enough space for every parameter to be the maximum size (a string buffer). +1 for return value.
        // Spaces for the actual strings is allocated separately (if necessary) in callM.
        size := paramSize * (C.YDB_MAX_PARMS + 1)
        cconn.paramBlock = calloc(C.size_t(size)) // must use our calloc, not malloc: see calloc doc
        // Note this gets freed by conn cleanup
        return cconn.paramBlock
}

// callM calls M routines.
//   - args are the parameters to pass to the M routine. In the current version, they are converted to
//     strings using fmt.Sprintf("%v") but this may change in future for improved efficiency.
//
// Return value is nil if the routine is not defined to return anything.
func (conn *Conn) callM(routine *RoutineData, args []any) (any, error) {
        if routine == nil {
                panic(errorf(ydberr.MCallNil, "routine data passed to Conn.CallM() must not be nil"))
        }
        cconn := conn.cconn

        if len(args) != len(routine.Types)-1 {
                panic(errorf(ydberr.MCallWrongNumberOfParameters, "%d parameters supplied whereas the M-call table specifies %d", len(args), len(routine.Types)-1))
        }
        printEntry("CallTable.CallM()")
        // If we haven't already fetched the call description from YDB, do that now.
        if routine.cinfo.nameDesc.handle == nil {
                // Lock out other instances of ydb_ci_tab_switch() so table doesn't switch again before fetching ci_name_descriptor routine info.
                // Release lock once we've called routine for the first time and populated ci_name_desriptor.
                callingM.Lock()
                defer callingM.Unlock()

                // Allocate a C storage place for ydb to store handle.
                oldhandle := (*C.uintptr_t)(C.malloc(C.sizeof_uintptr_t))
                defer C.free(unsafe.Pointer(oldhandle))
                conn.prepAPI()
                status := C.ydb_ci_tab_switch_t(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, routine.Table.handle, oldhandle)
                if status != YDB_OK {
                        // Not in coverage test because it's hard to know how to make it fail
                        return "", conn.lastError(status)
                }
                // On return, restore table handle since we changed it. Ignore errors.
                // This is commented out because it triggers a YottaDB bug which complains if environment variable
                // ydb_ci/CTMCI is not set even if it has already called the relevant M routine and thus already has
                // table data for it. The bug is slated for fixing in YDB at https://gitlab.com/YottaDB/DB/YDB/-/issues/1160
                //defer C.ydb_ci_tab_switch_t(conn.tptoken, &cconn.errstr, *oldhandle, oldhandle)
        }

        // Add each parameter to the vararg list required to call ydb_cip_t()
        conn.vpStart() // restart parameter list
        conn.vpAddParam64(conn.tptoken.Load())
        conn.vpAddParam(uintptr(unsafe.Pointer(&cconn.errstr)))
        conn.vpAddParam(uintptr(unsafe.Pointer(routine.cinfo.nameDesc)))

        // Calculate how much to preallocate = routine.preallocation + size of non-preallocated strings passed this call
        preallocation := routine.preallocation
        for _, arg := range args {
                switch val := arg.(type) {
                case string:
                        preallocation += len(val)
                        // Note: *string case is already pre-allocated and included in routine.preallocation
                }
        }

        // Allocate enough space for all parameters and any string preallocations.
        // Do this with a single malloc for speed, or don't do it at all if conn.value is large enough to accommodate us.
        paramBlock := conn.paramAlloc() // allocated in conn for in subsequent calls
        var prealloc unsafe.Pointer
        // Use statically-available connection space if prealloction fits within its maximum; otherwise malloc
        if preallocation < YDB_MAX_STR {
                conn.ensureValueSize(preallocation)
                prealloc = unsafe.Pointer(cconn.value.buf_addr)
        } else {
                prealloc = C.malloc(C.size_t(preallocation)) // no need to use calloc as it is only for string data: see calloc doc
                defer C.free(prealloc)
        }

        param := paramBlock // start param at beginning of paramBlock
        allotStr := func(alloc, length int) {
                length = min(alloc, length)
                if dbHandle.YDBRelease < 1.36 {
                        // Make ydb <1.36 always use 'ydb_string_t', since it doesn't have 'ydb_buffer_t'.
                        // It doesn't work as well for IO parameters because output length cannot be longer than input value,
                        // but at least it will maintain backward compatibility for any apps that previously used YDB <1.36
                        str := (*C.ydb_string_t)(param)
                        str.length = C.ulong(length)
                        str.address = (*C.char)(prealloc)
                } else {
                        buf := (*C.ydb_buffer_t)(param)
                        buf.len_used = C.uint(length)
                        buf.len_alloc = C.uint(alloc)
                        buf.buf_addr = (*C.char)(prealloc)
                }
                prealloc = unsafe.Add(prealloc, alloc)
        }

        // If there is a return value, store it first in the paramBlock space
        if routine.Types[0].typ != "" {
                typ := routine.Types[0]
                if typ.kind == reflect.String {
                        allotStr(typ.alloc, typ.alloc)
                }
                conn.vpAddParam(uintptr(unsafe.Pointer(param)))
                param = unsafe.Add(param, paramSize)
        }
        // Now store each parameter into the allocated paramBlock space and load it into our variadic parameter list
        for i, typ := range routine.Types[1:] {
                pointer := false
                // typeAssert() assistant function: check arg type against supplied type.
                // This retains speed as it only uses reflect.TypeOf in the case of errors.
                typeAssert := func(val any, kind reflect.Kind) {
                        if typ.kind == kind && typ.pointer == pointer {
                                return
                        }
                        // The following lines not in coverage test because the current design uses pointer types for all return values.
                        asterisk := ""
                        if typ.pointer {
                                asterisk = "*"
                        }
                        panic(errorf(ydberr.MCallTypeMismatch, "parameter %d is %s but %s%s is specified in the M-call table", i+1, reflect.TypeOf(val), asterisk, typ.typ))
                }
                switch val := args[i].(type) {
                case string:
                        typeAssert(val, reflect.String)
                        C.memcpy(prealloc, unsafe.Pointer(unsafe.StringData(val)), C.size_t(len(val)))
                        allotStr(len(val), len(val))
                case *string:
                        pointer = true
                        typeAssert(val, reflect.String)
                        length := len(*val)
                        // Produce error rather than truncate if string does not fit into preallocated space
                        if length > typ.alloc {
                                return nil, errorf(ydberr.InvalidStringLength, "parameter %d (%d bytes) is larger than the preallocation size of %d", i+1, length, typ.alloc)
                        }
                        C.memcpy(prealloc, unsafe.Pointer(unsafe.StringData(*val)), C.size_t(length))
                        allotStr(typ.alloc, length)
                case int:
                        typeAssert(val, reflect.Int)
                        *(*C.ydb_long_t)(param) = C.ydb_long_t(val)
                case *int:
                        pointer = true
                        typeAssert(val, reflect.Int)
                        *(*C.ydb_long_t)(param) = C.ydb_long_t(*val)
                case uint:
                        typeAssert(val, reflect.Uint)
                        *(*C.ydb_ulong_t)(param) = C.ydb_ulong_t(val)
                case *uint:
                        pointer = true
                        typeAssert(val, reflect.Uint)
                        *(*C.ydb_ulong_t)(param) = C.ydb_ulong_t(*val)
                case int32:
                        typeAssert(val, reflect.Int32)
                        *(*C.ydb_int_t)(param) = C.ydb_int_t(val)
                case *int32:
                        pointer = true
                        typeAssert(val, reflect.Int32)
                        *(*C.ydb_int_t)(param) = C.ydb_int_t(*val)
                case uint32:
                        typeAssert(val, reflect.Uint32)
                        *(*C.ydb_uint_t)(param) = C.ydb_uint_t(val)
                case *uint32:
                        pointer = true
                        typeAssert(val, reflect.Uint32)
                        *(*C.ydb_uint_t)(param) = C.ydb_uint_t(*val)
                case int64:
                        typeAssert(val, reflect.Int64)
                        *(*C.ydb_int64_t)(param) = C.ydb_int64_t(val)
                case *int64:
                        pointer = true
                        typeAssert(val, reflect.Int64)
                        *(*C.ydb_int64_t)(param) = C.ydb_int64_t(*val)
                case uint64:
                        typeAssert(val, reflect.Uint64)
                        *(*C.ydb_uint64_t)(param) = C.ydb_uint64_t(val)
                case *uint64:
                        pointer = true
                        typeAssert(val, reflect.Uint64)
                        *(*C.ydb_uint64_t)(param) = C.ydb_uint64_t(*val)
                case float32:
                        typeAssert(val, reflect.Float32)
                        *(*C.ydb_float_t)(param) = C.ydb_float_t(val)
                case *float32:
                        pointer = true
                        typeAssert(val, reflect.Float32)
                        *(*C.ydb_float_t)(param) = C.ydb_float_t(*val)
                case float64:
                        typeAssert(val, reflect.Float64)
                        *(*C.ydb_double_t)(param) = C.ydb_double_t(val)
                case *float64:
                        pointer = true
                        typeAssert(val, reflect.Float64)
                        *(*C.ydb_double_t)(param) = C.ydb_double_t(*val)
                default:
                        panic(errorf(ydberr.MCallTypeUnhandled, "unhandled type (%s) in parameter %d", reflect.TypeOf(val), i+1))
                }
                conn.vpAddParam(uintptr(unsafe.Pointer(param)))
                param = unsafe.Add(param, paramSize)
        }

        // vplist now contains the parameter list we want to send to ydb_cip_t(). But CGo doesn't permit us
        // to call or even get a function pointer to ydb_cip_t(). So call it via getfunc_ydb_cip_t().
        status := conn.vpCall(C.getfunc_ydb_cip_t()) // call ydb_cip_t()
        if status != YDB_OK {
                return nil, conn.lastError(status)
        }

        // Go through the parameters again to locate the pointer parameters and copy their values back into Go space
        param = paramBlock
        fetchStr := func() string {
                if dbHandle.YDBRelease < 1.36 {
                        // Make ydb <1.36 always use 'ydb_string_t', since it doesn't have 'ydb_buffer_t'.
                        // It doesn't work as well for IO parameters because output length cannot be longer than input value,
                        // but at least it will maintain backward compatibility for any apps that previously used YDB <1.36
                        str := (*C.ydb_string_t)(param)
                        return C.GoStringN(str.address, C.int(str.length))
                } else {
                        buf := (*C.ydb_buffer_t)(param)
                        return C.GoStringN(buf.buf_addr, C.int(buf.len_used))
                }
        }
        // If there is a return value, fetch it first from the paramBlock space
        var retval any
        if routine.Types[0].typ != "" {
                typ := routine.Types[0]
                switch typ.kind {
                case reflect.String:
                        retval = fetchStr()
                case reflect.Int:
                        ptr := (*C.ydb_long_t)(param)
                        retval = int(*ptr)
                case reflect.Int64:
                        ptr := (*C.ydb_int64_t)(param)
                        retval = int64(*ptr)
                case reflect.Float64:
                        ptr := (*C.ydb_double_t)(param)
                        retval = float64(*ptr)
                default:
                        // Not in coverage test because it should never fail
                        panic(errorf(ydberr.MCallTypeUnhandled, "unhandled type (%s) in return of return value; contact YottaDB support", typ.typ))
                }
                param = unsafe.Add(param, paramSize)
        }
        // Now fill each pointer parameter from the paramBlock space
        for i, typ := range routine.Types[1:] {
                if typ.pointer {
                        switch val := args[i].(type) {
                        case *string:
                                *val = fetchStr()
                        case *int:
                                ptr := (*C.ydb_long_t)(param)
                                *val = int(*ptr)
                        case *uint:
                                ptr := (*C.ydb_ulong_t)(param)
                                *val = uint(*ptr)
                        case *int32:
                                ptr := (*C.ydb_int_t)(param)
                                *val = int32(*ptr)
                        case *uint32:
                                ptr := (*C.ydb_uint_t)(param)
                                *val = uint32(*ptr)
                        case *int64:
                                ptr := (*C.ydb_int64_t)(param)
                                *val = int64(*ptr)
                        case *uint64:
                                ptr := (*C.ydb_uint64_t)(param)
                                *val = uint64(*ptr)
                        case *float32:
                                ptr := (*C.ydb_float_t)(param)
                                *val = float32(*ptr)
                        case *float64:
                                ptr := (*C.ydb_double_t)(param)
                                *val = float64(*ptr)
                        case string, int, uint, int32, uint32, int64, uint64, float32, float64:
                        default:
                                // Not in coverage test because it should never fail
                                panic(errorf(ydberr.MCallTypeUnhandled, "unhandled type (%s) in parameter %d; contact YottaDB support", reflect.TypeOf(val), i+1))
                        }
                }
                param = unsafe.Add(param, paramSize)
        }
        runtime.KeepAlive(conn) // ensure conn sticks around until we've finished copying data from it's C paramblock
        return retval, nil
}

// MustImport is the same as [Conn.Import] but panics on errors.
func (conn *Conn) MustImport(table string) *MFunctions {
        mfunctions, err := conn.Import(table)
        if err != nil {
                panic(err)
        }
        return mfunctions
}

//////////////////////////////////////////////////////////////////
//
// Copyright (c) 2025-2026 YottaDB LLC and/or its subsidiaries.
// All rights reserved.
//
//        This source code contains the intellectual property
//        of its copyright holder(s), and is made available
//        under a license.  If you do not know the terms of
//        the license, please stop and do not read further.
//
//////////////////////////////////////////////////////////////////

// Define Node type for access to YottaDB database

package yottadb

import (
        "bytes"
        "iter"
        "runtime"
        "strconv"
        "strings"
        "time"
        "unsafe"

        "lang.yottadb.com/go/yottadb/v2/ydberr"
)

/* #include "libyottadb.h"
#include "yottadb.h"

// It's tempting to apply `#cgo nocallback` directives to all ydb_* C functions (1% call speed increase - tested),
// but this is wrong because ydb_* function can call the Go function signalExitCallback() when a signal occurs.
*/
import "C"

// indexBuf finds the address of index i within a ydb_buffer_t array.
//   - This function is necessary because CGo discards Node.cnode.buffersn[] since it has no size.
func bufferIndex(buf *C.ydb_buffer_t, i int) *C.ydb_buffer_t {
        return (*C.ydb_buffer_t)(unsafe.Add(unsafe.Pointer(buf), C.sizeof_ydb_buffer_t*i))
}

// ---- Node object

// Node is an object containing strings that represents a YottaDB node.
//   - Stores all the supplied strings (varname and subscripts) in the Node object along with array of C.ydb_buffer_t
//     structs that point to each successive string, to provide fast access to YottaDB API functions.
//   - Regular Nodes are immutable. There is a mutable version of Node emitted by [Node.Next]() and Node iterators, which
//     will change each loop. If you need to take an immutable snapshot of a mutable node this may be done with [Node.Clone]().
//   - Concurrency: Do not run database actions on node objects created in another goroutine. If you want to
//     act on a node object passed in from another goroutine, first call [Node.Clone](conn) to make a copy of the
//     other goroutine's node object using the current goroutine's connection `conn`. Then perform methods on that.
//
// Node methods panic on errors because they are are all panic-worthy (e.g. invalid variable names).
// See [yottadb.Error] for error strategy and rationale.
type Node struct {
        // Node type wraps a C.node struct in a Go struct so Go can add methods to it.
        // Pointer to C.node rather than the item itself so we can point to it from C without Go moving it.
        cnode    *C.node
        Conn     *Conn // Node.Conn points to the Go conn; Node.cnode.conn will point directly to the C.conn
        original *Node // if this node is mutable, points to the originating node
        cap      int   // capacity of buffers stored in mutation[0] node which may be >cnode.len; used only by Index() for mutation0
        // List of mutated children of this node: one for each subscript depth indexed, or nil if no mutations exist yet
        mutations [](*Node)
}

// Convert []string to []any.
// Used to pass string arrays to Node
func stringArrayToAnyArray(strings []string) []any {
        array := make([]any, len(strings))
        for i, s := range strings {
                array[i] = s
        }
        return array
}

// _Node creates a `Node` type instance that represents a database node with methods that access YottaDB.
//   - The strings and array are stored in C-allocated space to give Node methods fast access to YottaDB API functions.
//   - The varname and subscripts may be of type string, []byte slice, or an integer or float type; numeric types are converted to a string using the appropriate strconv function.
//   - The varname may also be another node, in which case that node's subscript strings will be prepended to `subscripts` to build the new node.
func (conn *Conn) _Node(varname any, subscripts []any) (n *Node) {
        // Note: benchmarking shows that the use of any slows down node creation almost immeasurably (< 0.1%)
        // Concatenate strings the fastest Go way.
        // This involves creating an extra copy of subscripts but is probably faster than one C.memcpy call per subscript
        var joiner bytes.Buffer
        var firstLen int   // length of concatenated subscripts in varname
        var firstCount int // number of subscripts in varname
        var node1 *Node    // if varname is a node, store it in here
        subs := make([]string, len(subscripts))
        switch val := varname.(type) {
        case *Node:
                node1 = val
                cnode := node1.cnode
                firstCount = int(cnode.len)
                if node1.IsMutable() {
                        // if mutable, there may be gaps between strings due to overallocation, so remove gaps by concatenating each string individually
                        for i := range int(cnode.len) {
                                buffer := bufferIndex(cnode.buffers, i)
                                // unsafe.String gives fast temporary access to C data as if it were a string without copying the data to a string first
                                joiner.WriteString(unsafe.String((*byte)(unsafe.Pointer(buffer.buf_addr)), buffer.len_used))
                        }
                        firstLen = joiner.Len()
                } else {
                        // if immutable, we can copy all subscripts in a single lump as there are no gaps between strings
                        firstStringAddr := cnode.buffers.buf_addr
                        lastbuf := bufferIndex(cnode.buffers, firstCount-1)
                        // Calculate data size of all strings in node to copy: address of last string - address of first string + length of last string
                        datasize := C.uint(uintptr(unsafe.Pointer(lastbuf.buf_addr))) - C.uint(uintptr(unsafe.Pointer(firstStringAddr)))
                        datasize += lastbuf.len_used
                        // unsafe.String gives fast temporary access to C data as if it were a string without copying the data to a string first
                        joiner.WriteString(unsafe.String((*byte)(unsafe.Pointer(firstStringAddr)), datasize))
                        firstLen = int(datasize)
                }
        default:
                first := anyToString(val)
                joiner.WriteString(first)
                firstLen = len(first)
                firstCount = 1
        }
        for i, s := range subscripts {
                subs[i] = anyToString(s)
                joiner.WriteString(subs[i])
        }

        size := C.sizeof_node + C.sizeof_ydb_buffer_t*(firstCount+len(subscripts)) + joiner.Len()
        n = &Node{}
        cnode := (*C.node)(calloc(C.size_t(size))) // must use our calloc, not malloc: see calloc doc
        n.cnode = cnode
        // Queue the cleanup function to free it
        runtime.AddCleanup(n, func(cnode *C.node) {
                C.free(unsafe.Pointer(cnode))
        }, cnode)

        n.Conn = conn
        cnode.conn = conn.cconn // point to the C version of the conn
        cnode.len = C.int(len(subscripts) + firstCount)
        cnode.buffers = (*C.ydb_buffer_t)(unsafe.Add(unsafe.Pointer(cnode), C.sizeof_node))

        dataptr := unsafe.Pointer(bufferIndex(cnode.buffers, len(subscripts)+firstCount))
        if joiner.Len() > 0 {
                // Note: have tried to replace the following with copy() to avoid a CGo invocation, but it's slower
                C.memcpy(dataptr, unsafe.Pointer(&joiner.Bytes()[0]), C.size_t(joiner.Len()))
        }

        // Function to set each buffer to string[dataptr++] as I loop through strings
        setbuf := func(buf *C.ydb_buffer_t, length C.uint) {
                buf.buf_addr = (*C.char)(dataptr)
                buf.len_used, buf.len_alloc = length, length
                dataptr = unsafe.Add(dataptr, length)
        }

        // Now fill in ydb_buffer_t pointers
        if node1 != nil {
                // First set buffers for all strings copied from parent node
                for i := range firstCount {
                        buf := bufferIndex(cnode.buffers, i)
                        setbuf(buf, bufferIndex(node1.cnode.buffers, i).len_used)
                }
                runtime.KeepAlive(node1) // ensure node1 sticks around until we've finished copying data from it's C allocation
        } else {
                buf := bufferIndex(cnode.buffers, 0)
                setbuf(buf, C.uint(firstLen))
        }
        for i, s := range subs {
                buf := bufferIndex(cnode.buffers, i+firstCount)
                setbuf(buf, C.uint(len(s)))
        }
        return n
}

// Node method creates a `Node` type instance that represents a database node with methods that access YottaDB.
//   - The strings and array are stored in C-allocated space to give Node methods fast access to YottaDB API functions.
//   - The varname and subscripts may be of type string, []byte slice, or an integer or float type; numeric types are converted to a string using the appropriate strconv function.
func (conn *Conn) Node(varname string, subscripts ...any) (n *Node) {
        return conn._Node(varname, subscripts)
}

// CloneNode creates a copy of node associated with conn (in case node was created using a different conn).
// A node associated with a conn used by another goroutine must not be used by the current goroutine except as
// a parameter to CloneNode(). If this rule is not obeyed, then the two goroutines could overwrite each others transaction
// level and error message values. It is the programmer's responsibility to ensure this does not happen by using CloneNode.
// This does the same as n.Clone() except that it can switch to new conn.
// Mutable nodes passed to CloneNode will cause a panic. They must be converted to immutable nodes with Node.Clone()
// Only immutable nodes are returned.
func (conn *Conn) CloneNode(n *Node) *Node {
        if n.IsMutable() {
                panic(errorf(ydberr.InvalidMutableOperation, "mutable Node (%s) must not be cloned", n))
        }
        return conn._Node(n, nil)
}

// Child creates a child node of parent that represents parent with subscripts appended.
//   - [Node.Clone]() without parameters is equivalent to [Node.Child]() without parameters.
func (n *Node) Child(subscripts ...any) (child *Node) {
        return n.Conn._Node(n, subscripts)
}

// Clone creates an immutable copy of node.
//   - [Node.Clone]() is equivalent to calling [Node.Child]() without parameters.
//
// See [Node.IsMutable]() for notes on mutability.
func (n *Node) Clone() (clone *Node) {
        return n.Conn._Node(n, nil)
}

// _subscript is an implementation of Subscript() but without the bounds check or negative index access (for internal use by Index, to fetch cached depth strings)
func (n *Node) _subscript(index int) string {
        cnode := n.cnode // access C.node from Go node
        buf := bufferIndex(cnode.buffers, index)
        r := C.GoStringN(buf.buf_addr, C.int(buf.len_used))
        runtime.KeepAlive(n) // ensure n sticks around until we've finished copying data from it's C allocation
        return r
}

// Subscript returns a string that holds the specified varname or subscript of the given node.
// An index of zero returns the varname; higher numbers return the respective subscript.
// A negative index returns a subscript counted from the end (the last is -1).
// An out-of-range subscript panics.
func (n *Node) Subscript(index int) string {
        cnode := n.cnode // access C.node from Go node
        if index < 0 {
                index = int(cnode.len) + index
        }
        if index < 0 || index >= int(cnode.len) {
                panic(errorf(ydberr.InvalidSubscriptIndex, "subscript %d out of bounds (0-%d)", index, cnode.len))
        }
        return n._subscript(index)
}

// Subscripts returns a slice of strings that represent the varname and subscript names of the given node.
func (n *Node) Subscripts() []string {
        cnode := n.cnode // access C.node from Go node
        strings := make([]string, cnode.len)
        for i := range cnode.len {
                buf := bufferIndex(cnode.buffers, int(i))
                s := C.GoStringN(buf.buf_addr, C.int(buf.len_used))
                strings[i] = s
        }
        runtime.KeepAlive(n) // ensure n sticks around until we've finished copying data from it's C allocation
        return strings
}

// String returns a string representation of this database node in typical YottaDB format: `varname("sub1")("sub2")`.
//   - Output subscripts as unquoted numbers if they convert to float64 and back without change (using [Node.Quote]).
//   - Output strings in YottaDB ZWRITE format
func (n *Node) String() string {
        var bld strings.Builder
        subs := n.Subscripts()
        for i, s := range subs {
                if i == 0 {
                        bld.WriteString(s)
                        continue
                }
                if i == 1 {
                        bld.WriteString("(")
                }
                bld.WriteString(n.Conn.Quote(s))
                if i == len(subs)-1 {
                        bld.WriteString(")")
                } else {
                        bld.WriteString(",")
                }
        }
        runtime.KeepAlive(n) // ensure n sticks around until we've finished copying data from it's C allocation
        return bld.String()
}

// GoString makes print("%#v") dump the node and its contents with [Node.Dump](30, 80).
// For example the output format of Node("person",42).GoString() might look like this:
//
//        person(42)=1234
//        person(42)("age")="49"
//        person(42)("height")("centimeters")=190.5
//        person(42)("height")("inches")=1234
//        person(42)("name")="Joe Bloggs"
func (n *Node) GoString() string {
        return n.Dump(30, 80)
}

// Dump returns a string representation of this database node and subtrees with their contents in YottaDB ZWRITE format.
// For example output, see [Node.GoString]. Output lines are formatted as follows:
//   - Output subscripts and values as unquoted numbers if they convert to float64 and back without change (using [Node.Quote]).
//   - Output strings in YottaDB ZWRITE format.
//   - Every line output ends with "\n", including the final line.
//   - If the receiver is nil, output is "<nil>\n"
//   - If node has no value and no children, outputs the empty string.
//
// See [Node.GoString] for an example of the output format.
// Two optional integers may be supplied to specify maximums where both default to -1:
//   - first parameter specifies the maximum number of lines to output (not including a file "...\n" line indicating truncation). A maximum of -1 means infinite.
//     If lines are truncated, an additional line "...\n" is added so that the output ends with "\n...\n". A maximum of 0 lines is treated as 1.
//   - second parameter specifies the maximum number of characters at which to truncate values prior to output where -1 means infinite.
//     Truncated values are output with suffix "..." after any ending quotes. Note that conversion to ZWRITE format may expand this.
func (n *Node) Dump(args ...int) string {
        if len(args) > 2 {
                panic(errorf(ydberr.TooManyParameters, "%d parameters supplied to Dump() which only takes 2", len(args)))
        }
        args = append(args, -1, -1) // defaults
        maxLines, maxString := args[0], args[1]
        if maxLines == 0 {
                maxLines = 1 // This ensures ending is always "\n...\n" whenever lines are truncated
        }
        if n == nil {
                return "<nil>\n"
        }

        var bld strings.Builder
        // local func to output one line of the tree
        dumpLine := func(node *Node, val string) {
                bld.WriteString(node.String())
                bld.WriteString("=")
                if maxString != -1 && len(val) > maxString {
                        bld.WriteString(node.Conn.Quote(val[:maxString]))
                        bld.WriteString("...")
                } else {
                        bld.WriteString(node.Conn.Quote(val))
                }
                bld.WriteString("\n")
        }

        lines := 0
        val, ok := n.Lookup()
        if ok {
                lines++
                dumpLine(n, val)
        }
        for node := range n.Tree() {
                val, ok = node.Lookup()
                if !ok {
                        // Node subscript was deleted while iterating it, so don't print that subscript
                        continue
                }
                lines++
                if maxLines != -1 && lines > maxLines {
                        bld.WriteString("...\n")
                        break
                }
                dumpLine(node, val)
        }
        return bld.String()
}

// Set applies val to the value of a database node.
//   - The val may be a string, []byte slice, or an integer or float type; numeric types are converted to a string using the appropriate strconv function.
func (n *Node) Set(val any) {
        cnode := n.cnode // access C equivalents of Go types
        cconn := cnode.conn
        n.Conn.setAnyValue(val)
        n.Conn.prepAPI()
        status := C.ydb_set_st(C.uint64_t(n.Conn.tptoken.Load()), &cconn.errstr, cnode.buffers, cnode.len-1, bufferIndex(cnode.buffers, 1), &cconn.value)
        if status != YDB_OK {
                panic(n.Conn.lastError(status))
        }
}

// Get fetches and returns the value of a database node or defaultValue[0] if the database node is empty.
//   - defaultValue defaults to {""}
//
// Return defaultValue[0] if the node's value does not exist.
func (n *Node) Get(defaultValue ...string) string {
        ok := n._Lookup()
        if !ok {
                if len(defaultValue) == 0 {
                        return ""
                }
                return defaultValue[0]
        }
        cconn := n.cnode.conn
        // copy cconn.value into a Go type so that cconn.value can be re-used for another ydb call
        r := C.GoStringN(cconn.value.buf_addr, C.int(cconn.value.len_used))
        runtime.KeepAlive(n) // ensure n sticks around until we've finished copying data from it's C allocation
        return r
}

// GetInt fetches and returns the value of a database node as an integer.
//   - defaultValue defaults to {0}
//
// Return defaultValue[0] if the node's value does not exist or is not convertable to an integer.
func (n *Node) GetInt(defaultValue ...int) int {
        val := n.Get()
        num, err := strconv.ParseInt(val, 10, 0)
        if err != nil {
                if _, ok := err.(*strconv.NumError); !ok {
                        panic(err) // unknown error unrelated to conversion
                }
                if len(defaultValue) == 0 {
                        return 0
                }
                return defaultValue[0]
        }
        return int(num)
}

// GetBool fetches and returns the value of a database node as a bool.
//   - defaultValue defaults to {false}
//
// Return true if the node's value is a non-zero integer; false if it is integer zero; otherwise return defaultValue[0] (nonexistant, or is not convertable to an integer)
func (n *Node) GetBool(defaultValue ...bool) bool {
        val := n.Get()
        num, err := strconv.ParseInt(val, 10, 0)
        if err != nil {
                if _, ok := err.(*strconv.NumError); !ok {
                        panic(err) // unknown error unrelated to conversion
                }
                if len(defaultValue) == 0 {
                        return false
                }
                return defaultValue[0]
        }
        if num == 0 {
                return false
        }
        return true
}

// GetFloat fetches and returns the value of a database node as a float64.
//   - defaultValue defaults to {0}
//
// Return defaultValue[0] if the node's value does not exist or is not convertable to float64.
func (n *Node) GetFloat(defaultValue ...float64) float64 {
        val := n.Get()
        num, err := strconv.ParseFloat(val, 64)
        if err != nil {
                if _, ok := err.(*strconv.NumError); !ok {
                        panic(err) // unknown error unrelated to conversion
                }
                if len(defaultValue) == 0 {
                        return 0
                }
                return defaultValue[0]
        }
        return num
}

// GetBytes is the same as [Node.Get] except that it accepts and returns []byte slices rather than strings.
func (n *Node) GetBytes(defaultValue ...[]byte) []byte {
        ok := n._Lookup()
        if !ok {
                if len(defaultValue) == 0 {
                        return []byte{}
                }
                return defaultValue[0]
        }
        cconn := n.cnode.conn
        // copy cconn.value into a Go type so that cconn.value can be re-used for another ydb call
        r := C.GoBytes(unsafe.Pointer(cconn.value.buf_addr), C.int(cconn.value.len_used))
        runtime.KeepAlive(n) // ensure n sticks around until we've finished copying data from it's C allocation
        return r
}

// Lookup returns the value of a database node and true, or if the variable name could not be found, returns the empty string and false.
//   - If the node's variable name (M local or M global) exists but the subscripted node has no value, Lookup() will return the empty string and true.
//     If you need to distinguish between an empty string and a value-less node you must use [Node.HasValue]()
//   - bool false is returned on errors GVUNDEF (undefined M global) or LVUNDEF (undefined M local). Other errors panic.
//   - You may use [Node.Get]() to return a default value when an undefined variable is accessed.
func (n *Node) Lookup() (string, bool) {
        ok := n._Lookup()
        if !ok {
                return "", false
        }
        cconn := n.cnode.conn
        // copy cconn.value into a Go type so that cconn.value can be re-used for another ydb call
        r := C.GoStringN(cconn.value.buf_addr, C.int(cconn.value.len_used))
        runtime.KeepAlive(n) // ensure n sticks around until we've finished copying data from it's C allocation
        return r, true
}

// LookupBytes is the same as [Node.Lookup] except that it returns the value as a []byte slice rather than a string.
func (n *Node) LookupBytes() ([]byte, bool) {
        ok := n._Lookup()
        if !ok {
                return []byte{}, false
        }
        cconn := n.cnode.conn
        // copy cconn.value into a Go type so that cconn.value can be re-used for another ydb call
        r := C.GoBytes(unsafe.Pointer(cconn.value.buf_addr), C.int(cconn.value.len_used))
        runtime.KeepAlive(n) // ensure n sticks around until we've finished copying data from it's C allocation
        return r, true
}

// _Lookup returns the value of a database node in n.cconn.value and returns whether the variable name could be found.
func (n *Node) _Lookup() bool {
        cnode := n.cnode // access C equivalents of Go types
        cconn := cnode.conn
        n.Conn.prepAPI()
        status := C.ydb_get_st(C.uint64_t(n.Conn.tptoken.Load()), &cconn.errstr, cnode.buffers, cnode.len-1, bufferIndex(cnode.buffers, 1), &cconn.value)
        if status == ydberr.INVSTRLEN {
                // Allocate more space and retry the call
                n.Conn.ensureValueSize(int(cconn.value.len_used))
                n.Conn.prepAPI()
                status = C.ydb_get_st(C.uint64_t(n.Conn.tptoken.Load()), &cconn.errstr, cnode.buffers, cnode.len-1, bufferIndex(cnode.buffers, 1), &cconn.value)
        }
        if status == ydberr.GVUNDEF || status == ydberr.LVUNDEF {
                return false
        }
        if status != YDB_OK {
                panic(n.Conn.lastError(status))
        }
        return true
}

// data returns whether the database node has a value or subnodes as follows:
//   - 0: node has neither a value nor a subtree, i.e., it is undefined.
//   - 1: node has a value, but no subtree
//   - 10: node has no value, but does have a subtree
//   - 11: node has both value and subtree
//
// It is private because it really isn't a nice name.
func (n *Node) data() int {
        cnode := n.cnode // access C equivalents of Go types
        cconn := cnode.conn
        var val C.uint
        n.Conn.prepAPI()
        status := C.ydb_data_st(C.uint64_t(n.Conn.tptoken.Load()), &cconn.errstr, cnode.buffers, cnode.len-1, bufferIndex(cnode.buffers, 1), &val)
        if status != YDB_OK {
                panic(n.Conn.lastError(status))
        }
        return int(val)
}

// HasValue returns whether the database node has a value.
func (n *Node) HasValue() bool {
        return (n.data() & 1) == 1
}

// HasValueOnly returns whether the database node has a value but no tree.
func (n *Node) HasValueOnly() bool {
        return n.data() == 1
}

// HasTree returns whether the database node has a tree of subscripts containing data.
func (n *Node) HasTree() bool {
        return (n.data() & 10) == 10
}

// HasTreeOnly returns whether the database node has no value but does have a tree of subscripts that contain data
func (n *Node) HasTreeOnly() bool {
        return n.data() == 10
}

// HasBoth returns whether the database node has both tree and value.
func (n *Node) HasBoth() bool {
        return (n.data() & 11) == 11
}

// HasNone returns whether the database node has neither tree nor value.
func (n *Node) HasNone() bool {
        return (n.data() & 11) == 0
}

// Kill deletes a database node including its value and any subtree.
//   - To delete only the value of a node use [Node.Clear]()
func (n *Node) Kill() {
        cnode := n.cnode // access C equivalents of Go types
        cconn := cnode.conn
        n.Conn.prepAPI()
        status := C.ydb_delete_st(C.uint64_t(n.Conn.tptoken.Load()), &cconn.errstr, cnode.buffers, cnode.len-1, bufferIndex(cnode.buffers, 1), C.YDB_DEL_TREE)
        if status != YDB_OK {
                panic(n.Conn.lastError(status))
        }
}

// Clear deletes the node value, not its child subscripts.
//   - Equivalent to YottaDB M command ZKILL
func (n *Node) Clear() {
        cnode := n.cnode // access C equivalents of Go types
        cconn := cnode.conn
        n.Conn.prepAPI()
        status := C.ydb_delete_st(C.uint64_t(n.Conn.tptoken.Load()), &cconn.errstr, cnode.buffers, cnode.len-1, bufferIndex(cnode.buffers, 1), C.YDB_DEL_NODE)
        if status != YDB_OK {
                panic(n.Conn.lastError(status))
        }
}

// Incr atomically increments the value of database node by amount.
//   - The amount may be an integer, float or string representation of the same.
//   - YottaDB first converts the value of the node to a number by discarding any trailing non-digits and returning zero if it is still not a number.
//     Then it adds amount to the node, all atomically.
//   - Return the new value of the node as a string
func (n *Node) Incr(amount any) string {
        cnode := n.cnode // access C equivalents of Go types
        cconn := cnode.conn

        n.Conn.setAnyValue(amount)
        if cconn.value.len_used == 0 {
                panic(errorf(ydberr.IncrementEmpty, `cannot increment by the empty string ""`))
        }

        n.Conn.prepAPI()
        status := C.ydb_incr_st(C.uint64_t(n.Conn.tptoken.Load()), &cconn.errstr, cnode.buffers, cnode.len-1, bufferIndex(cnode.buffers, 1), &cconn.value, &cconn.value)
        if status != YDB_OK {
                panic(n.Conn.lastError(status))
        }

        valuestring := C.GoStringN(cconn.value.buf_addr, C.int(cconn.value.len_used))
        runtime.KeepAlive(n) // ensure n sticks around until we've finished copying data from it's C allocation
        return valuestring
}

// Lock attempts to acquire or increment the count of a lock matching this node, waiting up to timeout for availability.
// Equivalent to the M `LOCK +lockpath` command.
//   - If no timeout is supplied, wait forever. A timeout of zero means try only once.
//   - Return true if lock was acquired; otherwise false.
//   - Panics with TIME2LONG if the timeout exceeds YDB_MAX_TIME_NSEC or on other panic-worthy errors (e.g. invalid variable names).
func (n *Node) Lock(timeout ...time.Duration) bool {
        cnode := n.cnode // access C equivalents of Go types
        cconn := cnode.conn

        forever := len(timeout) == 0
        var timeoutNsec C.ulonglong
        if forever {
                timeoutNsec = YDB_MAX_TIME_NSEC
        } else {
                timeoutNsec = C.ulonglong(timeout[0].Nanoseconds())
        }

        for {
                n.Conn.prepAPI()
                status := C.ydb_lock_incr_st(C.uint64_t(n.Conn.tptoken.Load()), &cconn.errstr, timeoutNsec, cnode.buffers, cnode.len-1, bufferIndex(cnode.buffers, 1))
                if status == YDB_OK {
                        return true
                }
                if status == C.YDB_LOCK_TIMEOUT && !forever {
                        return false
                }
                if status != YDB_OK {
                        panic(n.Conn.lastError(status))
                }
        }
}

// Unlock decrements the count of a lock matching this node, releasing it if zero.
// Equivalent to the M `LOCK -lockpath` command.
//   - Returns nothing since releasing a lock cannot fail.
func (n *Node) Unlock() {
        cnode := n.cnode // access C equivalents of Go types
        cconn := cnode.conn

        n.Conn.prepAPI()
        status := C.ydb_lock_decr_st(C.uint64_t(n.Conn.tptoken.Load()), &cconn.errstr, cnode.buffers, cnode.len-1, bufferIndex(cnode.buffers, 1))
        if status != YDB_OK {
                panic(n.Conn.lastError(status))
        }
}

// stubNode creates a minimal mutable node that points to the buffers of another node.
// Used by Index().
func (conn *Conn) stubNode(original *Node, buffers *C.ydb_buffer_t, len int) (n *Node) {
        size := C.sizeof_node
        cnode := (*C.node)(calloc(C.size_t(size))) // must use our calloc, not malloc: see calloc doc
        cnode.conn = conn.cconn
        cnode.len = C.int(len)
        cnode.buffers = buffers
        n = &Node{}
        n.cnode = cnode
        n.Conn = conn
        n.original = original
        // Queue the cleanup function to free it
        runtime.AddCleanup(n, func(cnode *C.node) {
                C.free(unsafe.Pointer(cnode))
        }, cnode)
        return n
}

// This subscript string is used to preallocate subscript string space in mutable nodes that may have their final subscript name changed.
// When it is exceeded by subsequent Node.Index() iterations, the entire node must be cloned to achieve reallocation.
var preallocSubscript string = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
var preallocMutationDepth int = 5 // May expand on-the-fly up to YDB_MAX_SUBS

// Index allows fast temporary access to subnodes referenced by the given subscripts, but must be used with caution.
// It is most helpful for high-speed inner loops. If in doubt, use the slower [Node.Child].
// It indexes a node object with the given subscripts and returns a mutable node object that will change next time Index is invoked
// on the same parent node object (which is thread-safe because each Goroutine has separate node objects).
// Fast access is achieved by removing the overhead of creating a new child node object every access, e.g. each time around a loop.
// [Node.Children], for example, yields mutable nodes created by Index().
//
// *Caution*: for non-temporary access, the slower [Node.Child] should be used instead. For example, where nodes will be passed to subroutines
// (subroutines should be able to assume they have been passed immutable nodes).
// Node.Index must also be avoided when you wish a Go variable to retain a pointer to a specific node after another use of Node.Index.
// The second example below illustrates incorrect usage.
//
//   - Returns a mutable child of the given node with the given subscripts appended (see [Node.IsMutable] for mutability details).
//
// Usage is similar to [Node.Child] except that it typically runs about 4 times as fast and returns mutable instead of immutable nodes.
func (n *Node) Index(subscripts ...any) *Node {
        // This works by allocating a new mutable node (stored in Node.mutations[0]) and several stub nodes that use its buffers.
        // Stub nodes are stored in Node.mutations[1..n] for each depth of indexed subscripts (cf. 1. below).
        // The first node in Node.mutations[0] is a full node. Subsequent mutations point back to its subscript strings (cf. 2. below).
        //
        // 1. The reason a mutation/stub is required for each depth is illustrated by the programmer doing:
        //   for person := range n.Children() {
        //     name := person.Index("name").Get()
        //     age := person.Index("age").GetInt()
        //   }
        // Note that person is already a mutable node yielded by Children().
        // Without separate nodes for separate depths, the first Index() will append "name" to the mutable node person.
        // The second Index() will thus produce person("name","age") instead of person("age").
        //
        // 2. The reason subs in Node.mutations[1..n] subscript strings point back to mutations[0] is to prevent a bug if they had their own strings.
        // Using the same example above, in the second iteration of the FOR loop person.Index("name") will use
        // the mutable node for depth 2 which will still have its person subscript set to its previous value because it
        // was only incremented in the depth 1 mutable node.

        // Speed up the common case where only one subscript is supplied.
        // (faster partly because there's no need to create newsubs array for coverted strings)
        if len(subscripts) == 1 {
                return n.index1(subscripts[0])
        }
        if len(subscripts) == 0 {
                panic(errorf(ydberr.SubscriptRequired, "Index() method requires at least one subscript as a parameter"))
        }

        // Cast new subscripts to strings
        newsubs := make([]string, len(subscripts))
        for i, sub := range subscripts {
                newsubs[i] = anyToString(sub)
        }

        // Calculate depth of index from original parent node
        original := n // originating parent node from which index was taken
        if n.IsMutable() {
                original = n.original
        }
        originalLen := int(original.cnode.len)
        // indexing depth: number of subscripts the mutation adds to the *original* node
        depth := int(n.cnode.len) - originalLen + len(newsubs)
        if originalLen-1+depth > YDB_MAX_SUBS { // -1 for varname
                panic(errorf(ydberr.InvalidSubscriptIndex, "attempt to Index() node %s exceeded YDB_MAX_SUBS", n))
        }

        // Check whether an existing mutation[0] node has space for these new subscripts -- or we need to reallocate
        nLen := int(n.cnode.len)
        if original.mutations == nil || original.mutations[0] == nil || original.mutations[0].cap < nLen+len(newsubs) {
                return n.reallocateMutation(newsubs)
        }
        buffers := original.mutations[0].cnode.buffers
        for i, sub := range newsubs {
                space := int(bufferIndex(buffers, nLen+i).len_alloc)
                if space < len(sub) {
                        return n.reallocateMutation(newsubs)
                }
        }
        // No need to reallocate, so just store the new subscripts into the mutated node
        for i, sub := range newsubs {
                buffer := bufferIndex(buffers, nLen+i)
                C.memcpy(unsafe.Pointer(buffer.buf_addr), unsafe.Pointer(unsafe.StringData(sub)), C.size_t(len(sub)))
                buffer.len_used = C.uint(len(sub))
        }
        return original.mutations[depth-1]
}

// index1 is the same as Index except that it operates faster in the common case when only a single parameter is given.
// It is automatically invoked by Index() when that case is true.
func (n *Node) index1(subscript any) *Node {
        // Cast new subscript to string
        sub := anyToString(subscript)

        // Calculate depth of index from original parent node
        var depth int // indexing depth: number of subscripts the mutation adds to the *original* node
        original := n // originating parent node from which index was taken
        if n.IsMutable() {
                original = n.original
        }
        originalLen := int(original.cnode.len)
        depth = int(n.cnode.len) - originalLen + 1
        if originalLen-1+depth > YDB_MAX_SUBS { // -1 for varname
                panic(errorf(ydberr.InvalidSubscriptIndex, "attempt to Index() node %s exceeded YDB_MAX_SUBS", n))
        }

        // Check whether an existing mutation[0] node has space for this new subscript -- or we need to reallocate
        nLen := int(n.cnode.len)
        if original.mutations == nil || original.mutations[0] == nil || original.mutations[0].cap < nLen+1 {
                return n.reallocateMutation([]string{sub})
        }
        buffers := original.mutations[0].cnode.buffers
        space := int(bufferIndex(buffers, nLen).len_alloc)
        if space < len(sub) {
                return n.reallocateMutation([]string{sub})
        }
        // No need to reallocate, so just store the new subscript into the mutated node
        buffer := bufferIndex(buffers, nLen)
        C.memcpy(unsafe.Pointer(buffer.buf_addr), unsafe.Pointer(unsafe.StringData(sub)), C.size_t(len(sub)))
        buffer.len_used = C.uint(len(sub))
        return original.mutations[depth-1]
}

// reallocateMutation reallocates mutation0 when necessary for use by Index().
// This is only called after determining that reallocation really is necessary
func (n *Node) reallocateMutation(newsubs []string) *Node {
        // See explanation of how this works in the comment at the beginning of Index()
        original := n // originating parent node from which index was taken
        if n.IsMutable() {
                original = n.original
        }
        originalLen := int(original.cnode.len)
        // indexing depth: number of subscripts the mutation adds to the *original* node
        depth := int(n.cnode.len) - originalLen + len(newsubs)
        if original.mutations == nil {
                original.mutations = make([]*Node, 1, preallocMutationDepth)
        }
        mutation0 := original.mutations[0]
        // calculate max # subscripts = nLen + any more used previously on this mutable node
        maxDepth := depth
        if mutation0 != nil {
                maxDepth = max(maxDepth, mutation0.cap-originalLen)
        }
        subs := make([]any, 0, maxDepth)
        // Add indexes that n has before newsubs
        for i := originalLen; i < int(n.cnode.len); i++ {
                // append preallocSubscript to each allocated subscript so that we don't have to reallocate small increases
                subs = append(subs, n.Subscript(i)+preallocSubscript)
        }
        // Add new subscripts
        for _, sub := range newsubs {
                // append preallocSubscript to each allocated subscript so that we don't have to reallocate small increases
                subs = append(subs, sub+preallocSubscript)
        }
        // Add any surplus indexes above newsubs that were previously used on mutation0 even though they are not needed for this particular indexing operation
        // i.e. never shrink mutation0.cap
        for len(subs) < maxDepth {
                subs = append(subs, mutation0._subscript(originalLen+len(subs)))
        }
        mutation0 = n.Conn._Node(original, subs)
        mutation0.original = original                // indicate new node is mutable
        mutation0.cap = int(mutation0.cnode.len)     // actual buffer capacity of mutation0
        mutation0.cnode.len = C.int(originalLen + 1) // shrink mutation[0] node to present just one index subscript (its index depth=1)
        buffers := mutation0.cnode.buffers
        // remove preallocSubscript over-allocation from the end of each subscript
        for i := range depth {
                bufferIndex(buffers, originalLen+i).len_used -= C.uint(len(preallocSubscript))
        }
        original.mutations[0] = mutation0 // store mutation0
        // Re-point all stub buffers to the new mutation0 buffers
        for i := 1; i < len(original.mutations); i++ {
                original.mutations[i].cnode.buffers = buffers
        }
        // Create any stubs needed up to depth
        for i := len(original.mutations); i < depth; i++ {
                stub := n.Conn.stubNode(original, buffers, originalLen+i+1)
                original.mutations = append(original.mutations, stub)
        }
        return original.mutations[depth-1]
}

// IsMutable returns whether given node is mutable.
// Mutable nodes may have their subscripts changed each loop iteration or each call to Node.Index(). This means that
// the same mutable node object may reference different database nodes and will not always point to the same one.
//   - Mutable nodes are returned by [Node.Index], [Node.Next], [Node.Prev] and iterator [Node.Children].
//   - All standard node methods are valid operations on a mutable node except conn.CloneNode() which will panic.
//   - If an immutable copy of a mutable node is required, use [Node.Clone] or [Node.Child].
//   - If you need to take an immutable snapshot of a mutable node this may be done with [Node.Clone] or [Node.Child].
//
// A mutable node object is like a regular node object except that it will change to point to a different database
// node each time [Node.Index] is invoked on its originating node object n, so if you store a reference to it,
// that reference will no longer point to the same database node as it originally did. For example, the following
// code will print the most recent vehicle, not the heaviest vehicle as intended, because [Node.Children]() yields
// mutable nodes.
//
//        n := conn.Node("vehicles")
//        var heaviest *yottadb.Node
//        var maxWeight float64 = 0
//        for vehicle := range n.Children() {
//          if vehicle.Index("weight").GetFloat() > maxWeight {
//            heaviest = vehicle
//            maxWeight = vehicle.Index("weight")
//          }
//        }
//        fmt.Print(heaviest.Dump())
func (n *Node) IsMutable() bool {
        return n.original != nil
}

// _next returns the name of the next subscript at the same depth level as the given node.
// This implements the logic for both Next() and Prev().
//   - If the parameter reverse is true, fetch the next node in reverse order, i.e. Prev().
//   - bool returns with false only if there are no more subscripts
//
// See further documentation at Next().
func (n *Node) _next(reverse bool) (string, bool) {
        cnode := n.cnode // access C equivalents of Go types
        conn := n.Conn
        cconn := cnode.conn

        var status C.int
        for range 2 {
                n.Conn.prepAPI()
                if reverse {
                        status = C.ydb_subscript_previous_st(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, cnode.buffers, cnode.len-1, bufferIndex(cnode.buffers, 1), &cconn.value)
                } else {
                        status = C.ydb_subscript_next_st(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, cnode.buffers, cnode.len-1, bufferIndex(cnode.buffers, 1), &cconn.value)
                }
                if status == ydberr.INVSTRLEN {
                        // Allocate more space and retry the call
                        n.Conn.ensureValueSize(int(cconn.value.len_used))
                        continue
                }
                break
        }

        if status == ydberr.NODEEND {
                return "", false
        }
        if status != YDB_OK {
                panic(n.Conn.lastError(status))
        }
        r := C.GoStringN(cconn.value.buf_addr, C.int(cconn.value.len_used))
        runtime.KeepAlive(n) // ensure n sticks around until we've finished copying data from it's C allocation
        return r, true
}

// _Next implements both Next and Prev based on the reverse parameter.
// It returns a mutable Node object pointing to the next subscript (unlike _next() which returns the next subscript as a string).
func (n *Node) _Next(reverse bool) *Node {
        next, ok := n._next(reverse)
        if !ok {
                return nil
        }
        if n.cnode.len <= 1 {
                // Cannot index a node with no varname, so just return a new top-level immutable node with next as subscript
                return n.Conn.Node(next)
        }
        var parent *Node
        if n.IsMutable() {
                original := n.original // get the original immutable parent
                depth := int(n.cnode.len - original.cnode.len)
                // find parent node or parent index so I can index that
                if depth < 2 {
                        parent = original
                } else {
                        parent = original.mutations[(depth-1)-1]
                }
        } else {
                // If there is no mutable parent index, create a parent node with one less subscript than `n`,
                // then use that to index for fast iteration
                subs := n.Subscripts()
                parent = n.Conn._Node(subs[0], stringArrayToAnyArray(subs[1:max(1, len(subs)-1)]))
        }
        return parent.Index(next)
}

// Next returns a Node instance pointing to the next subscript at the same depth level.
// Return a mutable node pointing to the database node with the next subscript after the given node, at the same depth level.
// Unless you want to start half way through a sequence of subscripts, it's usually tidier to use Node.Iterate() instead.
//   - Equivalent to the M function [$ORDER()] and has the same treatment of 'null subscripts' (i.e. empty strings).
//   - The order of returned nodes matches the collation order of the M database.
//   - The node path supplied does not need to exist in the database to find the next match.
//   - If the supplied node n contains only a variable name without subscripts, the next variable (GLVN) name is returned instead of the next subscript.
//
// Returns nil when there are no more subscripts at the level of the supplied node path, or a mutable node as follows:
//   - if the supplied node is immutable, a mutable clone of n with its final subscript changed to the next node.
//   - if the supplied node is mutable, the same node n with its final subscript changed to the next node.
//
// If you need to take an immutable snapshot of the returned mutable node this may be done with [Node.Clone]()
//
// See:
//   - Compare [Node.Prev]()
//   - [Node.Iterate]() for an iterator version and [Node.TreeNext]() for traversal of nodes in a way that descends into the entire tree.
//
// [$ORDER()]: https://docs.yottadb.com/ProgrammersGuide/functions.html#order
func (n *Node) Next() *Node {
        return n._Next(false)
}

// Prev is the same as Next but operates in the reverse order.
// See [Node.Next]()
func (n *Node) Prev() *Node {
        return n._Next(true)
}

// _children returns an interator that can FOR-loop through all a node's single-depth child subscripts.
// This implements the logic for both Children() and ChildrenBackward().
//   - If the parameter reverse is true, iterate nodes in reverse order.
//
// See further documentation at Iterate().
func (n *Node) _children(reverse bool) iter.Seq2[*Node, string] {
        // Ensure our mutable access to this node doesn't clobber the caller's use of Index() on this node.
        // In theory the caller should know that it might since we've documented that Children() uses Index(),
        // but better to be safe even though it means creation of one extra node every FOR loop
        first := n.Clone()
        n = first.Index("")
        return func(yield func(*Node, string) bool) {
                for {
                        next, ok := n._next(reverse)
                        if !ok {
                                return
                        }
                        n = first.Index(next)
                        if !yield(n, next) {
                                return
                        }
                }
        }
}

// Children returns an interator over immediate child nodes for use in a FOR-loop.
// This iterator is a wrapper for [Node.Next](). It yields two values:
//   - a mutable node instance with final subscripts changed to successive subscript names
//   - the name of the child subscript (optionally assigned with a range statement)
//
// Notes:
//   - Treats 'null subscripts' (i.e. empty strings) in the same way as M function [$ORDER()].
//   - The order of returned nodes matches the collation order of the M database.
//   - This function never adjusts the supplied node even if it is mutable (it always creates its own mutable copy).
//   - If you need to take an immutable snapshot of the returned mutable node, use [Node.Clone]().
//
// See:
//   - [Node.ChildrenBackward]().
//   - [Node.Tree]() for traversal of nodes in a way that descends into the entire tree.
//
// [$ORDER()]: https://docs.yottadb.com/ProgrammersGuide/functions.html#order
func (n *Node) Children() iter.Seq2[*Node, string] {
        return n._children(false)
}

// ChildrenBackward is the same as Children but operates in reverse order.
// See [Node.Children]().
func (n *Node) ChildrenBackward() iter.Seq2[*Node, string] {
        return n._children(true)
}

// _treeNext returns the next node in the traversal of a database tree of a database variable.
// This implements the logic for both TreeNext() and TreePrev().
//   - If the parameter reverse is true, iterate the tree in reverse order.
//
// See further documentation at TreeNext().
func (n *Node) _treeNext(reverse bool) *Node {
        cnode := n.cnode // access C equivalents of Go types
        conn := n.Conn
        cconn := cnode.conn

        // Create new node to store result with a single preallocated child as an initial guess of space needed.
        retNode := n.Child(preallocSubscript)
        var retSubs C.int
        var malloced bool // whether we had to malloc() and hence defer free()
        var status C.int
        for {
                retSubs = retNode.cnode.len - 1 // -1 because cnode counts the varname as a subscript and ydb_node_next_st() does not
                n.Conn.prepAPI()
                if reverse {
                        status = C.ydb_node_previous_st(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, cnode.buffers, cnode.len-1, bufferIndex(cnode.buffers, 1), &retSubs, bufferIndex(retNode.cnode.buffers, 1))
                } else {
                        status = C.ydb_node_next_st(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, cnode.buffers, cnode.len-1, bufferIndex(cnode.buffers, 1), &retSubs, bufferIndex(retNode.cnode.buffers, 1))
                }
                if status == ydberr.INSUFFSUBS {
                        extraStrings := make([]any, retSubs-(retNode.cnode.len-1))
                        // Pre-fill node subscripts
                        for i := range extraStrings {
                                extraStrings[i] = preallocSubscript
                        }
                        retNode = retNode.Child(extraStrings...)
                        continue
                }
                if status == ydberr.INVSTRLEN {
                        buf := bufferIndex(retNode.cnode.buffers, int(retSubs+1)) // +1 because cnode counts the varname as a subscript and ydb_node_next_st() does not
                        len := buf.len_used
                        newbuf := C.malloc(C.size_t(len))
                        malloced = true // flag that we have to clone this node before freeing newbuf
                        defer C.free(newbuf)
                        buf.buf_addr = (*C.char)(newbuf)
                        buf.len_alloc = len
                        continue
                }
                break
        }
        if status == ydberr.NODEEND {
                return nil
        }
        if status != YDB_OK {
                panic(n.Conn.lastError(status))
        }
        retNode.cnode.len = C.int(retSubs + 1) // +1 because cnode counts the varname as a subscript and ydb_node_next_st() does not
        // if we malloced anything, make sure we take a copy of it before defer runs to free the mallocs on return
        if malloced {
                strings := stringArrayToAnyArray(retNode.Subscripts())
                retNode = n.Conn._Node(strings[0], strings[1:])
        }
        return retNode
}

// TreeNext returns the next node in the traversal of a database tree of a database variable.
// Equivalent to the M function [$QUERY()].
//   - It yields immutable nodes.
//   - The next node is chosen in depth-first order (i.e by descending deeper into the subscript tree before moving to the next node at the same level).
//   - The order of returned nodes matches the collation order of the M database.
//   - The node path supplied does not need to exist in the database to find the next match.
//   - Returns nil when there are no more nodes after the given node path within the given database variable (GLVN).
//   - Nodes that have 'null subscripts' (i.e. empty string) are all returned in their place except for the top-level GLVN(""), which is never returned.
//
// See:
//   - [Node.TreePrev]().
//   - [Node.LevelNext]() for traversal of nodes at the same level or to move from one database variable (GLVN) to another.
//
// [$QUERY()]: https://docs.yottadb.com/ProgrammersGuide/functions.html#query
func (n *Node) TreeNext() *Node {
        return n._treeNext(false)
}

// TreePrev is the same as TreeNext but operates in reverse order.
// See [Node.TreeNext]().
func (n *Node) TreePrev() *Node {
        return n._treeNext(true)
}

// Tree returns an interator over all descendants of node for use in a FOR-loop.
// This iterator is a wrapper for [Node.TreeNext](). It yields immutable node instances.
//   - The next node is chosen in depth-first order (i.e by descending deeper into the subscript tree before moving to the next node at the same level).
//   - The order of returned nodes matches the collation order of the M database.
//   - Nodes that have 'null subscripts' (i.e. empty string) are all returned in their place.
//
// See:
//   - [Node.TreeNext](), [Node.TreePrev]()
//   - [Node.Children]() for traversal of only immediate children.
//
// [$ORDER()]: https://docs.yottadb.com/ProgrammersGuide/functions.html#query
func (n *Node) Tree() iter.Seq[*Node] {
        len1 := int(n.cnode.len)
        subs1 := n.Subscripts()

        return func(yield func(*Node) bool) {
                for {
                        n = n.TreeNext()
                        if n == nil || int(n.cnode.len) < len1 {
                                return
                        }
                        // Ensure that returned node is still a descendent of first node; i.e. all initial subscripts match
                        // Don't need to check varname (i=0) as TreeNext() doesn't search beyond varname
                        for i := 1; i < len1; i++ {
                                buf := bufferIndex(n.cnode.buffers, int(i))
                                // Access buf string using unsafe.String because it doesn't make a time-consuming copy of the string like GoStringN does.
                                if unsafe.String((*byte)(unsafe.Pointer(buf.buf_addr)), buf.len_used) != subs1[i] {
                                        return
                                }
                        }
                        if !yield(n) {
                                return
                        }
                }
        }
}

//////////////////////////////////////////////////////////////////
//
// Copyright (c) 2020-2026 YottaDB LLC and/or its subsidiaries.
// All rights reserved.
//
//        This source code contains the intellectual property
//        of its copyright holder(s), and is made available
//        under a license.  If you do not know the terms of
//        the license, please stop and do not read further.
//
//////////////////////////////////////////////////////////////////

// Refer YottaDB-required signals to YottaDB and allow user to be notified of them if desired

package yottadb

import (
        "fmt"
        "log"
        "log/syslog"
        "os"
        "os/signal"
        "runtime"
        "runtime/debug"
        "sync"
        "sync/atomic"
        "syscall"
        "time"

        "lang.yottadb.com/go/yottadb/v2/ydberr"
)

// #include "libyottadb.h"
// extern void *ydb_signal_exit_callback(void);
import "C"

// Shutdown globals
var ydbSigPanicCalled atomic.Bool          // True when our exit is panic driven due to a signal
var ydbShutdownCheck = make(chan struct{}) // Flag that a channel has been shut down. Needs no buffering since we use blocking writes
var shutdownSigGoroutines bool             // Flag that we have completed shutdownSignalGoroutines()
var shutdownSigGoroutinesMutex sync.Mutex  // Serialize access to shutdownSignalGoroutines()

// YDBSignals lists all the signals that YottaDB must be notified of.
var YDBSignals = []os.Signal{
        syscall.SIGABRT,
        syscall.SIGALRM,
        syscall.SIGBUS,
        syscall.SIGCONT,
        syscall.SIGFPE,
        syscall.SIGHUP,
        syscall.SIGILL,
        syscall.SIGINT,
        // syscall.SIGIO - this is a duplicate of SIGURG
        // syscall.SIGIOT - this is a duplicate of SIGABRT
        syscall.SIGQUIT,
        syscall.SIGSEGV,
        syscall.SIGTERM,
        syscall.SIGTRAP,
        syscall.SIGTSTP,
        syscall.SIGTTIN,
        syscall.SIGTTOU,
        syscall.SIGURG,
        syscall.SIGUSR1,
}

// sigInfo holds info for each signal that YDBGo handles and defers to YottaDB.
type sigInfo struct {
        updating     sync.Mutex     // Indicate this struct is being modified
        signal       os.Signal      // signal number for this entry
        notifyChan   chan os.Signal // user-supplied channel to notify of incoming signal
        shutdownNow  chan struct{}  // Channel used to shutdown signal handling goroutine
        shutdownDone atomic.Bool    // indicate that goroutine shutdown is complete
        servicing    atomic.Bool    // indicate signal handler is active
        _conn        *Conn          // not a full Conn; just acts as a place to store errstr for use by this signal's NotifyYDB()
}

// init populates ydbSignalMap -- must occur before starting signal handler goroutines in Init()
var ydbSignalMap sync.Map // stores data for signals that the wrapper handles and passes to YottaDB.
func init() {
        for _, sig := range YDBSignals {
                info := sigInfo{sync.Mutex{}, sig, nil, make(chan struct{}, 1), atomic.Bool{}, atomic.Bool{}, _newConn()}
                ydbSignalMap.Store(sig, &info)
        }
}

// printEntry is a function to print the entry point of the function, when entered, if debugMode >= 1.
func printEntry(funcName string) {
        if DebugMode.Load() >= 1 {
                _, file, line, ok := runtime.Caller(2)
                if ok {
                        log.Println("Entered ", funcName, " from ", file, " at line ", line)
                } else {
                        log.Println("Entered ", funcName)
                }
        }
}

// syslogEntry records the given message in the syslog. Since these are rare or one-time per process type errors
// that get recorded here, we open a new syslog handle each time to reduce complexity of access across goroutines.
func syslogEntry(logMsg string) {
        syslogr, err := syslog.New(syslog.LOG_INFO+syslog.LOG_USER, "[YottaDB-Go-Wrapper]")
        if err != nil {
                panic(errorf(ydberr.Syslog, "syslog.New() failed unexpectedly with error: %s: while reporting error: %s", err, logMsg))
        }
        err = syslogr.Info(logMsg)
        if err != nil {
                panic(errorf(ydberr.Syslog, "syslogr.Info() failed unexpectedly with error: %s: while reporting error: %s", err, logMsg))
        }
        err = syslogr.Close()
        if err != nil {
                panic(errorf(ydberr.Syslog, "syslogr.Close() failed unexpectedly with error: %s: while reporting error: %s", err, logMsg))
        }
}

// lookupYDBSignal returns a pointer to the sigInfo entry related to signal sig.
func lookupYDBSignal(sig os.Signal) *sigInfo {
        value, ok := ydbSignalMap.Load(sig)
        if !ok {
                panic(errorf(ydberr.SignalUnsupported, "The specified signal %d (%v) is not a YottaDB signal so is unsupported for signal notification", sig, sig))
        }
        info := value.(*sigInfo)
        return info
}

// validateYDBSignal verifies that the specified signal is valid for SignalNotify()/SignalReset()
func validateYDBSignal(sig os.Signal) *sigInfo {
        // Verify the supplied signal is one that we support with this function. This list contains all of the signals
        // that the wrapper traps in Init().
        // It is up to the user to know which signals are duplicates of others so if separate handlers
        // are set for say SIGABRT and SIGIOT, whichever handler was set last is the one that gets both signals
        // (because both constants are for the same signal).

        info := lookupYDBSignal(sig)
        return info
}

// SignalNotify relays incoming signals to notifyChan specifically for signals used by YottaDB.
// If SignalNotify is used on a specific signal, the user is then responsible to call [NotifyYDB]() at the start or end
// of their own handler to allow YottaDB to process the signal. The user can revert behaviour to the YDBGo default
// with [SignalReset](), after which YDBGo will once again call [NotifyYDB]() itself.
//   - Users may opt to use the standard library's [Signal.Notify]() instead of this function to be notified of signals, but
//     this will notify them in parallel with YottaDB. However, they must not call Signal.Stop() (see below).
//   - Do not call [Signal.Stop](), [Signal.Ignore]() or [Signal.Reset]() for any of the YottaDB-specific signals
//     unless you understand that it will prevent [NotifyYDB]() from being called, and will affect YottaDB timers
//     or other functionality.
//   - Using SignalNotify to capture SIGSEGV is unreliable. Instead, see standard library function [debug.SetPanicOnFault](true)
//
// Goroutines used to handle signals should defer [ShutdownOnPanic](). YDBGo's own signal handlers already do so.
//
// YottaDB-specific signals are listed in the source in [YDBSignals].
//
// See [YottaDB signals].
//
// [YottaDB signals]: https://docs.yottadb.com/MultiLangProgGuide/programmingnotes.html#signals
func SignalNotify(notifyChan chan os.Signal, signals ...os.Signal) {
        // Do-nothing hack purely to prevent goimport from removing runtime/debug from imports since it's required for the docstring above
        debug.SetPanicOnFault(debug.SetPanicOnFault(false))

        // Although this routine itself does not interact with the YottaDB runtime, use of this routine has an expectation that
        // the runtime is going to handle signals so let's make sure it is initialized.
        initCheck()
        for _, sig := range signals {
                info := validateYDBSignal(sig)
                info.updating.Lock()
                info.notifyChan = notifyChan
                info.updating.Unlock()
        }
}

// SignalReset stops notifying the user of the given signals and reverts to default YDBGo signal behaviour which simply calls [NotifyYDB].
// No error is raised if the signal did not already have a notification request in effect.
func SignalReset(signals ...os.Signal) {
        for _, sig := range signals {
                info := lookupYDBSignal(sig)
                info.updating.Lock()
                info.notifyChan = nil
                info.updating.Unlock()
        }
}

// NotifyYDB calls the YottaDB signal handler for sig.
// If YottaDB deferred handling of the signal, return false; otherwise return true.
// Panic on YottaDB errors.
func NotifyYDB(sig os.Signal) bool {
        value, ok := ydbSignalMap.Load(sig)
        if !ok {
                panic(errorf(ydberr.SignalUnsupported, "goroutine-sighandler: called NotifyYDB with a non-YottaDB signal %d (%v)", sig, sig))
        }
        info := value.(*sigInfo)
        _conn := info._conn

        // Flag that YDB is servicing this signal
        info.servicing.Store(true)
        defer info.servicing.Store(false)

        if DebugMode.Load() >= 2 && sig != syscall.SIGURG {
                // SIGURG happens almost continually, so don't report it.
                log.Printf("goroutine-sighandler: calling YottaDB signal handler for signal %d (%v)\n", sig, sig)
        }
        signum := C.int(sig.(syscall.Signal)) // have to type-assert before converting to an int
        // Note this call to ydb_sig_dispatch() does not pass a tptoken. The reason for this is that inside
        // this routine the tptoken at the time the signal actually occurs is unknown. The ydb_sig_dispatch()
        // routine itself does not need the tptoken nor does anything it calls but we do still need an
        // error buffer in case an error occurs that we need to return.
        rc := C.ydb_sig_dispatch(&_conn.cconn.errstr, signum)
        // Handle errors so user doesn't have to
        switch rc {
        case YDB_OK:
                // Signal handling complete
        case YDB_DEFER_HANDLER:
                // Signal was deferred for some reason
                // Not an error, but the fact is logged
                // Hard to test code coverage for this as I don't know how to make YDB produce this condition
                if DebugMode.Load() >= 2 {
                        log.Printf("goroutine-sighandler: YottaDB deferred signal %d (%v)\n", sig, sig)
                }
                return false
        case ydberr.CALLINAFTERXIT:
                // If CALLINAFTERXIT, we're done - exit goroutine
                shutdownSignalGoroutine(info)
        default: // Some sort of error occurred during signal handling
                // Hard to test code coverage for this as I don't know how to make YDB produce this condition. It is an undocumented function.
                err := _conn.lastError(rc)
                panic(newError(ydberr.SignalHandling, fmt.Sprintf("goroutine_sighandler: error from ydb_sig_dispatch() of signal %d (%v): %s", sig, sig, err), err))
        }
        return true
}

// SignalWasFatal returns whether the currently unwinding panic was caused by a fatal signal like Ctrl-C.
// May be used in a deferred function like [ShutdownOnPanic] to check whether a fatal signal caused the current exit procedure.
func SignalWasFatal() bool {
        return ydbSigPanicCalled.Load()
}

// ShutdownOnPanic should be deferred at the start of every goroutine to ensure that the database is shut down on panic.
// Otherwise a panic that occurs within that goroutine will call [os.Exit] without properly shutting down the database,
// and MUPIP RUNDOWN will need to be run.
//
// Deferring this function has the side benefit of exiting its goroutine silently on ydberr.CALLINAFTERXIT panics if they come from
// a fatal signal panic that has already occurred in another goroutine.
//
//   - YDBGo's signal handlers already ensure shutdown is called if they panic (i.e. they defer [ShutdownOnPanic]).
//
// See: [Shutdown], [SignalWasFatal]
func ShutdownOnPanic() {
        err := recover()
        if err != nil {
                // Ensure database is shut down before panicing to avoid having to run MUPIP RUNDOWN after shutdown
                ShutdownHard(dbHandle)
                // Quit threads without panic only if SIGINT caused the shutdown
                quitAfterSIGINT(err)
                // Otherwise re-panic the err, including any traceback (if it's a YDB error; we don't know how to get stacktraces from other Go errors)
                yerr, ok := err.(*Error)
                if ok {
                        err = newError(yerr.Code, fmt.Sprintf("%s\n\n%s\nWas re-paniced as: %s", err, yerr.stack, err), yerr.chain...)
                }
                panic(err)
        }
}

// quitAfterSIGINT deferred, suppresses ydberr.CALLINAFTERXIT error messages from every goroutine after Ctrl-C is pressed.
// When Ctrl-C is pressed the signal is (by default) passed to YottaDB which shuts down the database.
// If goroutines are still running and access the database, they will panic with code ydberr.CALLINAFTERXIT.
// To silence these many panics and have each goroutine simply exit gracefully, defer quitAfterSIGINT()
// at the start of each goroutine. Then you will get just one panic from the Ctrl-C signal interrupt rather
// than one CALLINAFTERXIT panic per goroutine.
// This is automatically deferred if the user defers [SignalWasFatal].
func quitAfterSIGINT(err any) {
        if ErrorIs(err, ydberr.CALLINAFTERXIT) && SignalWasFatal() {
                // Silently and gracefully exit the goroutine
                // This prevents each and every goroutine from panicing when just one receives a fatal signal.
                runtime.Goexit()
        }
}

// handleSignal is used as a goroutine for each YottaDB-specific signal (listed in YDBSignals).
// It calls NotifyYDB() unless a user has requested notification of that signal using SignalNotify(),
// in which case it will notify the user who must call NotifyYDB().
// info specifies the signal to be handled by this particular goroutine.
func handleSignal(info *sigInfo) {
        defer ShutdownOnPanic()
        sig := info.signal

        // We only need one of each type of signal so buffer depth is 1, but let it queue one additional signal.
        sigchan := make(chan os.Signal, 2)
        // Create fresh channel for shutdown monitoring.
        info.shutdownNow = make(chan struct{}, 1) // Need to buffer only 1 element since shutdownSignalGoroutine() is non-blocking
        // Tell Go to pass this signal to our channel.
        signal.Notify(sigchan, sig)
        if DebugMode.Load() >= 2 {
                log.Printf("goroutine-sighandler: Signal handler initialized for %d (%v)\n", sig, sig)
        }
        wgSigInit.Done() // Signal parent goroutine that we have completed initializing signal handling
        allDone := false
        // Process incoming signals until we get told to stop on channel info.shutdownNow
        for !allDone {
                select {
                case <-sigchan:
                        // Wait for signal notification
                case <-info.shutdownNow:
                        allDone = true // Done if channel has data
                }
                if allDone {
                        break // Got a shutdown request - fall out!
                }

                // See if user asked to be notified of this signal
                info := lookupYDBSignal(sig)
                // Note that for fatal signals, the YDB handler probably won't return as it will (usually) exit,
                // but some fatal signals can be deferred under the right conditions (holding crit, interrupts-disabled-state, etc).
                info.updating.Lock()
                notifyChan := info.notifyChan
                info.updating.Unlock()
                if notifyChan != nil {
                        // Notify user code via the supplied channel
                        if DebugMode.Load() >= 2 {
                                log.Printf("goroutine-sighandler: notifying user-specified channel of signal %d (%v)\n", sig, sig)
                        }
                        // Send to channel without blocking (same as Signal.Notify)
                        select {
                        case notifyChan <- sig: // notify channel of signal, sending it a function to use to notify YDB
                        default:
                        }
                } else {
                        // otherwise just run YDB handler function ourselves since user didn't hook this signal
                        NotifyYDB(sig)
                }
        }
        signal.Stop(sigchan) // No more signal notification for this signal channel
        if DebugMode.Load() >= 2 {
                log.Printf("goroutine-sighandler: exiting goroutine for signal %d (%v)\n", sig, sig)
        }
        info.shutdownDone.Store(true)  // Indicate this channel is closed
        ydbShutdownCheck <- struct{}{} // Notify shutdownSignalGoroutines that it needs to check if all channels closed now
}

// shutdownSignalGoroutine tells the routine for the signal specified by info to shutdown.
// This is Non-blocking.
func shutdownSignalGoroutine(info *sigInfo) {
        // Perform non-blocking send
        select {
        // Wake up signal goroutine and make it exit
        case info.shutdownNow <- struct{}{}:
        default:
        }
}

// shutdownSignalGoroutines is a function to stop the signal handling goroutines used to tell the YDB engine what signals
// have occurred. No signals are recognized by the Go wrapper or YottaDB once this is done. All signal handling reverts to
// Go standard handling.
func shutdownSignalGoroutines() {
        printEntry("shutdownSignalGoroutines")
        shutdownSigGoroutinesMutex.Lock()
        if shutdownSigGoroutines { // Nothing to do if already doing this
                // Hard to coverage-test this because it would require calling shutdownSignalGoroutines() while it's already running, which is a small window
                shutdownSigGoroutinesMutex.Unlock()
                if DebugMode.Load() >= 2 {
                        log.Println("shutdownSignalGoroutines: Bypass shutdownSignalGoroutines as it has already run")
                }
                return
        }
        // Send shutdown signal to each goroutine
        for _, sig := range YDBSignals {
                value, _ := ydbSignalMap.Load(sig)
                shutdownSignalGoroutine(value.(*sigInfo))
        }
        // Wait for the signal goroutines to exit but with a timeout
        doneChan := make(chan struct{}) // Zero-length is OK because we signal it by closing it.
        go func() {
                // Loop handling channel notifications as goroutines shutdown. If we are currently handling a fatal signal
                // like a SIGQUIT, that channel is active but is busy so will not respond to a shutdown request. For this
                // reason, we treat active goroutines the same as successfully shutdown goroutines so we don't delay
                // shutdown. No need to wait for something that is not likely to occur (The YottaDB handlers for fatal signals
                // drive a process-ending panic and never return).
                for {
                        <-ydbShutdownCheck // A goroutine finished - check if all are shutdown or otherwise busy
                        done := true
                        for _, sig := range YDBSignals {
                                value, _ := ydbSignalMap.Load(sig)
                                sigData := value.(*sigInfo)
                                shutdownDone := sigData.shutdownDone.Load()
                                signalActive := sigData.servicing.Load()
                                if !shutdownDone && !signalActive {
                                        // A goroutine is not shutdown and not active so to wait for more
                                        // goroutine(s) to complete, break out of this scan loop
                                        done = false
                                        break
                                }
                        }
                        if done {
                                close(doneChan) // Notify select loop below that this is complete
                                return
                        }
                }
        }()
        select {
        case <-doneChan: // All signal monitoring goroutines are shutdown or are busy!
                if DebugMode.Load() >= 2 {
                        log.Println("shutdownSignalGoroutines: All signal goroutines successfully closed or active")
                }
        case <-time.After(MaxSigShutdownWait):
                // Notify syslog that this timeout happened
                if DebugMode.Load() >= 2 {
                        log.Println("shutdownSignalGoroutines: Timeout! Some signal goroutines did not shutdown")
                }
                syslogEntry("Shutdown of signal goroutines timed out")
        }
        shutdownSigGoroutines = true
        shutdownSigGoroutinesMutex.Unlock()
        // All signal routines should be finished or otherwise occupied
        if DebugMode.Load() >= 2 {
                log.Println("shutdownSignalGoroutines: Channel closings complete")
        }
}

// signalExitCallback is called from C by YottaDB to perform an exit when YottaDB gets a fatal signal.
// Its purpose is to make sure defers get called before exit, which it does by calling panic.
// This function is passed to YottaDB during init by ydb_main_lang_init().
// YDB calls this exit handler after rundown (equivalent to ydb_exit()).
// The sigNum parameter is reported in the panic message.
//
//export signalExitCallback
func signalExitCallback(sigNum C.int) {
        printEntry("YDBWrapperPanic()")
        ydbSigPanicCalled.Store(true) // Need "atomic" usage to avoid read/write DATA RACE issues
        shutdownSignalGoroutines()    // Close the goroutines down with their signal notification channels
        sig := syscall.Signal(sigNum) // Convert numeric signal number to Signal type for use in panic() message
        panic(errorf(ydberr.SignalFatal, "Fatal signal %d (%v) occurred", sig, sig))
}

// SignalExitCallback is an exported version of signalExitCallback() for use only by FatalSignal test
func SignalExitCallback(sigNum os.Signal) {
        signalExitCallback(C.int(sigNum.(syscall.Signal)))
}

//////////////////////////////////////////////////////////////////
//
// Copyright (c) 2025-2026 YottaDB LLC and/or its subsidiaries.
// All rights reserved.
//
//        This source code contains the intellectual property
//        of its copyright holder(s), and is made available
//        under a license.  If you do not know the terms of
//        the license, please stop and do not read further.
//
//////////////////////////////////////////////////////////////////

// Allow Go to perform YottaDB transactions

package yottadb

import (
        "fmt"
        "runtime"
        "runtime/cgo"
        "unsafe"

        "lang.yottadb.com/go/yottadb/v2/ydberr"
)

/* #include "libyottadb.h"
extern int tpCallbackWrapper(uint64_t tptoken, ydb_buffer_t *errstr, void *callback);
int tp_callback_wrapper(uint64_t tptoken, ydb_buffer_t *errstr, void *callback) {
  return tpCallbackWrapper(tptoken, errstr, callback);
}
*/
import "C"

// tpInfo struct stores callback function and connection used by Transaction() to run transaction logic.
type tpInfo struct {
        conn     *Conn
        callback func()
        err      any // variable to store transaction panic errors during transition back through the CGo boundary
}

// Transaction processes database logic inside a database transaction.
//   - `callback` must be a function that implements the required database logic.
//   - `transId` has its first 8 bytes recorded in the commit record of journal files for database regions participating in the transaction.
//     Note that a transId of case-insensitive "BATCH" or "BA" are special: see [Conn.TransactionFast]()
//   - `localsToRestore` are names of local M variables to be restored to their original values when a transaction is restarted.
//     If localsToRestore[0] equals "*" then all local M locals are restored on restart. Note that since Go has its own local
//     variables it is unlikely that you will need this feature in Go.
//   - Returns true to indicate that the transaction logic was successful and has been committed to the database, or false if a rollback was necessary.
//   - Panics on errors because they are are all panic-worthy (e.g. invalid variable names). See [yottadb.Error] for rationale.
//
// The callback function should:
//   - Implement the required database logic taking into account key considerations for [Transaction Processing] code.
//   - If there are database collisions, `callback` will be called repeatedly, rolling back the database before each call. On the
//     fourth try, YottaDB will resort to calling it with other processes locked out to ensure its success.
//   - Call [Conn.Restart] if it needs to rollback and immediately restart the transaction function
//   - Call [Conn.Rollback] if it needs to rollback and immediately exit the transaction function
//   - Finish quickly because database activity in other goroutines will be blocked until it is complete.
//   - Not create goroutines within the transaction unless absolutely necessary, in which case see [Conn.CloneConn].
//
// Transaction nesting level may be determined within the callback function by reading the special variable [$tlevel], and the number of restart
// repetitions by [$trestart]. These things are documented in more detail in [Transaction Processing].
//
// [Transaction Processing]: https://docs.yottadb.com/ProgrammersGuide/langfeat.html#transaction-processing
// [$trestart]: https://docs.yottadb.com/ProgrammersGuide/isv.html#trestart
// [$tlevel]: https://docs.yottadb.com/ProgrammersGuide/isv.html#tlevel
func (conn *Conn) Transaction(transID string, localsToRestore []string, callback func()) bool {
        cconn := conn.cconn
        info := tpInfo{conn, callback, nil}
        handle := cgo.NewHandle(&info)
        defer handle.Delete()

        names := stringArrayToAnyArray(localsToRestore)
        var status C.int
        transID += "\x00" // NUL-terminate transID because it's required by ydb_tp_st()
        if len(names) == 0 {
                conn.prepAPI()
                status = C.ydb_tp_st(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, C.ydb_tpfnptr_t(C.tp_callback_wrapper), unsafe.Pointer(&handle),
                        (*C.char)(unsafe.Pointer(unsafe.StringData(transID))), 0, nil)
        } else {
                // use a Node type just as a ydb_buffer_t array (i.e. not pointing to a node) as a handy way to list varnames to restore
                namelist := conn._Node(names[0], names[1:])
                conn.prepAPI()
                status = C.ydb_tp_st(C.uint64_t(conn.tptoken.Load()), &cconn.errstr, C.ydb_tpfnptr_t(C.tp_callback_wrapper), unsafe.Pointer(&handle),
                        (*C.char)(unsafe.Pointer(unsafe.StringData(transID))), C.int(len(names)), namelist.cnode.buffers)
                runtime.KeepAlive(namelist) // ensure namelist sticks around until we've finished copying data from it's C allocation
        }
        runtime.KeepAlive(transID) // ensure batch id doesn't disappear until transaction call returns and has finished using it
        // Propagate any panics that occurred during the transaction function and
        // sent to me by tpCallbackWrapper in info.err to avoid panics crossing the CGo boundary.
        err := info.err
        if err != nil {
                // Re-panic the err, including any traceback (if it's a YDB error; we don't know how to get stacktraces from other Go errors)
                yerr, ok := err.(*Error)
                if ok {
                        err = newError(yerr.Code, fmt.Sprintf("%s\n\n%s\nWas re-paniced as: %s", err, yerr.stack, err), yerr.chain...)
                }
                panic(err)
        }
        if status == YDB_TP_ROLLBACK {
                return false
        }
        if status != YDB_OK {
                // This line is not tested in coverage tests as I do not know how to make ydb_tp_st return an error that is not already
                // handled or already returned by a ydb function inside the transaction and handled there.
                // Nevertheless, this will handle it if it occurs.
                panic(conn.lastError(status))
        }
        return true
}

// TransactionFast is a faster version of Transaction that does not ensure durability,
// for applications that do not require durability or have alternate durability mechanisms (such as checkpoints).
// It is implemented by setting the transID to the special name "BATCH" as discussed in [Transaction Processing].
//   - Panics on errors because they are are all panic-worthy (e.g. invalid variable names). See [yottadb.Error] for rationale.
//
// [Transaction Processing]: https://docs.yottadb.com/ProgrammersGuide/langfeat.html#transaction-processing
func (conn *Conn) TransactionFast(localsToRestore []string, callback func()) bool {
        return conn.Transaction("BATCH", localsToRestore, callback)
}

// Constants used for TimeoutAction.
// These just happen to all map to YDB error codes that do something but the new names have a more consistent naming scheme
// and are more Goish.
const (
        TransactionCommit   = YDB_OK
        TransactionRollback = YDB_TP_ROLLBACK
        TransactionTimeout  = ydberr.TPTIMEOUT
)

// TimeoutAction sets the action that is performed when [conn.Transaction] times out.
// The following actions are valid and performed the named function:
//   - TransactionTimeout (default): rollback and panic with Error.Code = ydberr.TPTIMEOUT
//   - TransactionRollback: rollback the transaction and return false from the transaction function
//   - TransactionCommit: commit database activity that was done before the timeout
//
// Notes:
//   - Timeout only occurs if YottaDB special variable $ZMAXTPTIME is set.
//   - Although TimeoutAction is specific to the specified conn, $ZMAXTPTIME is global.
//   - If TimeoutAction has not been called on this Conn, the default action is TransactionTimeout.
func (conn *Conn) TimeoutAction(action int) {
        if action != TransactionTimeout && action != TransactionRollback && action != TransactionCommit {
                panic(errorf(ydberr.InvalidTimeoutAction, "invalid action constant %d passed to TimeoutAction()", action))
        }
        conn.timeoutAction = action
}

// Rollback and exit a transaction immediately.
func (conn *Conn) Rollback() {
        // This panic is caught by [Conn.Transaction] to make it do a rollback and exit
        panic(newError(YDB_TP_ROLLBACK, ""))
}

// Restart a transaction immediately (after first rolling back).
func (conn *Conn) Restart() {
        // This panic is caught by [Conn.Transaction] to make it do a restart
        panic(newError(YDB_TP_RESTART, ""))
}

// TransactionToken sets the transaction-level token being using by the given connection conn.
// This is for use only in the unusual situation of mixing YDBGo v1 and v2 code and you have a v2 transaction
// that needs to call a v1 function (which must therefore be passed the v2 Conn's tptoken).
// It would be tidier, however, to avoid mixing versions within a transaction, therefore this function is deprecated
// from its inception and will be removed in a future version once there has been plenty of time to migrate all code to v2.
// See [Conn.TransactionTokenSet]
func (conn *Conn) TransactionToken() (tptoken uint64) {
        return conn.tptoken.Load()
}

// TransactionTokenSet sets the transaction-level token being using by the given connection conn.
// This is for use only in the unusual situation of mixing YDBGo v1 and v2 code and you have a v1 transaction
// that needs to call a v2 function (which must therefore be run on a Conn with the v1 tptoken).
// It would be tidier, however, to avoid mixing versions within a transaction, therefore this function is deprecated
// from its inception and will be removed in a future version once there has been plenty of time to migrate all code to v2.
// See [Conn.TransactionToken]
func (conn *Conn) TransactionTokenSet(tptoken uint64) {
        conn.tptoken.Store(tptoken)
}

// CloneConn returns a new connection that initially begins with the same transaction token as the original connection conn.
// This may be used if you absolutely must have activity within one transaction spread across multiple goroutines, in which case
// each new goroutine will need a new connection that has the same transaction token as the original connection.
// However, be aware that spreading transaction activity across multiple goroutines is highly discouraged.
// If it is done, CloneConn should be called in the new goroutine that uses the conn, and it is also
// the user's responsibility to ensure that the spawned goroutines complete before the parent terminates the transaction.
// Before doing this the programmer should first read and understand [Threads and Transaction Processing].
//
// [Threads and Transaction Processing]: https://docs.yottadb.com/MultiLangProgGuide/programmingnotes.html#threads-and-transaction-processing
func (conn *Conn) CloneConn() *Conn {
        new := NewConn()
        new.tptoken.Store(conn.tptoken.Load()) // initially inherit the original conn's tptoken
        new.timeoutAction = TransactionTimeout
        return new
}

//////////////////////////////////////////////////////////////////
//
// Copyright (c) 2020-2025 YottaDB LLC and/or its subsidiaries.
// All rights reserved.
//
//        This source code contains the intellectual property
//        of its copyright holder(s), and is made available
//        under a license.  If you do not know the terms of
//        the license, please stop and do not read further.
//
//////////////////////////////////////////////////////////////////

package yottadb

// This file is a CGo workaround: if tpCallbackWrapper is included in conn.go we run into a known Go issue where the
// compiler gives duplicate entry point (or multiple definition) errors. So this routine is placed in its own module.

// #include "libyottadb.h"
import "C"
import (
        "runtime/cgo"
        "unsafe"

        "lang.yottadb.com/go/yottadb/v2/ydberr"
)

// transactionCallbackWrapper lets Transaction() invoke a Go callback closure from C.
//
//export tpCallbackWrapper
func tpCallbackWrapper(tptoken C.uint64_t, errstr *C.ydb_buffer_t, handle unsafe.Pointer) (retval C.int) {
        h := *(*cgo.Handle)(handle)
        info := h.Value().(*tpInfo) // type assertion never panics because tpCallbackWrapper is only called by Transaction which sets this to tpInfo
        conn := info.conn
        cconn := conn.cconn

        // Defer captures panics. Restart() and Rollback() panics are returned to the YDB transaction processor as the appropriate constants.
        // Other panic error values are stored in info.err to cross the CGo boundary and are re-paniced later back in Conn.Transaction().
        defer func() {
                recovered := recover()
                info.err = recovered
                if recovered == nil {
                        retval = YDB_OK
                        return
                }
                err, isYDBError := recovered.(*Error)
                if !isYDBError {
                        // non-YottaDB errors cause rollback and also return the error in info.err for re-panic
                        retval = YDB_TP_ROLLBACK
                        return
                }
                code := err.Code
                if code == ydberr.TPTIMEOUT {
                        // If timeout, set return code to timeoutAction, which is one of: YDB_OK, YDB_TP_ROLLBACK, or ydb.TPTIMEOUT
                        code = conn.timeoutAction
                }
                if code == YDB_TP_ROLLBACK || code == YDB_TP_RESTART || code == YDB_OK {
                        info.err = nil
                }
                // Handle any other error including YDB_TP_RESTART and YDB_TP_ROLLBACK
                retval = C.int(code)
        }()

        saveToken := conn.tptoken.Swap(uint64(tptoken))
        defer conn.tptoken.Store(saveToken)

        if errstr != &cconn.errstr {
                // This should not happen, so there's no way to coverage-test it.
                panic(errorf(ydberr.CallbackWrongGoroutine, "YDBGo design fault: transaction callback from a different connection than the one that initiated the transaction; contact YottaDB support."))
        }
        info.callback()
        return YDB_OK // retval may be changed by deferred func
}

//////////////////////////////////////////////////////////////////
//
// Copyright (c) 2025 YottaDB LLC and/or its subsidiaries.
// All rights reserved.
//
//        This source code contains the intellectual property
//        of its copyright holder(s), and is made available
//        under a license.  If you do not know the terms of
//        the license, please stop and do not read further.
//
//////////////////////////////////////////////////////////////////

// Allow Go to call a variadic C function
// This is used to call YottaDB C API's variadic function ydb_lock_st()

package yottadb

import (
        "fmt"
        "io"
        "runtime"
        "strconv"
        "unsafe"

        "lang.yottadb.com/go/yottadb/v2/ydberr"
)

// #include <stdlib.h> /* For uint64_t definition on Linux */
// #include "libyottadb.h"
import "C"

const maxARM32RegParms uint32 = 4                // Max number of parms passed in registers in ARM32 (affects passing of 64 bit parms)
const maxVariadicParams = C.MAX_GPARAM_LIST_ARGS // Make this constant available to tests

// ---- Variadic parameter (vp) support for C (despite CGo not supporting it directly).

// vpCall method calls a variadic function with the parameters previously added.
// The function pointer `vpfunc` must point to the C variadic function to call.
// The instance vplist must have been previously initialized with any parameters using call(s) to vpAddParam*().
func (conn *Conn) vpCall(vpfunc unsafe.Pointer) C.int {
        cconn := conn.cconn
        conn.prepAPI()
        retval := C.ydb_call_variadic_plist_func((C.ydb_vplist_func)(vpfunc), cconn.vplist)
        cconn.vplist.n = 0
        runtime.KeepAlive(conn) // ensure conn sticks around until we've finished copying data from it's C allocation
        return retval
}

// vpStart must be called before the first vpAddParam call to list parameters for a variadic call.
func (conn *Conn) vpStart() {
        vplist := conn.vpAlloc() // Lazily allocate vplist only if needed
        vplist.n = 0
}

// vpAlloc allocates and returns vplist if it hasn't already been allocated.
func (conn *Conn) vpAlloc() *C.gparam_list {
        cconn := conn.cconn
        // Lazily allocate vplist only if needed
        if cconn.vplist != nil {
                return cconn.vplist
        }
        cconn.vplist = (*C.gparam_list)(calloc(C.sizeof_gparam_list)) // must use our calloc, not malloc: see calloc doc
        // Note this gets freed by conn cleanup
        cconn.vplist.n = -1 // flags an error if the user forgets to call vpStart() before vpAddParam()
        return cconn.vplist
}

// vpAddParam adds another parameter to the variadic parameter list that will be used at the next invocation of conn.vpCall().
// Note that any supplied addresses must point to C allocated memory, not Go allocated memory, or CGo will panic.
func (conn *Conn) vpAddParam(value uintptr) {
        vplist := conn.vpAlloc() // Lazily allocate vplist only if needed
        n := vplist.n
        if n < 0 {
                panic(errorf(ydberr.Variadic, "programmer forgot to call vpStart() before vpAddParam()"))
        }
        if n >= maxVariadicParams {
                panic(errorf(ydberr.Variadic, "variadic parameter item count %d exceeds maximum count of %d", n+1, C.MAX_GPARAM_LIST_ARGS))
        }
        // Compute address of indexed element
        elemptr := (*uintptr)(unsafe.Pointer(&vplist.arg[n]))
        *elemptr = value
        vplist.n++
}

// vpAddParam64 adds a specifically 64-bit parameter to the variadic parameter list that will be used at the next invocation of conn.vpCall().
// On 32-bit platforms this will push the 64-bit value in two 32-bit slots in the correct endian order.
// Note that any supplied addresses must point to C allocated memory, not Go allocated memory, or CGo will panic.
func (conn *Conn) vpAddParam64(value uint64) {
        if strconv.IntSize == 64 { // if we're on a 64-bit machine
                conn.vpAddParam(uintptr(value))
                return
        }
        vplist := conn.vpAlloc() // Lazily allocate vplist only if needed
        if isLittleEndian() {
                if runtime.GOARCH == "arm" {
                        // If this is 32 bit ARM, there is a rule about the first 4 parms that go into registers. If
                        // there is a mix of 32 bit and 64 bit parameters, the 64 bit parm must always go into an
                        // even/odd pair of registers. If the next index is odd (meaning we could be loading into an
                        // odd/even pair, then that register is skipped and left unused. This only applies to parms
                        // loaded into registers and not to parms pushed on the stack.
                        const maxARM32RegParms = 4 // Max number of parms passed in registers in ARM32
                        if vplist.n&1 == 1 && int(vplist.n) < maxARM32RegParms {
                                conn.vpAddParam(0) // skip odd-indexed spots
                        }
                }
                conn.vpAddParam(uintptr(value & 0xffffffff))
                conn.vpAddParam(uintptr(value >> 32))
        } else {
                conn.vpAddParam(uintptr(value >> 32))
                conn.vpAddParam(uintptr(value & 0xffffffff))
        }
}

// vpDump dumps a variadic parameter list for debugging/test purposes.
func (conn *Conn) vpDump(w io.Writer) {
        vplist := conn.cconn.vplist
        if vplist == nil {
                panic(errorf(ydberr.Variadic, "could not dump nil vararg list"))
        }
        n := int(vplist.n)
        argbase := unsafe.Add(unsafe.Pointer(vplist), unsafe.Sizeof(vplist))
        fmt.Fprintf(w, "   Total of %d elements in this variadic plist\n", n)
        for i := range n {
                elemptr := unsafe.Add(argbase, i*int(unsafe.Sizeof(vplist)))
                fmt.Fprintf(w, "   Elem %d (%p) Value: %d (0x%x)\n", i, elemptr, *((*uintptr)(elemptr)), *((*uintptr)(elemptr)))
        }
        runtime.KeepAlive(conn) // ensure conn sticks around until we've finished copying data from it's C allocation
}

// isLittleEndian is a function to determine endianness.
func isLittleEndian() bool {
        var bittest = 1
        return *(*byte)(unsafe.Pointer(&bittest)) == 1
}

//////////////////////////////////////////////////////////////////
//
// Copyright (c) 2026 YottaDB LLC and/or its subsidiaries.
// All rights reserved.
//
//        This source code contains the intellectual property
//        of its copyright holder(s), and is made available
//        under a license.  If you do not know the terms of
//        the license, please stop and do not read further.
//
//////////////////////////////////////////////////////////////////

// See package doc in doc.go

package yottadb

// go 1.24 required for the use of AddCleanup() instead of SetFinalizer(), and to run tests: testing.Loop
// go 1.23 required for iterators, used to iterate database subscripts
// go 1.22 required for the range clause
// go 1.19 required for sync/atomic -- safer than previous options

import (
        "sync/atomic"
        "time"
        "unsafe"

        "lang.yottadb.com/go/yottadb/v2/ydberr"
)

// #cgo pkg-config: yottadb
// #include "libyottadb.h"
import "C"

// ---- Release version constants - be sure to change all of them appropriately

// MinYDBRelease - (string) Minimum YottaDB release name required by this wrapper.
// This is checked on init. It is a var rather than a const so we can change it purely to verify logic that parses it
var MinYDBRelease string = "r1.34"

// WrapperRelease - (string) The Go wrapper release version for YottaDB SimpleAPI. Note the third piece of this version
// will be even for a production release and odd for a development release. When released, depending
// on new content, either the third piece of the version will be bumped to an even value or the second piece of the
// version will be bumped by 1 and the third piece of the version set to 0. On rare occasions, we may bump the first
// piece of the version and zero the others when the changes are significant.
// Also, the version numbers may be followed by a hyphen and text, e.g. "v2.0.2-alpha"
const WrapperRelease string = "v2.0.4"

// ---- Wait times

// Set default exit wait times. The user may change these.
var (
        // MaxPanicExitWait is the maximum wait when a panic caused by a signal has occurred (likely unable to run Exit().
        // It specifies the wait in seconds that yottadb.Exit() will wait for ydb_exit() to run before
        // giving up and forcing the process to exit. Note the normal exit wait is longer as we expect ydb_exit() to be
        // successful so can afford to wait as long as needed to do the sync but for a signal exit, the rundown is likely
        // already done (exit handler called by the signal processing itself) but if ydb_exit() is not able to get
        // the system lock and is likely to hang, 3 seconds is about as much as we can afford to wait.
        MaxPanicExitWait time.Duration = 3 * time.Second

        // MaxNormalExitWait is maximum wait for a normal shutdown when no system lock hang in Exit() is likely.
        MaxNormalExitWait time.Duration = 60 * time.Second

        // MaxSigShutdownWait is maximum wait to close down signal handling goroutines (shouldn't take this long).
        MaxSigShutdownWait time.Duration = 5 * time.Second

        // MaxSigAckWait is maximum wait for notify via acknowledgement channel that a notified signal handler is
        // done handling the signal.
        MaxSigAckWait time.Duration = 10 * time.Second
)

// ---- Debug settings

// DebugMode greater than zero (1, 2, or 3) increases logging output
//   - DebugMode=0: no debug logging (default)
//   - DebugMode=1: log at entrypoint of M functions or Go signal callbacks; and don't remove temporary callback table file
//   - DebugMode=2: in addition, log extra signal processing info
var DebugMode atomic.Int64 // increasing values 1, 2 or 3 for increasing log output

// ---- Utility functions

// calloc allocates c memory and clears it; a wrapper for C.calloc() that panics on error.
// It must be used instead of C.malloc() if Go will write to pointers within the allocation.
// This is due to documented bug: https://golang.org/cmd/cgo/#hdr-Passing_pointers.
// The user must call C.free() to free the allocation.
func calloc(size C.size_t) unsafe.Pointer {
        // Use calloc: can't let Go store pointers in uninitialized C memory per CGo bug: https://golang.org/cmd/cgo/#hdr-Passing_pointers
        mem := C.calloc(1, size)
        if mem == nil {
                panic(errorf(ydberr.OutOfMemory, "out of memory"))
        }
        return mem
}

//go:generate ../scripts/gen_error_codes.sh ydberr/errorcodes ydbconst ydberr
//go:generate bash -c "cd .. && scripts/gen_error_codes.sh error_codes"