diff --git a/.github/workflows/go-sdk-test.yml b/.github/workflows/go-sdk-test.yml new file mode 100644 index 00000000..f81415f7 --- /dev/null +++ b/.github/workflows/go-sdk-test.yml @@ -0,0 +1,67 @@ +name: Go SDK test + +permissions: + contents: read + +on: + push: + branches: [main] + paths: + - 'sdks/go/**' + - '.github/workflows/go-sdk-test.yml' + pull_request: + paths: + - 'sdks/go/**' + - '.github/workflows/go-sdk-test.yml' + +jobs: + unit-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.22.2' + + - name: Run SDK unit tests without native bindings + working-directory: sdks/go/sdk + env: + CGO_ENABLED: '0' + run: go test ./... + + - name: Verify SDK standalone module mode + working-directory: sdks/go/sdk + env: + CGO_ENABLED: '0' + GOWORK: 'off' + run: go test ./... + + - name: Run native installer tests + working-directory: sdks/go/tools/install + run: go test ./... + + bindings-smoke: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.22.2' + + - name: Install linux-amd64 native library + working-directory: sdks/go + run: go run ./tools/install --release c-sdk-v0.9.0 + + - name: Verify bindings compile with CGO + working-directory: sdks/go/bindings + env: + CGO_ENABLED: '1' + run: go build -v . + + - name: Verify SDK compiles with CGO + working-directory: sdks/go/sdk + env: + CGO_ENABLED: '1' + run: go build -v . diff --git a/.github/workflows/publish-go-sdk.yml b/.github/workflows/publish-go-sdk.yml new file mode 100644 index 00000000..137ca708 --- /dev/null +++ b/.github/workflows/publish-go-sdk.yml @@ -0,0 +1,72 @@ +name: Publish Go SDK + +permissions: + contents: write + +on: + workflow_dispatch: + inputs: + c_sdk_version: + description: 'C SDK GitHub release tag' + required: true + default: 'c-sdk-v0.9.0' + go_version: + description: 'Go module version to publish' + required: true + default: 'v0.1.2' + +concurrency: + group: go-sdk-release-${{ github.event.inputs.go_version }} + cancel-in-progress: false + +jobs: + publish: + runs-on: ubuntu-latest + env: + C_SDK_VERSION: ${{ github.event.inputs.c_sdk_version }} + GO_VERSION: ${{ github.event.inputs.go_version }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.22.2' + + - name: Configure Git author + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + + - name: Update bindings native version metadata + shell: bash + run: | + if [[ ! "${C_SDK_VERSION}" =~ ^c-sdk-v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + echo "c_sdk_version must be a c-sdk-vMAJOR.MINOR.PATCH tag" >&2 + exit 1 + fi + if [[ ! "${GO_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + echo "go_version must be a vMAJOR.MINOR.PATCH tag" >&2 + exit 1 + fi + NATIVE_VERSION="${C_SDK_VERSION#c-sdk-v}" + cat > sdks/go/bindings/version.go < +- `CGO_ENABLED=1` (default on Linux and macOS) +- A C compiler (`gcc` on Linux, Xcode CLI tools on macOS) +- One of the supported native targets: Linux (`amd64`, `arm64`) or Apple Silicon + macOS (`arm64`) -For Linux `x86_64`, extract the archive somewhere local so you have: +Windows currently uses the bindings-unavailable stub. Native Windows support +will require a MinGW-compatible C SDK release artifact. -```text -/ -├── include/libmoss.h -└── lib/libmoss.so +## Layout + +``` +bindings/ + include/libmoss.h # committed C header + libmoss.go # CGO wrapper (requires CGO) + prebuilt__.go # per-platform CGO linker flags + generate.go # //go:generate install hook for checkouts + lib/ + linux-amd64/ # libmoss.a (gitignored, downloaded at build time) + linux-arm64/ + darwin-arm64/ ``` -Then build with: +Native `.a` / `.lib` files are gitignored. The +[`tools/install`](../tools/install) command fetches them from GitHub Releases and +verifies SHA256 checksums. + +## Local development + +Install the native library for your current machine: + +```bash +./sdks/go/scripts/link_dev_lib.sh c-sdk-v0.9.0 +``` + +Or fetch all supported platforms: ```bash -export CGO_CFLAGS="-I/include" -export CGO_LDFLAGS="-L/lib" -export LD_LIBRARY_PATH="/lib" -go test -tags libmoss ./... +./sdks/go/scripts/fetch-static-libs.sh c-sdk-v0.9.0 ``` -The Go SDK module can then be built with the same flags and tag. +Or use `go generate` from a checkout of this directory: + +```bash +cd sdks/go/bindings +go generate . +``` + +Then build with CGO enabled: + +```bash +CGO_ENABLED=1 go build . +``` + +## Publishing + +Maintainers run the **Publish Go SDK** GitHub Actions workflow. It creates +source-only module tags (no binaries in git): + +- `sdks/go/sdk/v0.1.2` +- `sdks/go/bindings/v0.1.2` + +Consumers download native libraries during the explicit `tools/install` step. + +## Build without CGO + +When `CGO_ENABLED=0`, this package builds a stub that returns +`ErrBindingsUnavailable`. The public SDK can still run unit tests and cloud query +fallback tests without native libraries. diff --git a/sdks/go/bindings/errors.go b/sdks/go/bindings/errors.go index 16ca55d4..52d22127 100644 --- a/sdks/go/bindings/errors.go +++ b/sdks/go/bindings/errors.go @@ -2,5 +2,5 @@ package mosscore import "errors" -var ErrBindingsUnavailable = errors.New("mosscore: libmoss bindings are unavailable; build with -tags libmoss and configure the libmoss C SDK") +var ErrBindingsUnavailable = errors.New("mosscore: libmoss bindings are unavailable; build with CGO_ENABLED=1 and install the platform native library (see sdks/go/bindings/README.md)") var ErrClientClosed = errors.New("mosscore: client is closed") diff --git a/sdks/go/bindings/generate.go b/sdks/go/bindings/generate.go new file mode 100644 index 00000000..da5903c1 --- /dev/null +++ b/sdks/go/bindings/generate.go @@ -0,0 +1,3 @@ +package mosscore + +//go:generate go run ../tools/install diff --git a/sdks/go/bindings/include/libmoss.h b/sdks/go/bindings/include/libmoss.h new file mode 100644 index 00000000..969f68b8 --- /dev/null +++ b/sdks/go/bindings/include/libmoss.h @@ -0,0 +1,287 @@ +/* Generated by cbindgen — do not edit manually. */ + +#ifndef LIBMOSS_H +#define LIBMOSS_H + +/* Warning: this file is auto-generated by cbindgen. Do not modify this manually. */ + +#include +#include +#include +#include + +/** + * Result codes returned by every fallible `moss_*` function. + */ +enum MossResult +#ifdef __cplusplus + : int32_t +#endif // __cplusplus + { + OK = 0, + ERR_NULL_POINTER = -1, + ERR_INVALID_ARG = -2, + ERR_CLOUD = -3, + ERR_INDEX_NOT_FOUND = -4, + ERR_MODEL = -5, + ERR_IO = -6, + ERR_INTERNAL = -7, +}; +#ifndef __cplusplus +typedef int32_t MossResult; +#endif // __cplusplus + +typedef struct MossClient MossClient; + +typedef struct MossSession MossSession; + +typedef struct MossMetadataEntry { + char *key; + char *value; +} MossMetadataEntry; + +typedef struct MossDocumentInfo { + char *id; + char *text; + struct MossMetadataEntry *metadata; + uintptr_t metadata_count; + float *embedding; + uintptr_t embedding_dim; +} MossDocumentInfo; + +typedef struct MossMutationResult { + char *job_id; + char *index_name; + uintptr_t doc_count; +} MossMutationResult; + +typedef struct MossMutationOptions { + bool upsert; +} MossMutationOptions; + +typedef struct MossModelRef { + char *id; + char *version; +} MossModelRef; + +typedef struct MossIndexInfo { + char *id; + char *name; + char *version; + char *status; + uintptr_t doc_count; + char *created_at; + char *updated_at; + struct MossModelRef model; +} MossIndexInfo; + +typedef struct MossJobStatusResponse { + char *job_id; + char *status; + double progress; + char *current_phase; + char *error; + char *created_at; + char *updated_at; + char *completed_at; +} MossJobStatusResponse; + +typedef struct MossLoadIndexOptions { + bool auto_refresh; + uint64_t polling_interval_secs; +} MossLoadIndexOptions; + +typedef struct MossQueryOptions { + uintptr_t top_k; + float alpha; + const char *filter_json; + const float *embedding; + uintptr_t embedding_dim; +} MossQueryOptions; + +typedef struct MossQueryResultDoc { + char *id; + char *text; + struct MossMetadataEntry *metadata; + uintptr_t metadata_count; + float score; +} MossQueryResultDoc; + +typedef struct MossSearchResult { + struct MossQueryResultDoc *docs; + uintptr_t doc_count; + char *query; + char *index_name; + uint64_t time_taken_ms; +} MossSearchResult; + +typedef struct MossRefreshResult { + char *index_name; + char *previous_updated_at; + char *new_updated_at; + bool was_updated; +} MossRefreshResult; + +typedef struct MossSessionOptions { + const char *model_id; +} MossSessionOptions; + +typedef struct MossAddDocsOptions { + bool upsert; +} MossAddDocsOptions; + +typedef struct MossPushIndexResult { + char *job_id; + char *index_name; + uintptr_t doc_count; + char *status; +} MossPushIndexResult; + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +/** + * Returns the SDK version string (e.g. "0.8.7"). + * The returned pointer is valid for the lifetime of the library — do not free it. + */ +const char *moss_sdk_version(void); + +MossResult moss_client_new(const char *project_id, + const char *project_key, + struct MossClient **out); + +void moss_client_free(struct MossClient *client); + +MossResult moss_client_create_index(struct MossClient *client, + const char *name, + const struct MossDocumentInfo *docs, + uintptr_t doc_count, + const char *model_id, + struct MossMutationResult **out); + +MossResult moss_client_add_docs(struct MossClient *client, + const char *name, + const struct MossDocumentInfo *docs, + uintptr_t doc_count, + const struct MossMutationOptions *opts, + struct MossMutationResult **out); + +MossResult moss_client_delete_docs(struct MossClient *client, + const char *name, + const char *const *doc_ids, + uintptr_t count, + struct MossMutationResult **out); + +MossResult moss_client_delete_index(struct MossClient *client, const char *name, bool *out_deleted); + +MossResult moss_client_get_index(struct MossClient *client, + const char *name, + struct MossIndexInfo **out); + +MossResult moss_client_list_indexes(struct MossClient *client, + struct MossIndexInfo **out, + uintptr_t *out_count); + +MossResult moss_client_get_docs(struct MossClient *client, + const char *name, + const char *const *doc_ids, + uintptr_t id_count, + struct MossDocumentInfo **out_docs, + uintptr_t *out_count); + +MossResult moss_client_get_job_status(struct MossClient *client, + const char *job_id, + struct MossJobStatusResponse **out); + +MossResult moss_client_load_index(struct MossClient *client, + const char *name, + const struct MossLoadIndexOptions *opts, + struct MossIndexInfo **out); + +MossResult moss_client_unload_index(struct MossClient *client, const char *name); + +MossResult moss_client_query(struct MossClient *client, + const char *name, + const char *query, + const struct MossQueryOptions *opts, + struct MossSearchResult **out); + +MossResult moss_client_refresh_index(struct MossClient *client, + const char *name, + struct MossRefreshResult **out); + +MossResult moss_client_session(struct MossClient *client, + const char *name, + const struct MossSessionOptions *opts, + struct MossSession **out); + +/** + * Returns a pointer to a null-terminated UTF-8 error description for the most + * recent failed `moss_*` call on the current thread. The pointer is valid until + * the next `moss_*` call on the same thread. Returns NULL if no error is stored. + */ +const char *moss_last_error(void); + +void moss_session_free(struct MossSession *session); + +/** + * Returns a pointer to the session name. The pointer is valid for the lifetime + * of the session — do not free it. + */ +const char *moss_session_name(const struct MossSession *session); + +uintptr_t moss_session_doc_count(const struct MossSession *session); + +MossResult moss_session_add_docs(struct MossSession *session, + const struct MossDocumentInfo *docs, + uintptr_t doc_count, + const struct MossAddDocsOptions *opts, + uintptr_t *out_added, + uintptr_t *out_updated); + +MossResult moss_session_delete_docs(struct MossSession *session, + const char *const *doc_ids, + uintptr_t count, + uintptr_t *out_deleted); + +MossResult moss_session_get_docs(struct MossSession *session, + const char *const *doc_ids, + uintptr_t id_count, + struct MossDocumentInfo **out_docs, + uintptr_t *out_count); + +MossResult moss_session_query(struct MossSession *session, + const char *query, + const struct MossQueryOptions *opts, + struct MossSearchResult **out); + +MossResult moss_session_load_index(struct MossSession *session, + const char *index_name, + uintptr_t *out_doc_count); + +MossResult moss_session_push_index(struct MossSession *session, struct MossPushIndexResult **out); + +void moss_free_string(char *s); + +void moss_free_documents(struct MossDocumentInfo *docs, uintptr_t count); + +void moss_free_search_result(struct MossSearchResult *result); + +void moss_free_index_info(struct MossIndexInfo *info); + +void moss_free_index_info_list(struct MossIndexInfo *infos, uintptr_t count); + +void moss_free_mutation_result(struct MossMutationResult *result); + +void moss_free_push_index_result(struct MossPushIndexResult *result); + +void moss_free_job_status_response(struct MossJobStatusResponse *resp); + +void moss_free_refresh_result(struct MossRefreshResult *result); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif /* LIBMOSS_H */ diff --git a/sdks/go/bindings/lib/darwin-arm64/.gitignore b/sdks/go/bindings/lib/darwin-arm64/.gitignore new file mode 100644 index 00000000..30f8296b --- /dev/null +++ b/sdks/go/bindings/lib/darwin-arm64/.gitignore @@ -0,0 +1,2 @@ +libmoss.a +.moss-install-checksum diff --git a/sdks/go/bindings/lib/linux-amd64/.gitignore b/sdks/go/bindings/lib/linux-amd64/.gitignore new file mode 100644 index 00000000..30f8296b --- /dev/null +++ b/sdks/go/bindings/lib/linux-amd64/.gitignore @@ -0,0 +1,2 @@ +libmoss.a +.moss-install-checksum diff --git a/sdks/go/bindings/lib/linux-arm64/.gitignore b/sdks/go/bindings/lib/linux-arm64/.gitignore new file mode 100644 index 00000000..30f8296b --- /dev/null +++ b/sdks/go/bindings/lib/linux-arm64/.gitignore @@ -0,0 +1,2 @@ +libmoss.a +.moss-install-checksum diff --git a/sdks/go/bindings/libmoss.go b/sdks/go/bindings/libmoss.go index 4adbf8ed..d25b3f0f 100644 --- a/sdks/go/bindings/libmoss.go +++ b/sdks/go/bindings/libmoss.go @@ -1,11 +1,9 @@ -//go:build libmoss +//go:build cgo && ((linux && (amd64 || arm64)) || (darwin && arm64)) package mosscore /* -#cgo linux LDFLAGS: -lmoss -ldl -lm -lpthread -#cgo darwin LDFLAGS: -lmoss -lc++ -#cgo windows LDFLAGS: -lmoss +#cgo CFLAGS: -I${SRCDIR}/include #include #include */ diff --git a/sdks/go/bindings/prebuilt_darwin_arm64.go b/sdks/go/bindings/prebuilt_darwin_arm64.go new file mode 100644 index 00000000..ffe18f89 --- /dev/null +++ b/sdks/go/bindings/prebuilt_darwin_arm64.go @@ -0,0 +1,9 @@ +//go:build cgo && darwin && arm64 + +package mosscore + +/* +#cgo CFLAGS: -I${SRCDIR}/include +#cgo LDFLAGS: -L${SRCDIR}/lib/darwin-arm64 -lmoss -lc++ -framework Security -framework SystemConfiguration +*/ +import "C" diff --git a/sdks/go/bindings/prebuilt_linux_amd64.go b/sdks/go/bindings/prebuilt_linux_amd64.go new file mode 100644 index 00000000..6f31905c --- /dev/null +++ b/sdks/go/bindings/prebuilt_linux_amd64.go @@ -0,0 +1,9 @@ +//go:build cgo && linux && amd64 + +package mosscore + +/* +#cgo CFLAGS: -I${SRCDIR}/include +#cgo LDFLAGS: -L${SRCDIR}/lib/linux-amd64 -lmoss -lstdc++ -ldl -lm -lpthread +*/ +import "C" diff --git a/sdks/go/bindings/prebuilt_linux_arm64.go b/sdks/go/bindings/prebuilt_linux_arm64.go new file mode 100644 index 00000000..eea75d94 --- /dev/null +++ b/sdks/go/bindings/prebuilt_linux_arm64.go @@ -0,0 +1,9 @@ +//go:build cgo && linux && arm64 + +package mosscore + +/* +#cgo CFLAGS: -I${SRCDIR}/include +#cgo LDFLAGS: -L${SRCDIR}/lib/linux-arm64 -lmoss -lstdc++ -ldl -lm -lpthread +*/ +import "C" diff --git a/sdks/go/bindings/stub.go b/sdks/go/bindings/stub.go index dc32277e..1f1d4071 100644 --- a/sdks/go/bindings/stub.go +++ b/sdks/go/bindings/stub.go @@ -1,4 +1,4 @@ -//go:build !libmoss +//go:build !cgo || !((linux && (amd64 || arm64)) || (darwin && arm64)) package mosscore diff --git a/sdks/go/bindings/version.go b/sdks/go/bindings/version.go new file mode 100644 index 00000000..6fd7584e --- /dev/null +++ b/sdks/go/bindings/version.go @@ -0,0 +1,7 @@ +package mosscore + +// NativeLibVersion is the Moss C SDK version bundled with published Go bindings. +const NativeLibVersion = "0.9.0" + +// NativeLibReleaseTag is the GitHub release tag used to fetch native artifacts. +const NativeLibReleaseTag = "c-sdk-v0.9.0" diff --git a/sdks/go/go.work b/sdks/go/go.work new file mode 100644 index 00000000..fd6edb4a --- /dev/null +++ b/sdks/go/go.work @@ -0,0 +1,7 @@ +go 1.22.2 + +use ( + ./bindings + ./sdk + ./tools/install +) diff --git a/sdks/go/scripts/bump-module-versions.sh b/sdks/go/scripts/bump-module-versions.sh new file mode 100755 index 00000000..88fb29ef --- /dev/null +++ b/sdks/go/scripts/bump-module-versions.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Pins the public sdk module to a published bindings version. +# +# Usage: +# ./sdks/go/scripts/bump-module-versions.sh v0.9.0 + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " >&2 + echo "Example: $0 v0.9.0" >&2 + exit 1 +fi + +VERSION="$1" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SDK_DIR="${ROOT_DIR}/sdk" + +( + cd "${SDK_DIR}" + go mod edit -require="github.com/usemoss/moss/sdks/go/bindings@${VERSION}" + go mod edit -dropreplace=github.com/usemoss/moss/sdks/go/bindings 2>/dev/null || true +) + +echo "Pinned sdk module to bindings ${VERSION}" diff --git a/sdks/go/scripts/fetch-static-libs.sh b/sdks/go/scripts/fetch-static-libs.sh new file mode 100755 index 00000000..d691d9ed --- /dev/null +++ b/sdks/go/scripts/fetch-static-libs.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Downloads static libmoss archives for all supported platforms. +# +# Usage: +# ./sdks/go/scripts/fetch-static-libs.sh [c-sdk-tag] + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +RELEASE_TAG="${1:-c-sdk-v0.9.0}" + +cd "${ROOT_DIR}" +go run ./tools/install --all --release "${RELEASE_TAG}" diff --git a/sdks/go/scripts/link_dev_lib.sh b/sdks/go/scripts/link_dev_lib.sh new file mode 100755 index 00000000..243374d6 --- /dev/null +++ b/sdks/go/scripts/link_dev_lib.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Installs the native library for the current machine (local dev helper). +# +# Usage: +# ./sdks/go/scripts/link_dev_lib.sh [c-sdk-tag] + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +RELEASE_TAG="${1:-c-sdk-v0.9.0}" + +cd "${ROOT_DIR}" +go run ./tools/install --release "${RELEASE_TAG}" diff --git a/sdks/go/scripts/publish-sdk-module-tags.sh b/sdks/go/scripts/publish-sdk-module-tags.sh new file mode 100755 index 00000000..3e5dc242 --- /dev/null +++ b/sdks/go/scripts/publish-sdk-module-tags.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Publishes source-only bindings + sdk module tags. +# +# Usage: +# ./sdks/go/scripts/publish-sdk-module-tags.sh v0.1.2 + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [remote]" >&2 + exit 1 +fi + +VERSION="$1" +REMOTE="${2:-origin}" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REPO_ROOT="$(cd "${ROOT_DIR}/../.." && pwd)" + +cd "${REPO_ROOT}" + +TAGS=( + "sdks/go/bindings/${VERSION}" + "sdks/go/sdk/${VERSION}" + "sdks/go/tools/install/${VERSION}" +) + +for tag in "${TAGS[@]}"; do + if git rev-parse --verify --quiet "refs/tags/${tag}" >/dev/null; then + echo "Refusing to overwrite existing local tag ${tag}" >&2 + exit 1 + fi + if [[ -n "$(git ls-remote --tags "${REMOTE}" "refs/tags/${tag}")" ]]; then + echo "Refusing to overwrite existing remote tag ${tag}" >&2 + exit 1 + fi +done + +"${ROOT_DIR}/scripts/bump-module-versions.sh" "${VERSION}" + +git add sdks/go/bindings/go.mod sdks/go/sdk/go.mod sdks/go/bindings/version.go +git add sdks/go/bindings/include/libmoss.h sdks/go/bindings/generate.go +git add sdks/go/tools/install + +if ! git diff --cached --quiet; then + git commit -m "chore(go): publish bindings and sdk ${VERSION}" +fi + +for tag in "${TAGS[@]}"; do + git tag "${tag}" +done + +REFS=("${TAGS[@]/#/refs/tags/}") +if ! git push --atomic "${REMOTE}" "${REFS[@]}"; then + git tag -d "${TAGS[@]}" >/dev/null + exit 1 +fi + +printf 'Published %s\n' "${TAGS[@]}" diff --git a/sdks/go/sdk/README.md b/sdks/go/sdk/README.md index 05ad78f8..b41804d6 100644 --- a/sdks/go/sdk/README.md +++ b/sdks/go/sdk/README.md @@ -19,21 +19,30 @@ The Go SDK now has two layers: ## Current limitations -- the SDK requires the `libmoss` C SDK and the `libmoss` build tag for real runtime operations +- CGO and a C compiler are required for full SDK functionality (`CGO_ENABLED=1`) +- Native bindings support Linux (`amd64`, `arm64`) and Apple Silicon macOS + (`arm64`); unsupported platforms use the bindings-unavailable stub - cloud query fallback supports `TopK` and caller-provided embeddings; `Alpha` and `Filter` require a locally loaded index - `LoadIndexOptions.CachePath` is not exposed by the current `libmoss` C API yet ## Installation -From this repository, import the package at: - -```go -github.com/usemoss/moss/sdks/go/sdk +```bash +go get github.com/usemoss/moss/sdks/go/sdk +go run github.com/usemoss/moss/sdks/go/tools/install@latest --vendor ``` -Download the `libmoss` C SDK release and build with `-tags libmoss`. The -bindings setup is documented in -[`../bindings/README.md`](../bindings/README.md). +The install tool downloads the static `libmoss` library for your platform. See +[`../bindings/README.md`](../bindings/README.md) for supported platforms and +toolchain notes. +The explicit `--vendor` option vendors the SDK so CGO can link the downloaded +library from a writable directory; commit `vendor/` if your project commits +vendored dependencies. Run it after your application imports the SDK so the +bindings package is included in `vendor/`. +In a Go workspace, the installer uses `go work vendor` instead. + +Monorepo development uses the workspace in [`../go.work`](../go.work) and +[`../scripts/link_dev_lib.sh`](../scripts/link_dev_lib.sh). ## Quick start @@ -139,14 +148,12 @@ Runnable examples live here: - [`../../../examples/go/basic/main.go`](../../../examples/go/basic/main.go) - [`../../../examples/go/custom-embeddings/main.go`](../../../examples/go/custom-embeddings/main.go) -Run them with native bindings enabled: +Run them from the monorepo: ```bash +../scripts/link_dev_lib.sh c-sdk-v0.9.0 cd ../../../examples/go -export CGO_CFLAGS="-I/include" -export CGO_LDFLAGS="-L/lib" -export LD_LIBRARY_PATH="/lib" -go run -tags libmoss ./basic +go run ./basic ``` ## Integration tests @@ -163,8 +170,7 @@ Then run: ```bash cd sdks/go/sdk go test ./... -CGO_CFLAGS="-I/include" \ -CGO_LDFLAGS="-L/lib" \ -LD_LIBRARY_PATH="/lib" \ -go test -tags libmoss ./... + +# Live integration (requires credentials + native lib from link_dev_lib.sh): +CGO_ENABLED=1 go test ./... ``` diff --git a/sdks/go/sdk/go.mod b/sdks/go/sdk/go.mod index fe443db1..9766bf29 100644 --- a/sdks/go/sdk/go.mod +++ b/sdks/go/sdk/go.mod @@ -2,6 +2,6 @@ module github.com/usemoss/moss/sdks/go/sdk go 1.22.2 -require github.com/usemoss/moss/sdks/go/bindings v0.0.0 +require github.com/usemoss/moss/sdks/go/bindings v0.1.2 replace github.com/usemoss/moss/sdks/go/bindings => ../bindings diff --git a/sdks/go/tools/install/go.mod b/sdks/go/tools/install/go.mod new file mode 100644 index 00000000..74c4e7e6 --- /dev/null +++ b/sdks/go/tools/install/go.mod @@ -0,0 +1,3 @@ +module github.com/usemoss/moss/sdks/go/tools/install + +go 1.22.2 diff --git a/sdks/go/tools/install/main.go b/sdks/go/tools/install/main.go new file mode 100644 index 00000000..8f3ee621 --- /dev/null +++ b/sdks/go/tools/install/main.go @@ -0,0 +1,537 @@ +// Command install downloads Moss C SDK static libraries from GitHub Releases +// into sdks/go/bindings for local CGO linking. +package main + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "errors" + "flag" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strings" +) + +const ( + defaultRepo = "usemoss/moss" + bindingsModulePath = "github.com/usemoss/moss/sdks/go/bindings" + installReceiptName = ".moss-install-checksum" +) + +type platform struct { + id string + triple string + libFile string + srcLib string +} + +var platforms = []platform{ + { + id: "linux-amd64", triple: "x86_64-unknown-linux-gnu", + libFile: "libmoss.a", srcLib: "lib/libmoss.a", + }, + { + id: "linux-arm64", triple: "aarch64-unknown-linux-gnu", + libFile: "libmoss.a", srcLib: "lib/libmoss.a", + }, + { + id: "darwin-arm64", triple: "aarch64-apple-darwin", + libFile: "libmoss.a", srcLib: "lib/libmoss.a", + }, +} + +func main() { + all := flag.Bool("all", false, "install libraries for all supported platforms") + releaseTag := flag.String("release", "", "C SDK GitHub release tag (default: target bindings metadata)") + repo := flag.String("repo", defaultRepo, "GitHub repository (owner/name)") + bindingsDir := flag.String("bindings", "", "bindings directory (default: auto-detect)") + vendor := flag.Bool("vendor", false, "run go mod vendor before installing into a downloaded module") + force := flag.Bool("force", false, "re-download even if the library is already installed") + flag.Parse() + + root, err := resolveWritableBindingsDir(*bindingsDir, *vendor) + if err != nil { + fatal(err) + } + if *releaseTag == "" { + *releaseTag, err = nativeReleaseTag(root) + if err != nil { + fatal(err) + } + } + + version := strings.TrimPrefix(*releaseTag, "c-sdk-") + version = strings.TrimPrefix(version, "v") + baseURL := fmt.Sprintf("https://github.com/%s/releases/download/%s", *repo, *releaseTag) + + checksums, err := fetchChecksums(baseURL) + if err != nil { + fatal(err) + } + + targets, err := selectPlatforms(*all) + if err != nil { + fatal(err) + } + + for _, p := range targets { + libDir := filepath.Join(root, "lib", p.id) + if err := installPlatform(root, libDir, true, baseURL, version, p, checksums, *force); err != nil { + fatal(fmt.Errorf("%s: %w", p.id, err)) + } + } + + fmt.Printf("Installed Moss C SDK %s\n", *releaseTag) + fmt.Printf("Native libraries installed under %s\n", root) +} + +// resolveWritableBindingsDir returns the bindings package CGO will compile. +// Downloaded Go modules are read-only, so consumers either need existing +// vendored bindings or must explicitly permit the installer to create them. +func resolveWritableBindingsDir(explicit string, allowVendor bool) (string, error) { + root, err := resolveBindingsDir(explicit) + if err != nil { + return "", err + } + if isWritableDir(root) && (explicit != "" || strings.TrimSpace(os.Getenv("MOSS_BINDINGS_DIR")) != "" || !isModuleCacheDir(root)) { + return root, nil + } + if explicit != "" || strings.TrimSpace(os.Getenv("MOSS_BINDINGS_DIR")) != "" { + return "", fmt.Errorf("bindings directory %s is not writable", root) + } + if vendoredRoot, err := bindingsDirFromGoList("-mod=vendor"); err == nil && isWritableDir(vendoredRoot) { + return vendoredRoot, nil + } + if !allowVendor { + return "", errors.New("the downloaded bindings module is read-only; run `go mod vendor` first, or rerun moss install with --vendor to allow it to regenerate vendor/") + } + + goWork, err := goEnv("GOWORK") + if err != nil { + return "", fmt.Errorf("detect Go workspace: %w", err) + } + if goWork == "" || goWork == "off" { + goMod, err := goEnv("GOMOD") + if err != nil || goMod == os.DevNull || goMod == "" { + return "", fmt.Errorf("the downloaded bindings module is read-only; run this command from a Go module that imports the Moss SDK") + } + } + if err := vendorDependencies(goWork); err != nil { + return "", err + } + + root, err = bindingsDirFromGoList("-mod=vendor") + if err != nil { + return "", fmt.Errorf("locate vendored Moss bindings after vendoring dependencies: %w", err) + } + if !isWritableDir(root) { + return "", fmt.Errorf("vendored bindings directory %s is not writable", root) + } + return root, nil +} + +func isModuleCacheDir(dir string) bool { + cacheDir, err := goEnv("GOMODCACHE") + if err != nil || cacheDir == "" { + return false + } + cacheDir, err = filepath.Abs(cacheDir) + if err != nil { + return false + } + dir, err = filepath.Abs(dir) + if err != nil { + return false + } + rel, err := filepath.Rel(cacheDir, dir) + return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +func vendorDependencies(goWork string) error { + args := vendorArgs(goWork) + out, err := exec.Command("go", args...).CombinedOutput() + if err != nil { + return fmt.Errorf("vendor Moss bindings for native installation: %w\n%s", err, strings.TrimSpace(string(out))) + } + return nil +} + +func vendorArgs(goWork string) []string { + if goWork != "" && goWork != "off" { + return []string{"work", "vendor"} + } + return []string{"mod", "vendor"} +} + +func resolveBindingsDir(explicit string) (string, error) { + if explicit != "" { + return filepath.Abs(explicit) + } + if env := strings.TrimSpace(os.Getenv("MOSS_BINDINGS_DIR")); env != "" { + return filepath.Abs(env) + } + if dir, err := bindingsDirFromGoList(); err == nil { + return dir, nil + } + _, file, _, ok := runtime.Caller(0) + if !ok { + return "", errors.New("unable to locate bindings directory; set MOSS_BINDINGS_DIR or run from a module that requires github.com/usemoss/moss/sdks/go/bindings") + } + return filepath.Abs(filepath.Join(filepath.Dir(file), "..", "..", "bindings")) +} + +func bindingsDirFromGoList(args ...string) (string, error) { + args = append([]string{"list"}, args...) + args = append(args, "-f", "{{.Dir}}", bindingsModulePath) + cmd := exec.Command("go", args...) + out, err := cmd.Output() + if err != nil { + return "", err + } + dir := strings.TrimSpace(string(out)) + if dir == "" { + return "", errors.New("bindings package directory not found") + } + return filepath.Abs(dir) +} + +func isWritableDir(dir string) bool { + if dir == "" { + return false + } + test := filepath.Join(dir, ".moss-install-write-test") + if err := os.WriteFile(test, []byte("ok"), 0o644); err != nil { + return false + } + _ = os.Remove(test) + return true +} + +func goEnv(name string) (string, error) { + out, err := exec.Command("go", "env", name).Output() + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} + +func nativeReleaseTag(bindingsRoot string) (string, error) { + f, err := os.Open(filepath.Join(bindingsRoot, "version.go")) + if err != nil { + return "", fmt.Errorf("read bindings native version: %w", err) + } + defer f.Close() + + const prefix = "const NativeLibReleaseTag = \"" + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if !strings.HasPrefix(line, prefix) || !strings.HasSuffix(line, "\"") { + continue + } + tag := strings.TrimSuffix(strings.TrimPrefix(line, prefix), "\"") + if tag != "" { + return tag, nil + } + } + if err := scanner.Err(); err != nil { + return "", fmt.Errorf("read bindings native version: %w", err) + } + return "", errors.New("NativeLibReleaseTag not found in bindings/version.go; pass -release explicitly") +} + +func selectPlatforms(all bool) ([]platform, error) { + if all { + return platforms, nil + } + id, err := currentPlatformID() + if err != nil { + return nil, err + } + for _, p := range platforms { + if p.id == id { + return []platform{p}, nil + } + } + return nil, fmt.Errorf("unsupported platform %s/%s", runtime.GOOS, runtime.GOARCH) +} + +func currentPlatformID() (string, error) { + switch runtime.GOOS + "/" + runtime.GOARCH { + case "linux/amd64": + return "linux-amd64", nil + case "linux/arm64": + return "linux-arm64", nil + case "darwin/arm64": + return "darwin-arm64", nil + default: + return "", fmt.Errorf("unsupported platform %s/%s", runtime.GOOS, runtime.GOARCH) + } +} + +func fetchChecksums(baseURL string) (map[string]string, error) { + resp, err := http.Get(baseURL + "/checksums-sha256.txt") + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("checksums: HTTP %d", resp.StatusCode) + } + + checksums := map[string]string{} + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + parts := strings.Fields(strings.TrimSpace(scanner.Text())) + if len(parts) == 2 { + checksums[parts[1]] = parts[0] + } + } + return checksums, scanner.Err() +} + +func installPlatform(bindingsRoot, libDir string, installHeader bool, baseURL, version string, p platform, checksums map[string]string, force bool) error { + archive := fmt.Sprintf("libmoss-v%s-%s.tar.gz", version, p.triple) + wantChecksum, ok := checksums[archive] + if !ok { + return fmt.Errorf("checksum not found for %s", archive) + } + + destDir := libDir + destLib := filepath.Join(destDir, p.libFile) + headerPath := filepath.Join(bindingsRoot, "include", "libmoss.h") + + if !force && fileExists(destLib) && (!installHeader || fileExists(headerPath)) && receiptMatches(filepath.Join(destDir, installReceiptName), archive, wantChecksum) { + fmt.Printf("skip %s (already installed)\n", p.id) + return nil + } + + if err := os.MkdirAll(destDir, 0o755); err != nil { + return err + } + if installHeader { + if err := os.MkdirAll(filepath.Dir(headerPath), 0o755); err != nil { + return err + } + } + + tmp, err := os.MkdirTemp("", "moss-install-*") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + + archivePath := filepath.Join(tmp, archive) + if err := downloadFile(baseURL+"/"+archive, archivePath, wantChecksum); err != nil { + return err + } + + extractedRoot := filepath.Join(tmp, fmt.Sprintf("libmoss-v%s-%s", version, p.triple)) + if err := extractTarGz(archivePath, extractedRoot, version, p.triple); err != nil { + return err + } + + if installHeader { + if err := copyFile(filepath.Join(extractedRoot, "include", "libmoss.h"), headerPath); err != nil { + return err + } + } + if err := copyFile(filepath.Join(extractedRoot, p.srcLib), destLib); err != nil { + return err + } + if err := writeReceipt(filepath.Join(destDir, installReceiptName), archive, wantChecksum); err != nil { + return err + } + + fmt.Printf("installed %s -> %s\n", p.id, destLib) + return nil +} + +func fileExists(path string) bool { + info, err := os.Stat(path) + return err == nil && !info.IsDir() +} + +func receiptMatches(path, archive, checksum string) bool { + contents, err := os.ReadFile(path) + if err != nil { + return false + } + return strings.TrimSpace(string(contents)) == checksum+" "+archive +} + +func writeReceipt(path, archive, checksum string) error { + tmp, err := os.CreateTemp(filepath.Dir(path), ".moss-install-*") + if err != nil { + return err + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + + if _, err := fmt.Fprintf(tmp, "%s %s\n", checksum, archive); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmpPath, path) +} + +func extractTarGz(archivePath, destRoot, version, triple string) error { + f, err := os.Open(archivePath) + if err != nil { + return err + } + defer f.Close() + + gz, err := gzip.NewReader(f) + if err != nil { + return err + } + defer gz.Close() + + prefix := fmt.Sprintf("libmoss-v%s-%s/", version, triple) + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + if hdr.Typeflag != tar.TypeReg || !strings.HasPrefix(hdr.Name, prefix) { + continue + } + rel := strings.TrimPrefix(hdr.Name, prefix) + target, err := safeArchiveTarget(destRoot, rel) + if err != nil { + return fmt.Errorf("invalid archive path %q: %w", hdr.Name, err) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + if err := writeFile(target, tr, hdr.Mode); err != nil { + return err + } + } +} + +func safeArchiveTarget(destRoot, rel string) (string, error) { + clean := path.Clean(rel) + if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || path.IsAbs(clean) { + return "", errors.New("path escapes extraction root") + } + + root, err := filepath.Abs(destRoot) + if err != nil { + return "", err + } + target, err := filepath.Abs(filepath.Join(root, filepath.FromSlash(clean))) + if err != nil { + return "", err + } + relative, err := filepath.Rel(root, target) + if err != nil { + return "", err + } + if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", errors.New("path escapes extraction root") + } + return target, nil +} + +func writeFile(path string, r io.Reader, mode int64) error { + out, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fileMode(mode)) + if err != nil { + return err + } + defer out.Close() + _, err = io.Copy(out, r) + return err +} + +func fileMode(mode int64) os.FileMode { + if mode == 0 { + return 0o644 + } + return os.FileMode(mode) +} + +func downloadFile(url, dest, wantChecksum string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download %s: HTTP %d", url, resp.StatusCode) + } + + tmp, err := os.CreateTemp(filepath.Dir(dest), ".download-*") + if err != nil { + return err + } + tmpPath := tmp.Name() + + hasher := sha256.New() + if _, err := io.Copy(io.MultiWriter(tmp, hasher), resp.Body); err != nil { + tmp.Close() + os.Remove(tmpPath) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(tmpPath) + return err + } + + got := hex.EncodeToString(hasher.Sum(nil)) + if got != wantChecksum { + os.Remove(tmpPath) + return fmt.Errorf("checksum mismatch for %s: got %s want %s", filepath.Base(dest), got, wantChecksum) + } + return os.Rename(tmpPath, dest) +} + +func copyFile(src, dest string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + tmp, err := os.CreateTemp(filepath.Dir(dest), ".moss-install-*") + if err != nil { + return err + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + + if err := tmp.Chmod(0o644); err != nil { + tmp.Close() + return err + } + if _, err := io.Copy(tmp, in); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmpPath, dest) +} + +func fatal(err error) { + fmt.Fprintf(os.Stderr, "moss install: %v\n", err) + os.Exit(1) +} diff --git a/sdks/go/tools/install/main_test.go b/sdks/go/tools/install/main_test.go new file mode 100644 index 00000000..22276ffd --- /dev/null +++ b/sdks/go/tools/install/main_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestSafeArchiveTarget(t *testing.T) { + root := t.TempDir() + + target, err := safeArchiveTarget(root, "include/libmoss.h") + if err != nil { + t.Fatalf("safeArchiveTarget returned an error: %v", err) + } + want := filepath.Join(root, "include", "libmoss.h") + if target != want { + t.Fatalf("safeArchiveTarget() = %q, want %q", target, want) + } + + for _, input := range []string{"..", "../outside", "/outside"} { + if _, err := safeArchiveTarget(root, input); err == nil { + t.Errorf("safeArchiveTarget(%q) succeeded, want error", input) + } + } +} + +func TestNativeReleaseTag(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "version.go"), []byte("package mosscore\n\nconst NativeLibReleaseTag = \"c-sdk-v1.2.3\"\n"), 0o644); err != nil { + t.Fatal(err) + } + + tag, err := nativeReleaseTag(root) + if err != nil { + t.Fatalf("nativeReleaseTag returned an error: %v", err) + } + if tag != "c-sdk-v1.2.3" { + t.Fatalf("nativeReleaseTag() = %q, want c-sdk-v1.2.3", tag) + } +} + +func TestInstallReceipt(t *testing.T) { + path := filepath.Join(t.TempDir(), installReceiptName) + if err := writeReceipt(path, "libmoss-v0.9.0-x86_64-unknown-linux-gnu.tar.gz", "abc123"); err != nil { + t.Fatalf("writeReceipt returned an error: %v", err) + } + if !receiptMatches(path, "libmoss-v0.9.0-x86_64-unknown-linux-gnu.tar.gz", "abc123") { + t.Fatal("receiptMatches() = false, want true") + } + if receiptMatches(path, "libmoss-v0.9.0-x86_64-unknown-linux-gnu.tar.gz", "different") { + t.Fatal("receiptMatches() = true for a different checksum, want false") + } +} + +func TestVendorArgs(t *testing.T) { + if got, want := vendorArgs(""), []string{"mod", "vendor"}; !reflect.DeepEqual(got, want) { + t.Fatalf("vendorArgs(\"\") = %v, want %v", got, want) + } + if got, want := vendorArgs("/work/go.work"), []string{"work", "vendor"}; !reflect.DeepEqual(got, want) { + t.Fatalf("vendorArgs(workspace) = %v, want %v", got, want) + } +} + +func TestCopyFileDoesNotTruncateDestinationOnSourceError(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "libmoss.a") + if err := os.WriteFile(dest, []byte("known-good"), 0o644); err != nil { + t.Fatal(err) + } + + if err := copyFile(filepath.Join(dir, "missing"), dest); err == nil { + t.Fatal("copyFile succeeded with a missing source") + } + contents, err := os.ReadFile(dest) + if err != nil { + t.Fatal(err) + } + if string(contents) != "known-good" { + t.Fatalf("destination = %q, want known-good", contents) + } +} + +func TestIsModuleCacheDir(t *testing.T) { + cacheDir := t.TempDir() + t.Setenv("GOMODCACHE", cacheDir) + + inside := filepath.Join(cacheDir, "github.com", "usemoss", "moss") + if !isModuleCacheDir(inside) { + t.Fatalf("isModuleCacheDir(%q) = false, want true", inside) + } + if isModuleCacheDir(filepath.Dir(cacheDir)) { + t.Fatal("isModuleCacheDir() = true outside the module cache") + } +}