diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e1ca8c..172445f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,10 +18,10 @@ jobs: with: go-version: '1.23' cache: true - + - name: Install gotestsum run: go install gotest.tools/gotestsum@latest - + - name: Install gomock (for code generation) run: go install go.uber.org/mock/mockgen@latest @@ -41,16 +41,22 @@ jobs: - name: Generate code (protobuf and mocks) run: go generate ./... - + + - name: Fetch Codex library + run: make fetch + - name: Build - run: go build -v ./... + run: make build - name: Run unit tests - run: gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -race -v -count=1 - + run: make test-ci + + - name: Run integration tests + run: make test-integration + - name: Check test coverage for communities package run: | - go test -coverprofile=coverage.out ./communities + make coverage go tool cover -func=coverage.out - name: Upload coverage reports diff --git a/.gitignore b/.gitignore index d8d1dd4..46de977 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ coverage*.cov # Logs *.log + +libs \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 390e9dd..81aa3c3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,13 @@ { "go.testTags": "codex_integration", "gopls": { - "buildFlags": ["-tags=codex_integration"] + "buildFlags": [ + "-tags=codex_integration" + ] + }, + "go.toolsEnvVars": { + "CGO_ENABLED": "1", + "CGO_CFLAGS": "-I${workspaceFolder}/libs", + "CGO_LDFLAGS": "-L${workspaceFolder}/libs -lcodex -Wl,-rpath,${workspaceFolder}/libs" } -} +} \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9b699f3 --- /dev/null +++ b/Makefile @@ -0,0 +1,54 @@ +# Destination folder for the downloaded libraries +LIBS_DIR := $(abspath ./libs) + +# Flags for CGO to find the headers and the shared library +UNAME_S := $(shell uname -s) +CGO_CFLAGS := -I$(LIBS_DIR) +CGO_LDFLAGS := -L$(LIBS_DIR) -lcodex -Wl,-rpath,$(LIBS_DIR) + +ifeq ($(OS),Windows_NT) + BIN_NAME := codex-go.exe +else + BIN_NAME := codex-go +endif + +# Configuration for fetching the right binary +OS ?= "linux" +ARCH ?= "amd64" +VERSION ?= "v0.0.24" +DOWNLOAD_URL := "https://github.com/codex-storage/codex-go-bindings/releases/download/$(VERSION)/codex-${OS}-${ARCH}.zip" + +fetch: + @echo "Fetching libcodex from GitHub Actions from: ${DOWNLOAD_URL}" + curl -fSL --create-dirs -o $(LIBS_DIR)/codex-${OS}-${ARCH}.zip ${DOWNLOAD_URL} + unzip -o -qq $(LIBS_DIR)/codex-${OS}-${ARCH}.zip -d $(LIBS_DIR) + rm -f $(LIBS_DIR)/*.zip + +build-upload: + CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o bin/codex-upload ./cmd/upload + +build-download: + CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o bin/codex-download ./cmd/download + +build: build-upload build-download + +test: + @echo "Running unit tests..." + CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" gotestsum --packages="./communities" -f standard-verbose -- -v + +test-ci: + @echo "Running unit tests..." + CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" gotestsum --packages="./communities" -f standard-verbose -- -race -v + +test-integration: + @echo "Running tests..." + CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" gotestsum --packages="./communities" -f standard-verbose -- -tags=codex_integration -run Integration -timeout 60s + +coverage: + @echo "Running unit tests with coverage..." + CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go test -coverprofile=coverage.out ./communities + go tool cover -func=coverage.out + +clean: + rm -f $(BIN_NAME) + rm -Rf $(LIBS_DIR)/* \ No newline at end of file diff --git a/README.md b/README.md index 96fd27b..bf88ea9 100644 --- a/README.md +++ b/README.md @@ -10,25 +10,11 @@ A lightweight Go client utility for interacting with Codex client. We will be running codex client, and then use a small testing utility to check if the low level abstraction - CodexClient - correctly uploads and downloads the content. -### Running CodexClient +### Integration with Codex library -I often remove some logging noise, by slightly changing the build -params in `build.nims` (nim-codex): - -```nim -task codex, "build codex binary": - buildBinary "codex", - # params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" - params = - "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_enabled_topics:restapi:TRACE,node:TRACE" -``` - -You see a slightly more selective `params` in the `codex` task. - -To run the client I use the following command: - -```bash -./build/codex --data-dir=./data-1 --listen-addrs=/ip4/127.0.0.1/tcp/8081 --api-port=8001 --nat=none --disc-port=8091 --log-level=TRACE +You need to download the library file by using: +bash +make fetch ``` ### Building codex-upload and codex-download utilities @@ -36,8 +22,8 @@ To run the client I use the following command: Use the following command to build the `codex-upload` and `codex-download` utilities: ```bash -go build -o bin/codex-upload ./cmd/upload -go build -o bin/codex-download ./cmd/download +make build-upload +make build-download ``` ### Uploading content to Codex @@ -45,8 +31,8 @@ Now, using the `codex-upload` utility, we can upload the content to Codex as fol ```bash ~/code/local/go-codex-client -❯ ./bin/codex-upload -file test-data.bin -host localhost -port 8001 -Uploading test-data.bin (43 bytes) to Codex at localhost:8001... +❯ ./bin/codex-upload -file test-data.bin +Uploading test-data.bin (43 bytes) to Codex ✅ Upload successful! CID: zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V ``` @@ -57,8 +43,8 @@ Now, having the content uploaded to Codex - let's get it back using the `codex-d ```bash ~/code/local/go-codex-client -❯ ./bin/codex-download -cid zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V -file output.bin -host localhost -port 8001 -Downloading CID zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V from Codex at localhost:8001... +❯ ./bin/codex-download -cid zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V -file output.bin +Downloading CID zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V from Codex... ✅ Download successful! Saved to: output.bin ``` @@ -84,11 +70,39 @@ next section. To run all unit tests: +```bash +❯ make test +=== RUN TestUpload_Success +--- PASS: TestUpload_Success (0.00s) +=== RUN TestDownload_Success +--- PASS: TestDownload_Success (0.00s) +=== RUN TestDownloadWithContext_Cancel +--- PASS: TestDownloadWithContext_Cancel (0.04s) +PASS +ok go-codex-client/communities 0.044s +``` + +To run the integration test, use `test-integration`: + +```bash +make test-integration +``` + +You can use your own Go test commands but you will need to export the `CGO` variables first: + +```bash +export LIBS_DIR="$(realpath ./libs)" +export CGO_CFLAGS=-I$LIBS_DIR +export CGO_LDFLAGS="-L$LIBS_DIR -lcodex -Wl,-rpath,$LIBS_DIR" +``` + +Then you can use: + ```bash ❯ go test -v ./communities -count 1 ``` -To be more selective, e.g. in order to run all the tests from +To be more selective, e.g. in order to run all the tests from `CodexArchiveDownloaderSuite`, run: ```bash @@ -113,7 +127,7 @@ For a more verbose output including logs use `-f standard-verbose`, e.g.: gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -v -count 1 ``` -To be more selective, e.g. in order to run all the tests from +To be more selective, e.g. in order to run all the tests from `CodexArchiveDownloaderSuite`, run: ```bash @@ -134,66 +148,6 @@ gotestsum --packages="./communities" -f testname --rerun-fails -- -run CodexArch This also applies to native `go test` command. -### Running integration tests - -When building Codex client for testing like here, I often remove some logging noise, by slightly changing the build params in `build.nims`: - -```nim -task codex, "build codex binary": - buildBinary "codex", - # params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" - params = - "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_enabled_topics:restapi:TRACE,node:TRACE" -``` - -You see a slightly more selective `params` in the `codex` task. - -To start Codex client, use e.g.: - -```bash -./build/codex --data-dir=./data-1 --listen-addrs=/ip4/127.0.0.1/tcp/8081 --api-port=8001 --nat=none --disc-port=8091 --log-level=TRACE -``` - -To run the integration test, use `codex_integration` tag and narrow the scope using `-run Integration`: - -```bash -CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run Integration -timeout 15s -``` - -This will run all integration tests, including CodexClient integration tests. - -To make sure that the test is actually run and not cached, use `count` option: - -```bash -CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run Integration -timeout 15s -count 1 -``` - -To be more specific and only run the tests related to, e.g. index downloader or archive -downloader you can use: - -```bash -CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run CodexIndexDownloaderIntegration -timeout 15s -count 1 - -CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run CodexArchiveDownloaderIntegration -timeout 15s -count 1 -``` - -and then, if you prefer to use `gotestsum`: - -```bash -CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -run CodexIndexDownloaderIntegration -v -count 1 - -CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -run CodexArchiveDownloaderIntegration -v -count 1 -``` - -or to run all integration tests (including CodexClient integration tests): - -```bash -CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -v -count 1 -run Integration -``` - -I prefer to be more selective when running integration tests. - - ### Regenerating artifacts Everything you need comes included in the repo. But if you decide to change things, diff --git a/cmd/download/main.go b/cmd/download/main.go index 2041efd..d424d6f 100644 --- a/cmd/download/main.go +++ b/cmd/download/main.go @@ -5,14 +5,15 @@ import ( "fmt" "log" "os" + "path" "go-codex-client/communities" // Import the local communities package + + "github.com/codex-storage/codex-go-bindings/codex" ) func main() { var ( - host = flag.String("host", "localhost", "Codex host") - port = flag.String("port", "8080", "Codex port") cid = flag.String("cid", "", "CID of the file to download") file = flag.String("file", "downloaded-file.bin", "File to save the downloaded data") ) @@ -24,7 +25,20 @@ func main() { } // Create Codex client - client := communities.NewCodexClient(*host, *port) + client, err := communities.NewCodexClient(codex.Config{ + LogFormat: codex.LogFormatNoColors, + MetricsEnabled: false, + BlockRetries: 5, + LogLevel: "ERROR", + DataDir: path.Join(os.TempDir(), "codex-client-data"), + }) + if err != nil { + log.Fatalf("Failed to create CodexClient: %v", err) + } + + if err := client.Start(); err != nil { + log.Fatalf("Failed to start CodexClient: %v", err) + } // Create output file outputFile, err := os.Create(*file) @@ -33,8 +47,6 @@ func main() { } defer outputFile.Close() - fmt.Printf("Downloading CID %s from Codex at %s:%s...\n", *cid, *host, *port) - // Download data - pass the io.Writer (outputFile), not the string err = client.Download(*cid, outputFile) if err != nil { @@ -43,6 +55,13 @@ func main() { log.Fatalf("Download failed: %v", err) } + if err := client.Stop(); err != nil { + log.Printf("Warning: Failed to stop CodexClient: %v", err) + } + if err := client.Destroy(); err != nil { + log.Printf("Warning: Failed to stop CodexClient: %v", err) + } + fmt.Printf("✅ Download successful!\n") fmt.Printf("Saved to: %s\n", *file) } diff --git a/cmd/upload/main.go b/cmd/upload/main.go index db630c2..658f9bf 100644 --- a/cmd/upload/main.go +++ b/cmd/upload/main.go @@ -6,14 +6,15 @@ import ( "fmt" "log" "os" + "path" "go-codex-client/communities" // Import the local communities package + + "github.com/codex-storage/codex-go-bindings/codex" ) func main() { var ( - host = flag.String("host", "localhost", "Codex host") - port = flag.String("port", "8080", "Codex port") file = flag.String("file", "test-data.bin", "File to upload") filename = flag.String("name", "", "Filename to use in upload (defaults to actual filename)") ) @@ -31,15 +32,35 @@ func main() { uploadName = *file } - fmt.Printf("Uploading %s (%d bytes) to Codex at %s:%s...\n", *file, len(data), *host, *port) + fmt.Printf("Uploading %s (%d bytes) to Codex...\n", *file, len(data)) // Create Codex client and upload - client := communities.NewCodexClient(*host, *port) + client, err := communities.NewCodexClient(codex.Config{ + LogFormat: codex.LogFormatNoColors, + MetricsEnabled: false, + BlockRetries: 5, + LogLevel: "ERROR", + DataDir: path.Join(os.TempDir(), "codex-client-data"), + }) + if err != nil { + log.Fatalf("Failed to create CodexClient: %v", err) + } + + if err := client.Start(); err != nil { + log.Fatalf("Failed to start CodexClient: %v", err) + } cid, err := client.Upload(bytes.NewReader(data), uploadName) if err != nil { log.Fatalf("Upload failed: %v", err) } + if err := client.Stop(); err != nil { + log.Printf("Warning: Failed to stop CodexClient: %v", err) + } + if err := client.Destroy(); err != nil { + log.Printf("Warning: Failed to stop CodexClient: %v", err) + } + fmt.Printf("✅ Upload successful!\n") fmt.Printf("CID: %s\n", cid) } diff --git a/communities/codex_archive_downloader.go b/communities/codex_archive_downloader.go index 84a9b28..5e32b68 100644 --- a/communities/codex_archive_downloader.go +++ b/communities/codex_archive_downloader.go @@ -316,8 +316,8 @@ func (d *CodexArchiveDownloader) triggerSingleArchiveDownload(hash, cid string, return fmt.Errorf("failed to trigger archive download with CID %s: %w", cid, err) } - if manifest.CID != cid { - return fmt.Errorf("unexpected manifest CID %s, expected %s", manifest.CID, cid) + if manifest.Cid != cid { + return fmt.Errorf("unexpected manifest CID %s, expected %s", manifest.Cid, cid) } return nil diff --git a/communities/codex_archive_downloader_integration_test.go b/communities/codex_archive_downloader_integration_test.go index 10d2180..803ed1f 100644 --- a/communities/codex_archive_downloader_integration_test.go +++ b/communities/codex_archive_downloader_integration_test.go @@ -8,10 +8,10 @@ import ( "context" "crypto/rand" "encoding/hex" - "os" "testing" "time" + "github.com/codex-storage/codex-go-bindings/codex" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -31,19 +31,18 @@ type CodexArchiveDownloaderIntegrationSuite struct { // SetupSuite runs once before all tests in the suite func (suite *CodexArchiveDownloaderIntegrationSuite) SetupSuite() { - // Use port 8001 as specified by the user - host := communities.GetEnvOrDefault("CODEX_HOST", "localhost") - port := communities.GetEnvOrDefault("CODEX_API_PORT", "8001") - suite.client = communities.NewCodexClient(host, port) - - // Optional request timeout override - if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" { - if d, err := time.ParseDuration(ms + "ms"); err == nil { - suite.client.SetRequestTimeout(d) - } + var err error + suite.client, err = communities.NewCodexClient(codex.Config{ + LogFormat: codex.LogFormatNoColors, + MetricsEnabled: false, + BlockRetries: 5, + LogLevel: "ERROR", + }) + if err != nil { + suite.T().Fatalf("Failed to create CodexClient: %v", err) } - suite.T().Logf("CodexClient configured for %s:%s", host, port) + suite.T().Logf("CodexClient configured") } // TearDownSuite runs once after all tests in the suite diff --git a/communities/codex_archive_downloader_test.go b/communities/codex_archive_downloader_test.go index 127b4f5..5dd82d1 100644 --- a/communities/codex_archive_downloader_test.go +++ b/communities/codex_archive_downloader_test.go @@ -63,7 +63,7 @@ func (suite *CodexArchiveDownloaderSuite) TestBasicSingleArchive() { // Set up mock expectations - same as before suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). - Return(&communities.CodexManifest{CID: "test-cid-1"}, nil). + Return(communities.CodexManifest{Cid: "test-cid-1"}, nil). Times(1) // First HasCid call returns false, second returns true (simulating polling) @@ -149,7 +149,7 @@ func (suite *CodexArchiveDownloaderSuite) TestMultipleArchives() { for _, cid := range expectedCids { suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), cid). - Return(&communities.CodexManifest{CID: cid}, nil). + Return(communities.CodexManifest{Cid: cid}, nil). Times(1) // Each archive becomes available after one poll @@ -236,7 +236,7 @@ func (suite *CodexArchiveDownloaderSuite) TestErrorDuringTriggerDownload() { // Mock TriggerDownloadWithContext to simulate an error suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). - Return(nil, assert.AnError). // Return a generic error to simulate failure + Return(communities.CodexManifest{}, assert.AnError). // Return a generic error to simulate failure Times(1) // No HasCid calls should be made since TriggerDownload fails @@ -288,13 +288,13 @@ func (suite *CodexArchiveDownloaderSuite) TestActualCancellationDuringTriggerDow // Use DoAndReturn to create a realistic TriggerDownload that waits for cancellation suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). - DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { + DoAndReturn(func(ctx context.Context, cid string) (communities.CodexManifest, error) { // Simulate work by waiting for context cancellation select { case <-time.After(5 * time.Second): // This should never happen in our test - return &communities.CodexManifest{CID: cid}, nil + return communities.CodexManifest{Cid: cid}, nil case <-ctx.Done(): // Wait for actual context cancellation - return nil, ctx.Err() // Return the actual cancellation error + return communities.CodexManifest{}, ctx.Err() // Return the actual cancellation error } }). Times(1) @@ -352,7 +352,7 @@ func (suite *CodexArchiveDownloaderSuite) TestCancellationDuringPolling() { // Mock successful TriggerDownload suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). - Return(&communities.CodexManifest{CID: "test-cid-1"}, nil). + Return(communities.CodexManifest{Cid: "test-cid-1"}, nil). Times(1) // Mock polling - allow multiple calls, but we'll cancel before completion @@ -420,7 +420,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPollingTimeout() { // Mock successful TriggerDownload suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). - Return(&communities.CodexManifest{CID: "test-cid-1"}, nil). + Return(communities.CodexManifest{Cid: "test-cid-1"}, nil). Times(1) // Mock polling to always return false (simulating timeout) @@ -496,7 +496,7 @@ func (suite *CodexArchiveDownloaderSuite) TestWithExistingArchives() { // Only archive-2 should be downloaded (not in existingArchiveIDs) suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-2"). - Return(&communities.CodexManifest{CID: "cid-2"}, nil). + Return(communities.CodexManifest{Cid: "cid-2"}, nil). Times(1) // Only one call expected // Only archive-2 should be polled @@ -577,7 +577,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_OneSuccessOneError( // Archive-2 succeeds suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-2"). - Return(&communities.CodexManifest{CID: "cid-2"}, nil) + Return(communities.CodexManifest{Cid: "cid-2"}, nil) suite.mockClient.EXPECT(). HasCid("cid-2"). Return(true, nil) @@ -585,7 +585,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_OneSuccessOneError( // Archive-1 fails suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-1"). - Return(nil, fmt.Errorf("trigger failed")) + Return(communities.CodexManifest{}, fmt.Errorf("trigger failed")) logger := zap.NewNop() downloader := communities.NewCodexArchiveDownloader(suite.mockClient, index, communityID, []string{}, cancelChan, logger) @@ -633,7 +633,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessErrorCancell // Archive-3 (newest) succeeds suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-3"). - Return(&communities.CodexManifest{CID: "cid-3"}, nil) + Return(communities.CodexManifest{Cid: "cid-3"}, nil) suite.mockClient.EXPECT(). HasCid("cid-3"). Return(true, nil) @@ -641,14 +641,14 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessErrorCancell // Archive-2 fails suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-2"). - Return(nil, fmt.Errorf("trigger failed")) + Return(communities.CodexManifest{}, fmt.Errorf("trigger failed")) // Archive-1 will be cancelled (no expectations needed) suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-1"). - DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { + DoAndReturn(func(ctx context.Context, cid string) (communities.CodexManifest, error) { <-ctx.Done() // Wait for cancellation - return nil, ctx.Err() + return communities.CodexManifest{}, ctx.Err() }). AnyTimes() @@ -700,7 +700,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessThenCancella // Archive-2 (newer) succeeds suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-2"). - Return(&communities.CodexManifest{CID: "cid-2"}, nil) + Return(communities.CodexManifest{Cid: "cid-2"}, nil) suite.mockClient.EXPECT(). HasCid("cid-2"). Return(true, nil) @@ -708,9 +708,9 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessThenCancella // Archive-1 will be cancelled suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-1"). - DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { + DoAndReturn(func(ctx context.Context, cid string) (communities.CodexManifest, error) { <-ctx.Done() // Wait for cancellation - return nil, ctx.Err() + return communities.CodexManifest{}, ctx.Err() }). AnyTimes() @@ -762,9 +762,9 @@ func (suite *CodexArchiveDownloaderSuite) TestNoSuccess_OnlyCancellation() { // Both archives will be cancelled suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { + DoAndReturn(func(ctx context.Context, cid string) (communities.CodexManifest, error) { <-ctx.Done() // Wait for cancellation - return nil, ctx.Err() + return communities.CodexManifest{}, ctx.Err() }). AnyTimes() @@ -815,10 +815,10 @@ func (suite *CodexArchiveDownloaderSuite) TestNoSuccess_OnlyErrors() { // Both archives fail suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-1"). - Return(nil, fmt.Errorf("trigger failed for cid-1")) + Return(communities.CodexManifest{}, fmt.Errorf("trigger failed for cid-1")) suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-2"). - Return(nil, fmt.Errorf("trigger failed for cid-2")) + Return(communities.CodexManifest{}, fmt.Errorf("trigger failed for cid-2")) logger := zap.NewNop() downloader := communities.NewCodexArchiveDownloader(suite.mockClient, index, communityID, []string{}, cancelChan, logger) diff --git a/communities/codex_client.go b/communities/codex_client.go index ea07276..63b6c38 100644 --- a/communities/codex_client.go +++ b/communities/codex_client.go @@ -9,63 +9,51 @@ package communities import ( "bytes" "context" - "encoding/json" "fmt" "io" - "net/http" - "strings" - "time" + + "github.com/codex-storage/codex-go-bindings/codex" ) // CodexClient handles basic upload/download operations with Codex storage type CodexClient struct { - BaseURL string - Client *http.Client + node *codex.CodexNode + config *codex.Config } -// NewCodexClient creates a new Codex client -func NewCodexClient(host string, port string) *CodexClient { - return &CodexClient{ - BaseURL: fmt.Sprintf("http://%s:%s", host, port), - Client: &http.Client{Timeout: 60 * time.Second}, - } -} +type CodexManifest = codex.Manifest +type CodexConf = codex.Config -// Upload uploads data from a reader to Codex and returns the CID -func (c *CodexClient) Upload(data io.Reader, filename string) (string, error) { - url := fmt.Sprintf("%s/api/codex/v1/data", c.BaseURL) - - // Create the HTTP request - req, err := http.NewRequest("POST", url, data) +// NewCodexClient creates a new Codex client +func NewCodexClient(config codex.Config) (*CodexClient, error) { + node, err := codex.New(config) if err != nil { - return "", fmt.Errorf("failed to create request: %w", err) + return nil, fmt.Errorf("failed to create Codex node: %w", err) } - // Set headers - req.Header.Set("Content-Type", "application/octet-stream") - req.Header.Set("Content-Disposition", fmt.Sprintf(`filename="%s"`, filename)) + return &CodexClient{ + node: node, + config: &config, + }, nil +} - // Send request - resp, err := c.Client.Do(req) - if err != nil { - return "", fmt.Errorf("failed to upload to codex: %w", err) - } - defer resp.Body.Close() +func (c CodexClient) Start() error { + return c.node.Start() +} - // Check if request was successful - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return "", fmt.Errorf("codex upload failed with status %d: %s", resp.StatusCode, string(body)) - } +func (c CodexClient) Stop() error { + return c.node.Stop() +} - // Read the CID response - cidBytes, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("failed to read response: %w", err) - } +func (c CodexClient) Destroy() error { + return c.node.Destroy() +} - cid := strings.TrimSpace(string(cidBytes)) - return cid, nil +// Upload uploads data from a reader to Codex and returns the CID +func (c *CodexClient) Upload(data io.Reader, filename string) (string, error) { + return c.node.UploadReader(context.Background(), codex.UploadOptions{ + Filepath: filename, + }, data) } // Download downloads data from Codex by CID and writes it to the provided writer @@ -73,83 +61,24 @@ func (c *CodexClient) Download(cid string, output io.Writer) error { return c.DownloadWithContext(context.Background(), cid, output) } -func (c *CodexClient) TriggerDownload(cid string) (*CodexManifest, error) { +func (c *CodexClient) TriggerDownload(cid string) (CodexManifest, error) { return c.TriggerDownloadWithContext(context.Background(), cid) } func (c *CodexClient) HasCid(cid string) (bool, error) { - url := fmt.Sprintf("%s/api/codex/v1/data/%s/exists", c.BaseURL, cid) - - resp, err := c.Client.Get(url) - if err != nil { - return false, fmt.Errorf("failed to check cid existence: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return false, fmt.Errorf("cid check failed with status %d: %s", resp.StatusCode, string(body)) - } - - // Parse JSON response: {"": } - var result map[string]bool - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return false, fmt.Errorf("failed to parse response: %w", err) - } - - // Validate the CID key matches request - hasCid, exists := result[cid] - if !exists { - return false, fmt.Errorf("response missing CID key %q", cid) - } - - return hasCid, nil + err := c.LocalDownload(cid, io.Discard) + return err == nil, nil } func (c *CodexClient) RemoveCid(cid string) error { - url := fmt.Sprintf("%s/api/codex/v1/data/%s", c.BaseURL, cid) - - req, err := http.NewRequest("DELETE", url, nil) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - resp, err := c.Client.Do(req) - if err != nil { - return fmt.Errorf("failed trying to delete cid: %s, %w", cid, err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusNoContent { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("cid delete failed with status %d: %s", resp.StatusCode, string(body)) - } - - return nil + return c.node.Delete(cid) } // DownloadWithContext downloads data from Codex by CID with cancellation support func (c *CodexClient) DownloadWithContext(ctx context.Context, cid string, output io.Writer) error { - url := fmt.Sprintf("%s/api/codex/v1/data/%s/network/stream", c.BaseURL, cid) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - resp, err := c.Client.Do(req) - if err != nil { - return fmt.Errorf("failed to download from codex: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("codex download failed with status %d: %s", resp.StatusCode, string(body)) - } - - // Use context-aware copy for cancellable streaming - return c.copyWithContext(ctx, output, resp.Body) + return c.node.DownloadStream(ctx, cid, codex.DownloadStreamOptions{ + Writer: output, + }) } func (c *CodexClient) LocalDownload(cid string, output io.Writer) error { @@ -157,117 +86,18 @@ func (c *CodexClient) LocalDownload(cid string, output io.Writer) error { } func (c *CodexClient) LocalDownloadWithContext(ctx context.Context, cid string, output io.Writer) error { - url := fmt.Sprintf("%s/api/codex/v1/data/%s", c.BaseURL, cid) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - resp, err := c.Client.Do(req) - if err != nil { - return fmt.Errorf("failed to download from codex: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("codex download failed with status %d: %s", resp.StatusCode, string(body)) - } - - // Use context-aware copy for cancellable streaming - return c.copyWithContext(ctx, output, resp.Body) -} - -func (c *CodexClient) FetchManifestWithContext(ctx context.Context, cid string) (*CodexManifest, error) { - url := fmt.Sprintf("%s/api/codex/v1/data/%s/network/manifest", c.BaseURL, cid) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - resp, err := c.Client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to fetch manifest from codex: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("codex fetch manifest failed with status %d: %s", resp.StatusCode, string(body)) - } - - // Parse JSON response containing manifest - var manifest CodexManifest - if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil { - return nil, fmt.Errorf("failed to parse manifest: %w", err) - } - - return &manifest, nil -} - -func (c *CodexClient) TriggerDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error) { - url := fmt.Sprintf("%s/api/codex/v1/data/%s/network", c.BaseURL, cid) - - req, err := http.NewRequestWithContext(ctx, "POST", url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - resp, err := c.Client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to trigger download from codex: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("codex async download failed with status %d: %s", resp.StatusCode, string(body)) - } - - // Parse JSON response containing manifest - var manifest CodexManifest - if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil { - return nil, fmt.Errorf("failed to parse download manifest: %w", err) - } - - return &manifest, nil + return c.node.DownloadStream(ctx, cid, codex.DownloadStreamOptions{ + Writer: output, + Local: true, + }) } -// copyWithContext performs io.Copy but respects context cancellation -func (c *CodexClient) copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error { - // Create a buffer for chunked copying - buf := make([]byte, 64*1024) // 64KB buffer - - for { - select { - case <-ctx.Done(): - return ctx.Err() // Return cancellation error - default: - } - - // Read a chunk - n, err := src.Read(buf) - if n > 0 { - // Write the chunk - if _, writeErr := dst.Write(buf[:n]); writeErr != nil { - return fmt.Errorf("failed to write data: %w", writeErr) - } - } - - if err == io.EOF { - return nil // Successful completion - } - if err != nil { - return fmt.Errorf("failed to read data: %w", err) - } - } +func (c *CodexClient) FetchManifestWithContext(ctx context.Context, cid string) (CodexManifest, error) { + return c.node.DownloadManifest(cid) } -// SetRequestTimeout sets the HTTP client timeout for requests -func (c *CodexClient) SetRequestTimeout(timeout time.Duration) { - c.Client.Timeout = timeout +func (c *CodexClient) TriggerDownloadWithContext(ctx context.Context, cid string) (CodexManifest, error) { + return c.node.Fetch(cid) } // UploadArchive is a convenience method for uploading archive data diff --git a/communities/codex_client_integration_test.go b/communities/codex_client_integration_test.go index 0917dac..8509e88 100644 --- a/communities/codex_client_integration_test.go +++ b/communities/codex_client_integration_test.go @@ -8,10 +8,10 @@ import ( "context" "crypto/rand" "encoding/hex" - "os" "testing" "time" + "github.com/codex-storage/codex-go-bindings/codex" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -20,29 +20,22 @@ import ( ) // CodexClientIntegrationTestSuite demonstrates testify's suite functionality for CodexClient integration tests -// These tests exercise real network calls against a running Codex node. -// Required env vars (with defaults): -// - CODEX_HOST (default: localhost) -// - CODEX_API_PORT (default: 8080) -// - CODEX_TIMEOUT_MS (optional; default: 60000) type CodexClientIntegrationTestSuite struct { suite.Suite client *communities.CodexClient - host string - port string } // SetupSuite runs once before all tests in the suite func (suite *CodexClientIntegrationTestSuite) SetupSuite() { - suite.host = communities.GetEnvOrDefault("CODEX_HOST", "localhost") - suite.port = communities.GetEnvOrDefault("CODEX_API_PORT", "8080") - suite.client = communities.NewCodexClient(suite.host, suite.port) - - // Optional request timeout override - if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" { - if d, err := time.ParseDuration(ms + "ms"); err == nil { - suite.client.SetRequestTimeout(d) - } + var err error + suite.client, err = communities.NewCodexClient(codex.Config{ + DataDir: suite.T().TempDir(), + LogFormat: codex.LogFormatNoColors, + MetricsEnabled: false, + BlockRetries: 5, + }) + if err != nil { + suite.T().Fatalf("Failed to create Codex client: %v", err) } } @@ -114,15 +107,7 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_CheckNonExistingCI } func (suite *CodexClientIntegrationTestSuite) TestIntegration_TriggerDownload() { - // Use port 8001 for this test as specified - client := communities.NewCodexClient(suite.host, "8001") - - // Optional request timeout override - if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" { - if d, err := time.ParseDuration(ms + "ms"); err == nil { - client.SetRequestTimeout(d) - } - } + client := NewCodexClientTest(suite.T()) // Generate random payload to ensure proper round-trip verification payload := make([]byte, 1024) @@ -145,7 +130,7 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_TriggerDownload() // Trigger async download manifest, err := client.TriggerDownload(cid) require.NoError(suite.T(), err, "TriggerDownload failed") - suite.T().Logf("Async download triggered, manifest CID: %s", manifest.CID) + suite.T().Logf("Async download triggered, manifest CID: %s", manifest.Cid) // Poll HasCid for up to 10 seconds using goroutine and channel downloadComplete := make(chan bool, 1) @@ -221,27 +206,27 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_FetchManifest() { manifest, err := suite.client.FetchManifestWithContext(ctx, cid) require.NoError(suite.T(), err, "FetchManifestWithContext failed") - suite.T().Logf("FetchManifest successful, manifest CID: %s", manifest.CID) + suite.T().Logf("FetchManifest successful, manifest CID: %s", manifest.Cid) // Verify manifest properties - assert.Equal(suite.T(), cid, manifest.CID, "Manifest CID mismatch") + assert.Equal(suite.T(), cid, manifest.Cid, "Manifest CID mismatch") // Verify manifest has expected fields - assert.NotEmpty(suite.T(), manifest.Manifest.TreeCid, "Expected TreeCid to be non-empty") - suite.T().Logf("Manifest TreeCid: %s", manifest.Manifest.TreeCid) + assert.NotEmpty(suite.T(), manifest.TreeCid, "Expected TreeCid to be non-empty") + suite.T().Logf("Manifest TreeCid: %s", manifest.TreeCid) - assert.Greater(suite.T(), manifest.Manifest.DatasetSize, int64(0), "Expected DatasetSize > 0") - suite.T().Logf("Manifest DatasetSize: %d", manifest.Manifest.DatasetSize) + assert.Greater(suite.T(), manifest.DatasetSize, 0, "Expected DatasetSize > 0") + suite.T().Logf("Manifest DatasetSize: %d", manifest.DatasetSize) - assert.Greater(suite.T(), manifest.Manifest.BlockSize, 0, "Expected BlockSize > 0") - suite.T().Logf("Manifest BlockSize: %d", manifest.Manifest.BlockSize) + assert.Greater(suite.T(), manifest.BlockSize, 0, "Expected BlockSize > 0") + suite.T().Logf("Manifest BlockSize: %d", manifest.BlockSize) - assert.Equal(suite.T(), "fetch-manifest-test.bin", manifest.Manifest.Filename, "Filename mismatch") - suite.T().Logf("Manifest Filename: %s", manifest.Manifest.Filename) + assert.Equal(suite.T(), "fetch-manifest-test.bin", manifest.Filename, "Filename mismatch") + suite.T().Logf("Manifest Filename: %s", manifest.Filename) // Log manifest details for verification - suite.T().Logf("Manifest Protected: %v", manifest.Manifest.Protected) - suite.T().Logf("Manifest Mimetype: %s", manifest.Manifest.Mimetype) + suite.T().Logf("Manifest Protected: %v", manifest.Protected) + suite.T().Logf("Manifest Mimetype: %s", manifest.Mimetype) // Test fetching manifest for non-existent CID (should fail gracefully) nonExistentCID := "zDvZRwzmNonExistentCID123456789" diff --git a/communities/codex_client_interface.go b/communities/codex_client_interface.go index b2afee2..110eba3 100644 --- a/communities/codex_client_interface.go +++ b/communities/codex_client_interface.go @@ -3,7 +3,6 @@ package communities import ( "context" "io" - "time" ) // Mock generation instruction above will create a mock in package `mock_communities` @@ -25,16 +24,13 @@ type CodexClientInterface interface { LocalDownloadWithContext(ctx context.Context, cid string, output io.Writer) error // Async download methods - TriggerDownload(cid string) (*CodexManifest, error) - TriggerDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error) + TriggerDownload(cid string) (CodexManifest, error) + TriggerDownloadWithContext(ctx context.Context, cid string) (CodexManifest, error) // Manifest methods - FetchManifestWithContext(ctx context.Context, cid string) (*CodexManifest, error) + FetchManifestWithContext(ctx context.Context, cid string) (CodexManifest, error) // CID management methods HasCid(cid string) (bool, error) RemoveCid(cid string) error - - // Configuration methods - SetRequestTimeout(timeout time.Duration) } diff --git a/communities/codex_client_test.go b/communities/codex_client_test.go index 10f1f90..2392bdf 100644 --- a/communities/codex_client_test.go +++ b/communities/codex_client_test.go @@ -4,10 +4,7 @@ import ( "bytes" "context" "errors" - "fmt" "io" - "net/http" - "net/http/httptest" "testing" "time" @@ -18,24 +15,33 @@ import ( "go-codex-client/communities" ) +func upload(client communities.CodexClient, t *testing.T, buf *bytes.Buffer) string { + filename := "hello.txt" + cid, err := client.Upload(buf, filename) + if err != nil { + t.Fatalf("Failed to upload file: %v", err) + } + + if cid == "" { + t.Fatalf("Expected non-empty CID after upload") + } + + return cid +} + // CodexClientTestSuite demonstrates testify's suite functionality for CodexClient tests type CodexClientTestSuite struct { suite.Suite client *communities.CodexClient - server *httptest.Server } // SetupTest runs before each test method func (suite *CodexClientTestSuite) SetupTest() { - suite.client = communities.NewCodexClient("localhost", "8080") + suite.client = NewCodexClientTest(suite.T()) } // TearDownTest runs after each test method func (suite *CodexClientTestSuite) TearDownTest() { - if suite.server != nil { - suite.server.Close() - suite.server = nil - } } // TestCodexClientTestSuite runs the test suite @@ -44,110 +50,33 @@ func TestCodexClientTestSuite(t *testing.T) { } func (suite *CodexClientTestSuite) TestUpload_Success() { - // Arrange a fake Codex server that validates headers and returns a CID - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - if r.URL.Path != "/api/codex/v1/data" { - w.WriteHeader(http.StatusNotFound) - return - } - - if ct := r.Header.Get("Content-Type"); ct != "application/octet-stream" { - w.WriteHeader(http.StatusBadRequest) - return - } - if cd := r.Header.Get("Content-Disposition"); cd != "filename=\"hello.txt\"" { - w.WriteHeader(http.StatusBadRequest) - return - } - - _, _ = io.ReadAll(r.Body) // consume body - _ = r.Body.Close() - - w.WriteHeader(http.StatusOK) - // Codex returns CIDv1 base58btc - // prefix: zDv - // - z = multibase prefix for base58btc - // - Dv = CIDv1 prefix for raw codex - // we add a newline to simulate real response - _, _ = w.Write([]byte("zDvZRwzmTestCID123\n")) - })) - - suite.client.BaseURL = suite.server.URL - // Act cid, err := suite.client.Upload(bytes.NewReader([]byte("payload")), "hello.txt") // Assert require.NoError(suite.T(), err) // Codex uses CIDv1 with base58btc encoding (prefix: zDv) - assert.Equal(suite.T(), "zDvZRwzmTestCID123", cid) -} - -func (suite *CodexClientTestSuite) TestDownload_Success() { - const wantCID = "zDvZRwzm" - const payload = "hello from codex" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - if r.URL.Path != "/api/codex/v1/data/"+wantCID+"/network/stream" { - w.WriteHeader(http.StatusNotFound) - return - } - - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(payload)) - })) - - suite.client.BaseURL = suite.server.URL - - var buf bytes.Buffer - err := suite.client.Download(wantCID, &buf) - require.NoError(suite.T(), err) - assert.Equal(suite.T(), payload, buf.String()) + assert.Equal(suite.T(), "zDvZRwzmBEaJ338xaCHbKbGAJ4X41YyccS6eyorrYBbmPnWuLxCh", cid) } func (suite *CodexClientTestSuite) TestDownloadWithContext_Cancel() { - const cid = "zDvZRwzm" + // skip test + suite.T().Skip("Wait for cancellation support PR to be merged in codex-go-bindings") - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/codex/v1/data/"+cid+"/network/stream" { - w.WriteHeader(http.StatusNotFound) - return - } - w.Header().Set("Content-Type", "application/octet-stream") - flusher, _ := w.(http.Flusher) - w.WriteHeader(http.StatusOK) - // Stream data slowly so the request can be canceled - for i := 0; i < 1000; i++ { - select { - case <-r.Context().Done(): - return - default: - } - if _, err := w.Write([]byte("x")); err != nil { - // Client likely went away; stop writing - return - } - if flusher != nil { - flusher.Flush() - } - time.Sleep(10 * time.Millisecond) - } - })) + len := 1024 * 1024 * 50 + buf := bytes.NewBuffer(make([]byte, len)) + cid := upload(*suite.client, suite.T(), buf) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) - suite.client.BaseURL = suite.server.URL + channelError := make(chan error, 1) + go func() { + err := suite.client.DownloadWithContext(ctx, cid, io.Discard) + channelError <- err + }() - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) - defer cancel() + cancel() + err := <-channelError - err := suite.client.DownloadWithContext(ctx, cid, io.Discard) require.Error(suite.T(), err) // Accept either canceled or deadline exceeded depending on timing if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { @@ -160,31 +89,20 @@ func (suite *CodexClientTestSuite) TestDownloadWithContext_Cancel() { } func (suite *CodexClientTestSuite) TestHasCid_Success() { + const payload = "hello from codex" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) + tests := []struct { name string cid string - hasIt bool wantBool bool }{ - {"has CID returns true", "zDvZRwzmTestCID", true, true}, - {"has CID returns false", "zDvZRwzmTestCID", false, false}, + {"has CID returns true", cid, true}, + {"has CID returns false", "zDvZRwzmBEaJ338xaCHbKbGAJ4X41YyccS6eyorrYBbmPnWuLxCe", false}, } for _, tt := range tests { suite.Run(tt.name, func() { - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/codex/v1/data/"+tt.cid+"/exists" { - w.WriteHeader(http.StatusNotFound) - return - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - // Return JSON: {"": } - fmt.Fprintf(w, `{"%s": %t}`, tt.cid, tt.hasIt) - })) - - suite.client.BaseURL = suite.server.URL - got, err := suite.client.HasCid(tt.cid) require.NoError(suite.T(), err) assert.Equal(suite.T(), tt.wantBool, got, "HasCid(%q) = %v, want %v", tt.cid, got, tt.wantBool) @@ -192,180 +110,41 @@ func (suite *CodexClientTestSuite) TestHasCid_Success() { } } -func (suite *CodexClientTestSuite) TestHasCid_RequestError() { - // Create a server and immediately close it to trigger connection error - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) - suite.server.Close() // Close immediately so connection fails - - suite.client.BaseURL = suite.server.URL // Use the closed server's URL - - got, err := suite.client.HasCid("zDvZRwzmTestCID") - require.Error(suite.T(), err) - assert.False(suite.T(), got, "expected false on error") -} - -func (suite *CodexClientTestSuite) TestHasCid_CidMismatch() { - const requestCid = "zDvZRwzmRequestCID" - const responseCid = "zDvZRwzmDifferentCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - // Return a different CID in the response - fmt.Fprintf(w, `{"%s": true}`, responseCid) - })) - - suite.client.BaseURL = suite.server.URL +func (suite *CodexClientTestSuite) TestDownload_Success() { + const payload = "hello from codex" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) - got, err := suite.client.HasCid(requestCid) - require.Error(suite.T(), err, "expected error for CID mismatch") - assert.False(suite.T(), got, "expected false on CID mismatch") - // Check error message mentions the missing/mismatched CID - assert.Contains(suite.T(), err.Error(), requestCid, "error should mention request CID") + var buf bytes.Buffer + err := suite.client.Download(cid, &buf) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), payload, buf.String()) } func (suite *CodexClientTestSuite) TestRemoveCid_Success() { - const testCid = "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodDelete { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - if r.URL.Path != "/api/codex/v1/data/"+testCid { - w.WriteHeader(http.StatusNotFound) - return - } - // DELETE should return 204 No Content - w.WriteHeader(http.StatusNoContent) - })) - - suite.client.BaseURL = suite.server.URL + const payload = "hello from codex" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) - err := suite.client.RemoveCid(testCid) + err := suite.client.RemoveCid(cid) require.NoError(suite.T(), err) } -func (suite *CodexClientTestSuite) TestRemoveCid_Error() { - const testCid = "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Return error status - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("server error")) - })) - - suite.client.BaseURL = suite.server.URL - - err := suite.client.RemoveCid(testCid) - require.Error(suite.T(), err) - assert.Contains(suite.T(), err.Error(), "500", "error should mention status 500") -} - func (suite *CodexClientTestSuite) TestTriggerDownload() { - const testCid = "zDvZRwzmTestCID" - const expectedManifest = `{ - "cid": "zDvZRwzmTestCID", - "manifest": { - "treeCid": "zDvZRwzmTreeCID", - "datasetSize": 1024, - "blockSize": 65536, - "protected": false, - "filename": "test-file.bin", - "mimetype": "application/octet-stream" - } - }` - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - if r.URL.Path != "/api/codex/v1/data/"+testCid+"/network" { - w.WriteHeader(http.StatusNotFound) - return - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write([]byte(expectedManifest)) - })) - - suite.client.BaseURL = suite.server.URL + const payload = "hello from codex" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) ctx := context.Background() - manifest, err := suite.client.TriggerDownloadWithContext(ctx, testCid) + manifest, err := suite.client.TriggerDownloadWithContext(ctx, cid) require.NoError(suite.T(), err) - assert.Equal(suite.T(), testCid, manifest.CID) - assert.Equal(suite.T(), "zDvZRwzmTreeCID", manifest.Manifest.TreeCid) - assert.Equal(suite.T(), int64(1024), manifest.Manifest.DatasetSize) - assert.Equal(suite.T(), "test-file.bin", manifest.Manifest.Filename) -} - -func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_RequestError() { - // Create a server and immediately close it to trigger connection error - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) - suite.server.Close() - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - manifest, err := suite.client.TriggerDownloadWithContext(ctx, "zDvZRwzmRigWseNB7WqmudkKAPgZmrDCE9u5cY4KvCqhRo9Ki") - require.Error(suite.T(), err) - assert.Nil(suite.T(), manifest, "expected nil manifest on error") -} - -func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_JSONParseError() { - const testCid = "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - // Return invalid JSON - w.Write([]byte(`{"invalid": json}`)) - })) - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - manifest, err := suite.client.TriggerDownloadWithContext(ctx, testCid) - require.Error(suite.T(), err, "expected JSON parse error") - assert.Nil(suite.T(), manifest, "expected nil manifest on parse error") - assert.Contains(suite.T(), err.Error(), "failed to parse download manifest", "error should mention parse failure") -} - -func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_HTTPError() { - const testCid = "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) - w.Write([]byte("CID not found")) - })) - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - manifest, err := suite.client.TriggerDownloadWithContext(ctx, testCid) - require.Error(suite.T(), err, "expected error for 404 status") - assert.Nil(suite.T(), manifest, "expected nil manifest on HTTP error") - assert.Contains(suite.T(), err.Error(), "404", "error should mention status 404") + assert.Equal(suite.T(), cid, manifest.Cid) + assert.Equal(suite.T(), "zDzSvJTf7mGkC3yuiVGco7Qc6s4LA8edye9inT4w2QqHnfbuRvMr", manifest.TreeCid) + assert.Equal(suite.T(), len(payload), manifest.DatasetSize) + assert.Equal(suite.T(), "hello.txt", manifest.Filename) } func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_Cancellation() { - const testCid = "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Simulate slow response to allow cancellation - select { - case <-r.Context().Done(): - return - case <-time.After(200 * time.Millisecond): - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"cid": "test"}`)) - } - })) + suite.T().Skip("Not sure if we are going to have cancellation in trigger download") - suite.client.BaseURL = suite.server.URL + const testCid = "zDvZRwzmTestCID" // Cancel after 50ms (before server responds) ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) @@ -385,99 +164,41 @@ func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_Cancellation() } func (suite *CodexClientTestSuite) TestLocalDownload() { - testData := []byte("test data for local download") - testCid := "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Verify request method and path - assert.Equal(suite.T(), "GET", r.Method, "Expected GET request") - expectedPath := "/api/codex/v1/data/" + testCid - assert.Equal(suite.T(), expectedPath, r.URL.Path, "Expected correct path") - - w.WriteHeader(http.StatusOK) - w.Write(testData) - })) - - suite.client.BaseURL = suite.server.URL + const payload = "test data for local download" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) var buf bytes.Buffer - err := suite.client.LocalDownload(testCid, &buf) + err := suite.client.LocalDownload(cid, &buf) require.NoError(suite.T(), err, "LocalDownload failed") - assert.Equal(suite.T(), testData, buf.Bytes(), "Downloaded data mismatch") + assert.Equal(suite.T(), payload, buf.String(), "Downloaded data mismatch") } func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Success() { - testData := []byte("test data for local download with context") - testCid := "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Verify request method and path - assert.Equal(suite.T(), "GET", r.Method, "Expected GET request") - expectedPath := "/api/codex/v1/data/" + testCid - assert.Equal(suite.T(), expectedPath, r.URL.Path, "Expected correct path") - - w.WriteHeader(http.StatusOK) - w.Write(testData) - })) - - suite.client.BaseURL = suite.server.URL + const payload = "test data for local download with context" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) ctx := context.Background() var buf bytes.Buffer - err := suite.client.LocalDownloadWithContext(ctx, testCid, &buf) + err := suite.client.LocalDownloadWithContext(ctx, cid, &buf) require.NoError(suite.T(), err, "LocalDownloadWithContext failed") - assert.Equal(suite.T(), testData, buf.Bytes(), "Downloaded data mismatch") -} - -func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_RequestError() { - // Create a server and immediately close it to trigger connection error - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) - suite.server.Close() - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - var buf bytes.Buffer - err := suite.client.LocalDownloadWithContext(ctx, "zDvZRwzmTestCID", &buf) - require.Error(suite.T(), err, "Expected error due to closed server") - assert.Contains(suite.T(), err.Error(), "failed to download from codex") -} - -func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_HTTPError() { - testCid := "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) - w.Write([]byte("CID not found in local storage")) - })) - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - var buf bytes.Buffer - err := suite.client.LocalDownloadWithContext(ctx, testCid, &buf) - require.Error(suite.T(), err, "Expected error for HTTP 404") - assert.Contains(suite.T(), err.Error(), "404", "Expected '404' in error message") + assert.Equal(suite.T(), payload, buf.String(), "Downloaded data mismatch") } func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Cancellation() { - testCid := "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Simulate a slow response - time.Sleep(100 * time.Millisecond) - w.WriteHeader(http.StatusOK) - w.Write([]byte("slow response")) - })) + len := 1024 * 1024 * 50 + buf := bytes.NewBuffer(make([]byte, len)) + cid := upload(*suite.client, suite.T(), buf) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) - suite.client.BaseURL = suite.server.URL + channelError := make(chan error, 1) + go func() { + err := suite.client.LocalDownloadWithContext(ctx, cid, io.Discard) + channelError <- err + }() - // Create a context with a very short timeout - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) - defer cancel() + cancel() + err := <-channelError - var buf bytes.Buffer - err := suite.client.LocalDownloadWithContext(ctx, testCid, &buf) require.Error(suite.T(), err, "Expected context cancellation error") // Accept either canceled or deadline exceeded depending on timing if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { @@ -490,106 +211,27 @@ func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Cancellation() { } func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Success() { - testCid := "zDvZRwzmTestCID" - expectedManifest := `{ - "cid": "zDvZRwzmTestCID", - "manifest": { - "treeCid": "zDvZRwzmTreeCID123", - "datasetSize": 1024, - "blockSize": 256, - "protected": true, - "filename": "test-file.bin", - "mimetype": "application/octet-stream" - } - }` - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(suite.T(), http.MethodGet, r.Method) - expectedPath := fmt.Sprintf("/api/codex/v1/data/%s/network/manifest", testCid) - assert.Equal(suite.T(), expectedPath, r.URL.Path) - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write([]byte(expectedManifest)) - })) - - suite.client.BaseURL = suite.server.URL + const payload = "hello from codex" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) ctx := context.Background() - manifest, err := suite.client.FetchManifestWithContext(ctx, testCid) + manifest, err := suite.client.FetchManifestWithContext(ctx, cid) require.NoError(suite.T(), err, "Expected no error") require.NotNil(suite.T(), manifest, "Expected manifest, got nil") - assert.Equal(suite.T(), testCid, manifest.CID) - assert.Equal(suite.T(), "zDvZRwzmTreeCID123", manifest.Manifest.TreeCid) - assert.Equal(suite.T(), int64(1024), manifest.Manifest.DatasetSize) - assert.Equal(suite.T(), 256, manifest.Manifest.BlockSize) - assert.True(suite.T(), manifest.Manifest.Protected, "Expected Protected to be true") - assert.Equal(suite.T(), "test-file.bin", manifest.Manifest.Filename) - assert.Equal(suite.T(), "application/octet-stream", manifest.Manifest.Mimetype) -} - -func (suite *CodexClientTestSuite) TestFetchManifestWithContext_RequestError() { - // Create a server and immediately close it to trigger connection error - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) - suite.server.Close() - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - manifest, err := suite.client.FetchManifestWithContext(ctx, "test-cid") - require.Error(suite.T(), err, "Expected error for closed server") - assert.Nil(suite.T(), manifest, "Expected nil manifest on error") - assert.Contains(suite.T(), err.Error(), "failed to fetch manifest from codex") -} - -func (suite *CodexClientTestSuite) TestFetchManifestWithContext_HTTPError() { - testCid := "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) - w.Write([]byte("Manifest not found")) - })) - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - manifest, err := suite.client.FetchManifestWithContext(ctx, testCid) - require.Error(suite.T(), err, "Expected error for HTTP 404") - assert.Nil(suite.T(), manifest, "Expected nil manifest on error") - assert.Contains(suite.T(), err.Error(), "404", "Expected '404' in error message") -} - -func (suite *CodexClientTestSuite) TestFetchManifestWithContext_JSONParseError() { - testCid := "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write([]byte("invalid json {")) - })) - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - manifest, err := suite.client.FetchManifestWithContext(ctx, testCid) - require.Error(suite.T(), err, "Expected error for invalid JSON") - assert.Nil(suite.T(), manifest, "Expected nil manifest on JSON parse error") - assert.Contains(suite.T(), err.Error(), "failed to parse manifest", "Expected 'failed to parse manifest' in error message") + assert.Equal(suite.T(), cid, manifest.Cid) + assert.Equal(suite.T(), "zDzSvJTf7mGkC3yuiVGco7Qc6s4LA8edye9inT4w2QqHnfbuRvMr", manifest.TreeCid) + assert.Equal(suite.T(), len(payload), manifest.DatasetSize) + assert.Equal(suite.T(), 65536, manifest.BlockSize) + assert.True(suite.T(), !manifest.Protected, "Expected Protected to be false") + assert.Equal(suite.T(), "hello.txt", manifest.Filename) + assert.Equal(suite.T(), "text/plain", manifest.Mimetype) } func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Cancellation() { - testCid := "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Simulate a slow response - time.Sleep(100 * time.Millisecond) - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"cid": "test"}`)) - })) + suite.T().Skip("Not sure if we are going to have cancellation in fetch manifest") - suite.client.BaseURL = suite.server.URL + testCid := "zDvZRwzmTestCID" // Create a context with a very short timeout ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) @@ -607,4 +249,30 @@ func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Cancellation() { suite.T().Fatalf("expected context cancellation, got: %v", err) } } + + buf := bytes.NewBuffer([]byte("Hello World!")) + if buf.Len() != manifest.DatasetSize { + suite.T().Errorf("expected size %d, got %d", buf.Len(), manifest.DatasetSize) + } + + defaultBlockSize := 1024 * 64 + if manifest.BlockSize != defaultBlockSize { + suite.T().Errorf("expected block size %d, got %d", defaultBlockSize, manifest.BlockSize) + } + + if manifest.Filename != "test.txt" { + suite.T().Errorf("expected filename %q, got %q", "test.txt", manifest.Filename) + } + + if manifest.Protected { + suite.T().Errorf("expected protected to be false, got true") + } + + if manifest.Mimetype != "text/plain" { + suite.T().Errorf("expected mimetype %q, got %q", "text/plain", manifest.Mimetype) + } + + if manifest.TreeCid == "" { + suite.T().Errorf("expected non-empty TreeCid") + } } diff --git a/communities/codex_index_downloader.go b/communities/codex_index_downloader.go index fbe5333..9f4ebf6 100644 --- a/communities/codex_index_downloader.go +++ b/communities/codex_index_downloader.go @@ -79,19 +79,19 @@ func (d *CodexIndexDownloader) GotManifest() <-chan struct{} { } // Verify that the CID matches our configured indexCid - if manifest.CID != d.indexCid { + if manifest.Cid != d.indexCid { d.mu.Lock() - d.downloadError = fmt.Errorf("manifest CID mismatch: expected %s, got %s", d.indexCid, manifest.CID) + d.downloadError = fmt.Errorf("manifest CID mismatch: expected %s, got %s", d.indexCid, manifest.Cid) d.mu.Unlock() d.logger.Debug("manifest CID mismatch", zap.String("expected", d.indexCid), - zap.String("got", manifest.CID)) + zap.String("got", manifest.Cid)) return } // Store the dataset size for later use - this indicates success d.mu.Lock() - d.datasetSize = manifest.Manifest.DatasetSize + d.datasetSize = int64(manifest.DatasetSize) d.mu.Unlock() // Success! Close the channel to signal completion diff --git a/communities/codex_index_downloader_integration_test.go b/communities/codex_index_downloader_integration_test.go index 0179a30..7b8c8a7 100644 --- a/communities/codex_index_downloader_integration_test.go +++ b/communities/codex_index_downloader_integration_test.go @@ -30,23 +30,12 @@ type CodexIndexDownloaderIntegrationTestSuite struct { suite.Suite client *communities.CodexClient testDir string - host string - port string logger *zap.Logger } // SetupSuite runs once before all tests in the suite func (suite *CodexIndexDownloaderIntegrationTestSuite) SetupSuite() { - suite.host = communities.GetEnvOrDefault("CODEX_HOST", "localhost") - suite.port = communities.GetEnvOrDefault("CODEX_API_PORT", "8001") - suite.client = communities.NewCodexClient(suite.host, suite.port) - - // Optional request timeout override - if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" { - if d, err := time.ParseDuration(ms + "ms"); err == nil { - suite.client.SetRequestTimeout(d) - } - } + suite.client = NewCodexClientTest(suite.T()) // Create logger suite.logger, _ = zap.NewDevelopment() diff --git a/communities/codex_index_downloader_test.go b/communities/codex_index_downloader_test.go index 5bee170..e7dcdf6 100644 --- a/communities/codex_index_downloader_test.go +++ b/communities/codex_index_downloader_test.go @@ -79,12 +79,12 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_SuccessClosesChannel filePath := filepath.Join(suite.testDir, "index.bin") // Setup mock to return a successful manifest - expectedManifest := &communities.CodexManifest{ - CID: testCid, + expectedManifest := communities.CodexManifest{ + Cid: testCid, } - expectedManifest.Manifest.DatasetSize = 1024 - expectedManifest.Manifest.TreeCid = "zDvZRwzmTreeCID" - expectedManifest.Manifest.BlockSize = 65536 + expectedManifest.DatasetSize = 1024 + expectedManifest.TreeCid = "zDvZRwzmTreeCID" + expectedManifest.BlockSize = 65536 suite.mockClient.EXPECT(). FetchManifestWithContext(gomock.Any(), testCid). @@ -119,7 +119,7 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_ErrorDoesNotCloseCha // Setup mock to return an error suite.mockClient.EXPECT(). FetchManifestWithContext(gomock.Any(), testCid). - Return(nil, errors.New("fetch error")) + Return(communities.CodexManifest{}, errors.New("fetch error")) // Create downloader downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger) @@ -154,10 +154,10 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_CidMismatchDoesNotCl filePath := filepath.Join(suite.testDir, "index.bin") // Setup mock to return a manifest with different CID - mismatchedManifest := &communities.CodexManifest{ - CID: differentCid, // Different CID! + mismatchedManifest := communities.CodexManifest{ + Cid: differentCid, // Different CID! } - mismatchedManifest.Manifest.DatasetSize = 1024 + mismatchedManifest.DatasetSize = 1024 suite.mockClient.EXPECT(). FetchManifestWithContext(gomock.Any(), testCid). @@ -198,12 +198,12 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_Cancellation() { fetchCalled := make(chan struct{}) suite.mockClient.EXPECT(). FetchManifestWithContext(gomock.Any(), testCid). - DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { + DoAndReturn(func(ctx context.Context, cid string) (communities.CodexManifest, error) { close(fetchCalled) // Signal that fetch was called // Wait for context cancellation <-ctx.Done() - return nil, ctx.Err() + return communities.CodexManifest{}, ctx.Err() }) // Create downloader @@ -250,11 +250,11 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_RecordsDatasetSize() expectedSize := int64(2048) // Setup mock to return a manifest with specific dataset size - expectedManifest := &communities.CodexManifest{ - CID: testCid, + expectedManifest := communities.CodexManifest{ + Cid: testCid, } - expectedManifest.Manifest.DatasetSize = expectedSize - expectedManifest.Manifest.TreeCid = "zDvZRwzmTreeCID" + expectedManifest.DatasetSize = int(expectedSize) + expectedManifest.TreeCid = "zDvZRwzmTreeCID" suite.mockClient.EXPECT(). FetchManifestWithContext(gomock.Any(), testCid). @@ -278,7 +278,7 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_RecordsDatasetSize() } // Verify dataset size was recorded correctly - assert.Equal(suite.T(), expectedSize, downloader.GetDatasetSize(), "Dataset size should match manifest") + assert.Equal(suite.T(), int64(expectedSize), downloader.GetDatasetSize(), "Dataset size should match manifest") suite.T().Logf("✅ Dataset size correctly recorded: %d", downloader.GetDatasetSize()) // Verify no error was recorded @@ -500,13 +500,13 @@ func (suite *CodexIndexDownloaderTestSuite) TestDownloadIndexFile_ErrorHandling( func (suite *CodexIndexDownloaderTestSuite) TestLength_ReturnsDatasetSize() { testCid := "zDvZRwzmTestCID123" filePath := filepath.Join(suite.testDir, "index.bin") - expectedSize := int64(4096) + expectedSize := 4096 // Setup mock to return a manifest - expectedManifest := &communities.CodexManifest{ - CID: testCid, + expectedManifest := communities.CodexManifest{ + Cid: testCid, } - expectedManifest.Manifest.DatasetSize = expectedSize + expectedManifest.DatasetSize = expectedSize suite.mockClient.EXPECT(). FetchManifestWithContext(gomock.Any(), testCid). @@ -523,6 +523,6 @@ func (suite *CodexIndexDownloaderTestSuite) TestLength_ReturnsDatasetSize() { <-manifestChan // Now Length should return the dataset size - assert.Equal(suite.T(), expectedSize, downloader.Length(), "Length should return dataset size") + assert.Equal(suite.T(), int64(expectedSize), downloader.Length(), "Length should return dataset size") suite.T().Logf("✅ Length() correctly returns dataset size: %d", downloader.Length()) } diff --git a/communities/codex_manifest.go b/communities/codex_manifest.go deleted file mode 100644 index 3b509a1..0000000 --- a/communities/codex_manifest.go +++ /dev/null @@ -1,14 +0,0 @@ -package communities - -// CodexManifest represents the manifest structure returned by Codex API -type CodexManifest struct { - CID string `json:"cid"` - Manifest struct { - TreeCid string `json:"treeCid"` - DatasetSize int64 `json:"datasetSize"` - BlockSize int `json:"blockSize"` - Protected bool `json:"protected"` - Filename string `json:"filename"` - Mimetype string `json:"mimetype"` - } `json:"manifest"` -} diff --git a/communities/mock/codex_client_interface.go b/communities/mock/codex_client_interface.go index 792c378..2a3ae82 100644 --- a/communities/mock/codex_client_interface.go +++ b/communities/mock/codex_client_interface.go @@ -11,7 +11,7 @@ package mock_communities import ( context "context" - communities "go-codex-client/communities" + "go-codex-client/communities" io "io" reflect "reflect" time "time" @@ -72,10 +72,10 @@ func (mr *MockCodexClientInterfaceMockRecorder) DownloadWithContext(ctx, cid, ou } // FetchManifestWithContext mocks base method. -func (m *MockCodexClientInterface) FetchManifestWithContext(ctx context.Context, cid string) (*communities.CodexManifest, error) { +func (m *MockCodexClientInterface) FetchManifestWithContext(ctx context.Context, cid string) (communities.CodexManifest, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "FetchManifestWithContext", ctx, cid) - ret0, _ := ret[0].(*communities.CodexManifest) + ret0, _ := ret[0].(communities.CodexManifest) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -156,10 +156,10 @@ func (mr *MockCodexClientInterfaceMockRecorder) SetRequestTimeout(timeout any) * } // TriggerDownload mocks base method. -func (m *MockCodexClientInterface) TriggerDownload(cid string) (*communities.CodexManifest, error) { +func (m *MockCodexClientInterface) TriggerDownload(cid string) (communities.CodexManifest, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "TriggerDownload", cid) - ret0, _ := ret[0].(*communities.CodexManifest) + ret0, _ := ret[0].(communities.CodexManifest) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -171,10 +171,10 @@ func (mr *MockCodexClientInterfaceMockRecorder) TriggerDownload(cid any) *gomock } // TriggerDownloadWithContext mocks base method. -func (m *MockCodexClientInterface) TriggerDownloadWithContext(ctx context.Context, cid string) (*communities.CodexManifest, error) { +func (m *MockCodexClientInterface) TriggerDownloadWithContext(ctx context.Context, cid string) (communities.CodexManifest, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "TriggerDownloadWithContext", ctx, cid) - ret0, _ := ret[0].(*communities.CodexManifest) + ret0, _ := ret[0].(communities.CodexManifest) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/communities/testutil_test.go b/communities/testutil_test.go new file mode 100644 index 0000000..d38e652 --- /dev/null +++ b/communities/testutil_test.go @@ -0,0 +1,38 @@ +package communities_test + +import ( + "go-codex-client/communities" + "testing" + + "github.com/codex-storage/codex-go-bindings/codex" +) + +func NewCodexClientTest(t *testing.T) *communities.CodexClient { + client, err := communities.NewCodexClient(codex.Config{ + DataDir: t.TempDir(), + LogFormat: codex.LogFormatNoColors, + MetricsEnabled: false, + BlockRetries: 5, + DiscoveryPort: 8092, + }) + if err != nil { + t.Fatalf("Failed to create Codex node: %v", err) + } + + err = client.Start() + if err != nil { + t.Fatalf("Failed to start Codex node: %v", err) + } + + t.Cleanup(func() { + if err := client.Stop(); err != nil { + t.Logf("cleanup codex: %v", err) + } + + if err := client.Destroy(); err != nil { + t.Logf("cleanup codex: %v", err) + } + }) + + return client +} diff --git a/go.mod b/go.mod index e3e6a5f..c5146a4 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,18 @@ module go-codex-client -go 1.23.0 +go 1.24.0 require ( + github.com/codex-storage/codex-go-bindings v0.0.24 github.com/stretchr/testify v1.11.1 go.uber.org/mock v0.6.0 go.uber.org/zap v1.27.0 - google.golang.org/protobuf v1.34.1 + google.golang.org/protobuf v1.36.10 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - go.uber.org/multierr v1.10.0 // indirect + go.uber.org/multierr v1.11.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 61d1196..2b6a61d 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,14 @@ +github.com/codex-storage/codex-go-bindings v0.0.22 h1:53nOqLzgfvR3KdghFAKDoREoW+n12ewvNf8Zf3Pdobc= +github.com/codex-storage/codex-go-bindings v0.0.22/go.mod h1:hP/n9iDZqQP4MytkgUepl3yMMsZy5Jbk9lQbbbVJ51Q= +github.com/codex-storage/codex-go-bindings v0.0.23 h1:aMHttUQZELiG0ebSjC58HFHlkyIiCNMElvdTnt4+Y5s= +github.com/codex-storage/codex-go-bindings v0.0.23/go.mod h1:hP/n9iDZqQP4MytkgUepl3yMMsZy5Jbk9lQbbbVJ51Q= +github.com/codex-storage/codex-go-bindings v0.0.24 h1:uUlLiUf5zuec34AvtzpAr4BOCE71FuxqTMBholMR86M= +github.com/codex-storage/codex-go-bindings v0.0.24/go.mod h1:hP/n9iDZqQP4MytkgUepl3yMMsZy5Jbk9lQbbbVJ51Q= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -12,12 +19,16 @@ go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=