fea: Codex library integration

.github/workflows/ci.yml

@@ -18,10 +18,10 @@ jobs:
       with:
         go-version: '1.23'
         cache: true
-    
+
     - name: Install gotestsum
       run: go install gotest.tools/gotestsum@latest
-    
+
     - name: Install gomock (for code generation)
       run: go install go.uber.org/mock/mockgen@latest
 
@@ -41,16 +41,22 @@ jobs:
 
     - name: Generate code (protobuf and mocks)
       run: go generate ./...
-
+
+    - name: Fetch Codex library
+      run: make fetch
+
     - name: Build
-      run: go build -v ./...
+      run: make build
 
     - name: Run unit tests
-      run: gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -race -v -count=1
-
+      run: make test-ci
+
+    - name: Run integration tests
+      run: make test-integration
+
     - name: Check test coverage for communities package
       run: |
-        go test -coverprofile=coverage.out ./communities
+        make coverage
         go tool cover -func=coverage.out
 
     - name: Upload coverage reports

.gitignore

@@ -21,3 +21,5 @@ coverage*.cov
 
 # Logs
 *.log
+
+libs

.vscode/settings.json

@@ -1,6 +1,13 @@
 {
   "go.testTags": "codex_integration",
   "gopls": {
-    "buildFlags": ["-tags=codex_integration"]
+    "buildFlags": [
+      "-tags=codex_integration"
+    ]
+  },
+  "go.toolsEnvVars": {
+    "CGO_ENABLED": "1",
+    "CGO_CFLAGS": "-I${workspaceFolder}/libs",
+    "CGO_LDFLAGS": "-L${workspaceFolder}/libs -lcodex -Wl,-rpath,${workspaceFolder}/libs"
   }
-}
+}

Makefile

@@ -0,0 +1,54 @@
+# Destination folder for the downloaded libraries
+LIBS_DIR := $(abspath ./libs)
+
+# Flags for CGO to find the headers and the shared library
+UNAME_S := $(shell uname -s)
+CGO_CFLAGS  := -I$(LIBS_DIR)
+CGO_LDFLAGS := -L$(LIBS_DIR) -lcodex -Wl,-rpath,$(LIBS_DIR)
+
+ifeq ($(OS),Windows_NT)
+  BIN_NAME := codex-go.exe
+else
+  BIN_NAME := codex-go
+endif
+
+# Configuration for fetching the right binary
+OS ?= "linux"
+ARCH ?= "amd64"
+VERSION ?= "v0.0.24"
+DOWNLOAD_URL := "https://github.com/codex-storage/codex-go-bindings/releases/download/$(VERSION)/codex-${OS}-${ARCH}.zip"		
+
+fetch: 
+	@echo "Fetching libcodex from GitHub Actions from: ${DOWNLOAD_URL}"
+	curl -fSL --create-dirs -o $(LIBS_DIR)/codex-${OS}-${ARCH}.zip ${DOWNLOAD_URL}
+	unzip -o -qq $(LIBS_DIR)/codex-${OS}-${ARCH}.zip -d $(LIBS_DIR)
+	rm -f $(LIBS_DIR)/*.zip
+
+build-upload:
+	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o bin/codex-upload ./cmd/upload
+
+build-download:
+	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o bin/codex-download ./cmd/download
+
+build: build-upload build-download
+
+test:
+	@echo "Running unit tests..."
+	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" gotestsum --packages="./communities" -f standard-verbose -- -v
+
+test-ci:
+	@echo "Running unit tests..."
+	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" gotestsum --packages="./communities" -f standard-verbose -- -race -v
+
+test-integration:
+	@echo "Running tests..."
+	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" gotestsum --packages="./communities" -f standard-verbose -- -tags=codex_integration -run Integration -timeout 60s
+
+coverage: 
+	@echo "Running unit tests with coverage..."
+	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go test -coverprofile=coverage.out ./communities
+	go tool cover -func=coverage.out
+
+clean:
+	rm -f $(BIN_NAME)
+	rm -Rf $(LIBS_DIR)/*

README.md

@@ -10,43 +10,29 @@ A lightweight Go client utility for interacting with Codex client.
 
 We will be running codex client, and then use a small testing utility to check if the low level abstraction - CodexClient - correctly uploads and downloads the content.
 
-### Running CodexClient
+### Integration with Codex library
 
-I often remove some logging noise, by slightly changing the build
-params in `build.nims` (nim-codex):
-
-```nim
-task codex, "build codex binary":
-  buildBinary "codex",
-    # params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
-    params =
-      "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_enabled_topics:restapi:TRACE,node:TRACE"
-```
-
-You see a slightly more selective `params` in the `codex` task.
-
-To run the client I use the following command:
-
-```bash
-./build/codex --data-dir=./data-1 --listen-addrs=/ip4/127.0.0.1/tcp/8081 --api-port=8001 --nat=none --disc-port=8091 --log-level=TRACE
+You need to download the library file by using: 
+bash
+make fetch
 ```
 
 ### Building codex-upload and codex-download utilities
 
 Use the following command to build the `codex-upload` and `codex-download` utilities:
 
 ```bash
-go build -o bin/codex-upload ./cmd/upload
-go build -o bin/codex-download ./cmd/download
+make build-upload
+make build-download
 ```
 ### Uploading content to Codex
 
 Now, using the `codex-upload` utility, we can upload the content to Codex as follows:
 
 ```bash
 ~/code/local/go-codex-client
-❯ ./bin/codex-upload -file test-data.bin -host localhost -port 8001
-Uploading test-data.bin (43 bytes) to Codex at localhost:8001...
+❯ ./bin/codex-upload -file test-data.bin
+Uploading test-data.bin (43 bytes) to Codex
 ✅ Upload successful!
 CID: zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V
 ```
@@ -57,8 +43,8 @@ Now, having the content uploaded to Codex - let's get it back using the `codex-d
 
 ```bash
 ~/code/local/go-codex-client
-❯ ./bin/codex-download -cid zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V -file output.bin -host localhost -port 8001
-Downloading CID zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V from Codex at localhost:8001...
+❯ ./bin/codex-download -cid zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V -file output.bin
+Downloading CID zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V from Codex...
 ✅ Download successful!
 Saved to: output.bin
 ```
@@ -84,11 +70,39 @@ next section.
 
 To run all unit tests:
 
+```bash
+❯ make test
+=== RUN   TestUpload_Success
+--- PASS: TestUpload_Success (0.00s)
+=== RUN   TestDownload_Success
+--- PASS: TestDownload_Success (0.00s)
+=== RUN   TestDownloadWithContext_Cancel
+--- PASS: TestDownloadWithContext_Cancel (0.04s)
+PASS
+ok  	go-codex-client/communities	0.044s
+```
+
+To run the integration test, use `test-integration`:
+
+```bash
+make test-integration
+```
+
+You can use your own Go test commands but you will need to export the `CGO` variables first:
+
+```bash
+export LIBS_DIR="$(realpath ./libs)"
+export CGO_CFLAGS=-I$LIBS_DIR
+export CGO_LDFLAGS="-L$LIBS_DIR -lcodex -Wl,-rpath,$LIBS_DIR"
+```
+
+Then you can use:
+
 ```bash
 ❯ go test -v ./communities -count 1
 ```
 
-To be more selective, e.g. in order to run all the tests from 
+To be more selective, e.g. in order to run all the tests from
 `CodexArchiveDownloaderSuite`, run:
 
 ```bash
@@ -113,7 +127,7 @@ For a more verbose output including logs use `-f standard-verbose`, e.g.:
 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -v -count 1
 ```
 
-To be more selective, e.g. in order to run all the tests from 
+To be more selective, e.g. in order to run all the tests from
 `CodexArchiveDownloaderSuite`, run:
 
 ```bash
@@ -134,66 +148,6 @@ gotestsum --packages="./communities" -f testname --rerun-fails -- -run CodexArch
 
 This also applies to native `go test` command.
 
-### Running integration tests
-
-When building Codex client for testing like here, I often remove some logging noise, by slightly changing the build params in `build.nims`:
-
-```nim
-task codex, "build codex binary":
-  buildBinary "codex",
-    # params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
-    params =
-      "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_enabled_topics:restapi:TRACE,node:TRACE"
-```
-
-You see a slightly more selective `params` in the `codex` task.
-
-To start Codex client, use e.g.:
-
-```bash
-./build/codex --data-dir=./data-1 --listen-addrs=/ip4/127.0.0.1/tcp/8081 --api-port=8001 --nat=none --disc-port=8091 --log-level=TRACE
-```
-
-To run the integration test, use `codex_integration` tag and narrow the scope using `-run Integration`:
-
-```bash
-CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run Integration -timeout 15s
-```
-
-This will run all integration tests, including CodexClient integration tests.
-
-To make sure that the test is actually run and not cached, use `count` option:
-
-```bash
-CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run Integration -timeout 15s -count 1
-```
-
-To be more specific and only run the tests related to, e.g. index downloader or archive
-downloader you can use:
-
-```bash
-CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run CodexIndexDownloaderIntegration -timeout 15s -count 1
-
-CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run CodexArchiveDownloaderIntegration -timeout 15s -count 1
-```
-
-and then, if you prefer to use `gotestsum`:
-
-```bash
-CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -run CodexIndexDownloaderIntegration -v -count 1
-
-CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -run CodexArchiveDownloaderIntegration -v -count 1
-```
-
-or to run all integration tests (including CodexClient integration tests):
-
-```bash
-CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -v -count 1 -run Integration
-```
-
-I prefer to be more selective when running integration tests.
-
-
 ### Regenerating artifacts
 
 Everything you need comes included in the repo. But if you decide to change things,

cmd/download/main.go

@@ -5,14 +5,15 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path"
 
 	"go-codex-client/communities" // Import the local communities package
+
+	"github.com/codex-storage/codex-go-bindings/codex"
 )
 
 func main() {
 	var (
-		host = flag.String("host", "localhost", "Codex host")
-		port = flag.String("port", "8080", "Codex port")
 		cid  = flag.String("cid", "", "CID of the file to download")
 		file = flag.String("file", "downloaded-file.bin", "File to save the downloaded data")
 	)
@@ -24,7 +25,20 @@ func main() {
 	}
 
 	// Create Codex client
-	client := communities.NewCodexClient(*host, *port)
+	client, err := communities.NewCodexClient(codex.Config{
+		LogFormat:      codex.LogFormatNoColors,
+		MetricsEnabled: false,
+		BlockRetries:   5,
+		LogLevel:       "ERROR",
+		DataDir:        path.Join(os.TempDir(), "codex-client-data"),
+	})
+	if err != nil {
+		log.Fatalf("Failed to create CodexClient: %v", err)
+	}
+
+	if err := client.Start(); err != nil {
+		log.Fatalf("Failed to start CodexClient: %v", err)
+	}
 
 	// Create output file
 	outputFile, err := os.Create(*file)
@@ -33,8 +47,6 @@ func main() {
 	}
 	defer outputFile.Close()
 
-	fmt.Printf("Downloading CID %s from Codex at %s:%s...\n", *cid, *host, *port)
-
 	// Download data - pass the io.Writer (outputFile), not the string
 	err = client.Download(*cid, outputFile)
 	if err != nil {
@@ -43,6 +55,13 @@ func main() {
 		log.Fatalf("Download failed: %v", err)
 	}
 
+	if err := client.Stop(); err != nil {
+		log.Printf("Warning: Failed to stop CodexClient: %v", err)
+	}
+	if err := client.Destroy(); err != nil {
+		log.Printf("Warning: Failed to stop CodexClient: %v", err)
+	}
+
 	fmt.Printf("✅ Download successful!\n")
 	fmt.Printf("Saved to: %s\n", *file)
 }