yasstake · yasstake · Sep 1, 2025 · May 20, 2025 · May 21, 2025 · May 21, 2025
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -0,0 +1,108 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+# Update CLAUDE.md
+
+This file should be updated periodically to reflect the current state of the project. If you make changes to the codebase, please update this file accordingly.
+
+## Project Overview
+
+rbot is a Python crypto trading bot framework written in Rust, designed for tick-based backtesting and live trading. The core architecture includes:
+
+- **Exchange modules** (`exchanges/`) - Exchange-specific implementations (Binance, Bybit, Bitbank, Bitflyer)
+- **Core library** (`modules/rbot_lib/`) - Common utilities, database, networking components
+- **Session management** (`modules/rbot_session/`) - Trading session, order management, and Python interface
+- **Market data** (`modules/rbot_market/`) - Market data handling and abstraction
+- **Server components** (`modules/rbot_server/`) - Server functionality
+
+The framework supports three execution modes: backtest, dry_run, and production, allowing bots to run without modification across different environments.
+
+## Recent Testing Activities
+
+### Bitbank Market Data Download (2025-01-27)
+Successfully tested Bitbank BTC/JPY market data download functionality:
+- **Script**: `test/bitbank_test/download.py`
+- **Data Range**: 2025-05-12 to 2025-06-17 (36 days total)
+- **Archive Data**: 35 days (2025-05-12 to 2025-06-16)
+- **Database Data**: 1 day (2025-06-16 to 2025-06-17)
+- **Status**: ✅ Working correctly with proper data synchronization
+- **Key Features Tested**:
+  - Historical data download from Bitbank public API
+  - Data archiving and database storage
+  - Real-time data integration
+  - Market object HTML representation
+
+### Environment Setup Notes
+- Virtual environment activation: `source .venv/bin/activate`
+- Current rbot version: 0.1.8
+- Database uses hybrid storage: Parquet for historical data, SQLite for recent data
+
+## Build and Development Commands
+
+### Rust Development
+```bash
+`.venv/bin/python3 -m pip install .`
+
+# Run tests
+cargo test
+
+# Build Python extension
+maturin develop
+
+# Build for release
+maturin build --release
+```
+
+### Python Testing
+```bash
+# Run Python tests (uses pytest)
+pytest test/
+
+# Run specific test suite
+pytest test/00_suite/
+
+# Run with verbose output
+pytest -v test/
+```
+
+### Environment Setup
+- Exchange API keys are configured in `~/.rusty-bot/.env` file
+- Database location can be overridden with `RBOT_DB_ROOT` environment variable
+- API key is stored in `{EXCHANGE_NAME}_API_KEY`
+- API secret is stored in `{EXCHANGE_NAME}_API_SECRET`
+
+## Architecture Notes
+
+### Module Dependencies
+- `rbot_lib` contains foundational components (database, networking, common utilities)
+- `rbot_session` handles trading logic and Python bindings using PyO3
+- Exchange modules (`exchanges/`) implement exchange-specific APIs and WebSocket handling
+- The project uses a workspace structure with shared dependencies
+
+### Key Components
+- **Session**: Provides abstraction layer for market interaction, ordering, and data access
+- **Market**: API and database wrapper for specific trading pairs per exchange
+- **Runner**: Orchestrates session execution across different modes (backtest/dry_run/production)
+- **Logger**: Stores execution results in Polars DataFrames for analysis
+
+### Data Storage
+- Old Historical data stored in Parquet format (changed from SQLite in v0.4.0)
+- Recent(with in a day) historical data is stored in SQLite
+- Uses Polars for efficient data manipulation and analysis
+
+### Each market structure
+Each market directory contains the files below;
+- `market.rs` main file to provide `{ExchangeName} Market` object
+- `message.rs` defines structs for REST and websocket data structure(JSON)
+- `rest.rs` implements rest request for exchange according to each exchanges documentation.
+- `ws.rs` implements realtime stream(public and private). Usually, its implement with WebSocket.
+
+
+### Testing Structure
+- Test files are organized in `test/` directory
+- Exchange-specific tests in subdirectories (`binance_test/`, `bybit_test/`, etc.)
+- Integration tests in `test/00_suite/`
+- Manual testing notebooks in `test/manual/`
+
+
diff --git a/Cargo.toml b/Cargo.toml
@@ -13,6 +13,7 @@ crate-type = ["cdylib", "rlib"]
 bybit = {path = "exchanges/bybit"}
 binance = {path = "exchanges/binance"}
 bitbank = {path= "exchanges/bitbank"}
+# hyperliquid = {path = "exchanges/hyperliquid"}
 
 rbot_lib = {path="modules/rbot_lib"}
 rbot_session = {path="modules/rbot_session"}
@@ -35,6 +36,7 @@ members = [
     "exchanges/bitbank",
     "exchanges/bitflyer",
     "exchanges/bybit",
+    # "exchanges/hyperliquid",
 ]
 
 resolver = "2"
@@ -54,6 +56,7 @@ bitbank = { path = "./exchanges/bitbank" }
 bitflyer = { path = "./exchanges/bitflyer" }
 bybit = { path = "./exchanges/bybit" }
 binance = { path = "./exchanges/binance" }
+# hyperliquid = { path = "./exchanges/hyperliquid" }
 
 anyhow = { version = "1.0.79" }
 
@@ -65,6 +68,8 @@ serde_json = { version = "1.0" }
 serde_derive = { version = "1.0" }
 serde_with = { version = "3.6.1" }
 
+json5 = { version = "0.4.1" }
+
 log = { version = "0.4" }
 env_logger = { version = "0.11.0" }
 tracing = { version = "0.1.40" }
@@ -111,12 +116,10 @@ pyo3-polars = {version = "0.20.0"}
 polars-arrow = {version = "0.46.0"}
 arrow = {version = "55.1.0"}
 
-
 parquet = "55.1.0"
 
 numpy = { version = "0.23.0" }
 
-
 ndarray = { version = "0.16.1" }
 
 reqwest = { version = "0.12.2", features = ["blocking", "gzip", "stream"] }
@@ -151,6 +154,8 @@ once_cell = { version = "1.19.0" }
 actix-web={version="4.5"}
 actix-rt={version="2.9"}
 
+pubnub = { version = "0.6.0" }
+
 pyo3 = { version = "0.23.5", features = ["rust_decimal", "abi3-py38", "anyhow", "extension-module"] }
 
 
diff --git a/doc/environment.md b/doc/environment.md
@@ -0,0 +1,24 @@
+## 取引所アクセスキーの設定方法
+
+### 設定ファイル名
+ホームディレクトリの`.env/rusty-bot/`ディレクトリに`{取引所名}.env`という名前のファイルを作成し、アクセスキーを設定します。テストネットを使う場合には`{取引所名}_TEST.env`というファイル名前にします。
+
+例
+```
+.env/rusty-bot/
+└── BITBANK.env
+    BINANCE.env
+    BINANCE_TEST.env
+```
+
+### 設定ファイル中身
+
+APIキーとAPIシークレットを以下のように設定してください
+
+
+```{取引所名.env}
+API_KEY=取引所から提供されるAPIキー
+API_SECRET=取引所から提供されるAPIシークレット
+```
+
+
diff --git a/exchanges/binance/Cargo.toml b/exchanges/binance/Cargo.toml
@@ -32,7 +32,5 @@ tokio = {workspace = true}
 futures = {workspace=true}
 async-stream = {workspace = true}
 
-
-pyo3 = { workspace = true }
-
+pyo3 = {workspace = true}
 
diff --git a/exchanges/binance/src/config.rs b/exchanges/binance/src/config.rs
@@ -2,9 +2,9 @@
 // Copyright(c) 2022-2024. yasstake. All rights reserved.
 
 use pyo3::{pyclass, pymethods};
-use rust_decimal_macros::dec;
 
-use rbot_lib::common::{env_api_key, env_api_secret, FeeType, MarketConfig, SecretString, ExchangeConfig};
+use rbot_lib::common::MarketConfig;
+use rbot_lib::common::ExchangeConfig;
 
 use crate::BINANCE;
 

diff --git a/exchanges/binance/src/rest.rs b/exchanges/binance/src/rest.rs
@@ -357,75 +357,6 @@ impl RestApi for BinanceRestApi {
         Ok(df)
 
     }
-
-    /*
-    fn format_historical_data_url(
-    }
-
-    fn line_to_trade(_line: &str) -> Trade {
-        println!("NOT IMPLEMENTED line_to_trade");
-        Trade::default()
-    }
-
-    fn convert_archive_line(_line: &str) -> String {
-        println!("NOT IMPLEMENTED convert_archive_line");
-        "".to_string()
-    }
-
-    /// binance csv Formatter
-    /// example;
-    ///  630277243,16681.46000000,0.00298000,49.71075080,1668816000029,True,True
-    ///  630277244,16681.43000000,0.00299000,49.87747570,1668816000053,True,True
-    ///  630277245,16681.00000000,0.00299000,49.87619000,1668816000075,True,True
-    fn rec_to_trade(rec: &StringRecord) -> Trade {
-        let id = rec.get(0).unwrap_or_default().to_string();
-        let price = rec
-            .get(1)
-            .unwrap_or_default()
-            .parse::<f64>()
-            .unwrap_or_default();
-
-        let price = Decimal::from_f64(price).unwrap_or_default();
-
-        let size = rec
-            .get(2)
-            .unwrap_or_default()
-            .parse::<f64>()
-            .unwrap_or_default();
-
-        let size = Decimal::from_f64(size).unwrap_or_default();
-
-        let timestamp = rec
-            .get(4)
-            .unwrap_or_default()
-            .parse::<MicroSec>()
-            .unwrap_or_default()
-            * 1_000;
-
-        let is_buyer_make = rec.get(5).unwrap_or_default();
-        let order_side = match is_buyer_make {
-            "True" => OrderSide::Buy,
-            "False" => OrderSide::Sell,
-            _ => OrderSide::Unknown,
-        };
-
-        let trade = Trade::new(
-            timestamp,
-            order_side,
-            price,
-            size,
-            LogStatus::FixArchiveBlock,
-            &id,
-        );
-
-        return trade;
-    }
-
-    /// binance csv dose not have header.
-    fn archive_has_header() -> bool {
-        false
-    }
-    */
 }
 
 impl BinanceRestApi {

diff --git a/exchanges/binance/src/ws.rs b/exchanges/binance/src/ws.rs
@@ -18,8 +18,7 @@ use tokio::time::sleep;
 use crate::BinanceRestApi;
 use crate::BinanceUserWsMessage;
 use crate::BinanceWsRawMessage;
-use crate::{BinancePublicWsMessage, BinanceServerConfig};
-
+use crate::BinancePublicWsMessage;
 use serde_derive::{Deserialize, Serialize};
 
 use anyhow::anyhow;
@@ -88,6 +87,7 @@ impl WebSocketClient for BinancePublicWsClient {
             SYNC_WAIT_RECORDS_FOR_PUBLIC,
             None,
             None,
+            false,
         );
 
         public_ws.subscribe(&vec![
@@ -164,6 +164,7 @@ impl BinancePublicWsClient{
 
 pub struct BinancePrivateWsClient {
     ws: AutoConnectClient<BinanceWsOpMessage>,
+    #[allow(dead_code)]
     server: ExchangeConfig,
     _handler: Option<JoinHandle<()>>,
     listen_key: String,
@@ -187,6 +188,7 @@ impl BinancePrivateWsClient {
             SYNC_WAIT_RECORDS_FOR_PRIVATE,
             None,
             None,
+            false,
         );
 
         Self {
@@ -280,6 +282,7 @@ impl BinancePrivateWsClient {
 mod tests {
     use super::*;
     use crate::BinanceConfig;
+    use crate::BinanceServerConfig;
     use rbot_lib::common::init_debug_log;
 
     #[tokio::test]
@@ -315,6 +318,7 @@ mod tests {
     }
 
     #[tokio::test]
+
     async fn test_make_connect_url() {
         let server = BinanceServerConfig::new(false);
         let api = BinanceRestApi::new(&server);

diff --git a/exchanges/bitbank/Cargo.toml b/exchanges/bitbank/Cargo.toml
@@ -25,6 +25,7 @@ rust_decimal_macros = {workspace = true}
 serde = {workspace = true}
 serde_derive = {workspace = true}
 serde_json = {workspace = true}
+json5 = {workspace = true}
 
 crossbeam-channel = {workspace = true}
 csv = {workspace = true}
@@ -40,6 +41,8 @@ async-stream = {workspace = true}
 
 tempfile = {workspace=true}
 
+pubnub = {workspace = true}
+
 pyo3 = { workspace = true }
 
 
diff --git a/exchanges/bitbank/src/config.rs b/exchanges/bitbank/src/config.rs
@@ -0,0 +1,18 @@
+use pyo3::prelude::*;
+use rbot_lib::common::{ExchangeConfig, MarketConfig};
+
+#[pyclass]
+pub struct BitbankConfig {}
+
+#[pymethods]
+impl BitbankConfig {
+    #[new]
+    pub fn new() -> Self {
+        return BitbankConfig {};
+    }
+
+    #[classattr]
+    pub fn BTCJPY() -> MarketConfig {
+        ExchangeConfig::open_exchange_market("bitbank", "BTC/JPY").unwrap()
+    }
+} 
diff --git a/exchanges/bitbank/src/lib.rs b/exchanges/bitbank/src/lib.rs
@@ -1,7 +1,15 @@
-
-mod rest;
+mod config;
+mod market;
 mod message;
+mod rest;
+mod ws;
+mod socket;
 
-pub use rest::*;
+pub use config::*;
+pub use market::*;
 pub use message::*;
+pub use rest::*;
+pub use ws::*;
+pub use socket::*;
 
+pub const BITBANK_BOARD_DEPTH: u32 = 200;