wip

.vscode/launch.json

@@ -17,6 +17,24 @@
             "name": "Release App (Backend)",
             "program": "${workspaceFolder:AutomaInfraCore}/Backend/.build/release/App",
             "preLaunchTask": "swift: Build Release App (Backend)"
+        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "args": [],
+            "cwd": "${workspaceFolder:AutomaInfraCore}/CLI",
+            "name": "Debug AutomaCLI (CLI)",
+            "program": "${workspaceFolder:AutomaInfraCore}/CLI/.build/debug/AutomaCLI",
+            "preLaunchTask": "swift: Build Debug AutomaCLI (CLI)"
+        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "args": [],
+            "cwd": "${workspaceFolder:AutomaInfraCore}/CLI",
+            "name": "Release AutomaCLI (CLI)",
+            "program": "${workspaceFolder:AutomaInfraCore}/CLI/.build/release/AutomaCLI",
+            "preLaunchTask": "swift: Build Release AutomaCLI (CLI)"
         }
     ]
 }

Backend/DataTypes/Sources/DataTypes/RSSFeedDTO.swift

@@ -0,0 +1,26 @@
+// RSSFeedDTO.swift
+// Copyright (c) 2025 GetAutomaApp
+// All source code and related assets are the property of GetAutomaApp.
+// All rights reserved.
+
+import Vapor
+
+public struct RSSFeedDTO: Content {
+    public var id: UUID
+    public var link: URL
+    public var createdAt: Date
+    public var updatedAt: Date
+    public var deletedAt: Date?
+
+    public init(id: UUID, link: String, createdAt: Date, updatedAt: Date, deletedAt: Date?) throws {
+        guard let link = URL(string: link) else {
+            throw GenericErrors.invalidUrl
+        }
+
+        self.id = id
+        self.link = link
+        self.createdAt = createdAt
+        self.updatedAt = updatedAt
+        self.deletedAt = deletedAt
+    }
+}

Backend/DataTypes/Sources/DataTypes/RssFeedItemDTO.swift

@@ -0,0 +1,34 @@
+// RssFeedItemDTO.swift
+// Copyright (c) 2025 GetAutomaApp
+// All source code and related assets are the property of GetAutomaApp.
+// All rights reserved.
+
+import Vapor
+
+public struct RssFeedItemDTO: Content {
+    public var id: UUID?
+    public var rssFeedId: UUID
+    public var content: String
+    public var link: String
+    public var createdAt: Date?
+    public var updatedAt: Date?
+    public var deletedAt: Date?
+
+    public init(
+        id: UUID?,
+        rssFeedId: UUID,
+        content: String,
+        link: String,
+        createdAt: Date?,
+        updatedAt: Date?,
+        deletedAt: Date?
+    ) {
+        self.id = id
+        self.rssFeedId = rssFeedId
+        self.content = content
+        self.link = link
+        self.createdAt = createdAt
+        self.updatedAt = updatedAt
+        self.deletedAt = deletedAt
+    }
+}

Backend/DataTypes/Sources/DataTypes/UserFeedMappingDTO.swift

@@ -0,0 +1,24 @@
+// UserFeedMappingDTO.swift
+// Copyright (c) 2025 GetAutomaApp
+// All source code and related assets are the property of GetAutomaApp.
+// All rights reserved.
+
+import Vapor
+
+public struct UserFeedMappingDTO: Content {
+    public var id: UUID
+    public var feedId: UUID
+    public var userId: UUID
+    public var createdAt: Date
+    public var updatedAt: Date
+    public var deletedAt: Date?
+
+    public init(id: UUID, feedId: UUID, userId: UUID, createdAt: Date, updatedAt: Date, deletedAt: Date?) {
+        self.id = id
+        self.userId = userId
+        self.feedId = feedId
+        self.createdAt = createdAt
+        self.updatedAt = updatedAt
+        self.deletedAt = deletedAt
+    }
+}

Backend/Sources/App/Clients/Crawl4AIClient.swift

@@ -0,0 +1,119 @@
+// Crawl4AIClient.swift
+// Copyright (c) 2025 GetAutomaApp
+// All source code and related assets are the property of GetAutomaApp.
+// All rights reserved.
+
+import Foundation
+
+struct CrawlRequest: Codable {
+    let urls: [String]
+    let browserConfig: BrowserConfig?
+    let crawlerConfig: CrawlerConfig?
+
+    enum CodingKeys: String, CodingKey {
+        case urls
+        case browserConfig = "browser_config"
+        case crawlerConfig = "crawler_config"
+    }
+}
+
+struct BrowserConfig: Codable {
+    let headless: Bool
+
+    init(headless: Bool = true) {
+        self.headless = headless
+    }
+}
+
+struct CrawlerConfig: Codable {
+    let stream: Bool
+
+    init(stream: Bool = false) {
+        self.stream = stream
+    }
+}
+
+struct CrawlTaskId: Codable {
+    let taskId: UUID
+
+    enum CodingKeys: String, CodingKey {
+        case taskId = "task_id"
+    }
+}
+
+struct CrawlResult: Codable {
+    let results: [Result]
+
+    struct Result: Codable {
+        let url: String
+        let extractedContent: String?
+        let markdown: String?
+
+        enum CodingKeys: String, CodingKey {
+            case url
+            case extractedContent = "extracted_content"
+            case markdown
+        }
+    }
+}
+
+class Crawl4AIClient {
+    private let baseURL: URL
+    private let apiKey: String
+    private let session: URLSession
+
+    init(baseURL: String = "http://localhost:11235", apiKey: String) {
+        self.baseURL = URL(string: baseURL)!
+        self.apiKey = apiKey
+        session = URLSession.shared
+    }
+
+    func startCrawl(urls: [String], browserConfig: BrowserConfig? = nil,
+                    crawlerConfig: CrawlerConfig? = nil) async throws -> CrawlTaskId
+    {
+        let request = CrawlRequest(urls: urls, browserConfig: browserConfig, crawlerConfig: crawlerConfig)
+
+        var urlRequest = URLRequest(url: baseURL.appendingPathComponent("crawl"))
+        urlRequest.httpMethod = "POST"
+        urlRequest.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
+        urlRequest.setValue("application/json", forHTTPHeaderField: "Content-Type")
+
+        let encoder = JSONEncoder()
+        urlRequest.httpBody = try encoder.encode(request)
+
+        let (data, response) = try await session.data(for: urlRequest)
+
+        guard let httpResponse = response as? HTTPURLResponse else {
+            throw URLError(.badServerResponse)
+        }
+
+        guard httpResponse.statusCode == 200 else {
+            throw URLError(.badServerResponse)
+        }
+
+        let decoder = JSONDecoder()
+        let decodedResponse = try decoder.decode(CrawlTaskId.self, from: data)
+        return decodedResponse
+    }
+
+    func getCrawl
+}
+
+// Example usage:
+/*
+ let client = Crawl4AIClient(apiKey: "YOUR_API_KEY")
+
+ Task {
+ do {
+ let result = try await client.crawl(urls: ["https://example.com"])
+ print("Crawl success: \(result.success)")
+ for item in result.results {
+ print("URL: \(item.url)")
+ print("Content: \(item.extractedContent ?? "")")
+ print("Markdown: \(item.markdown ?? "")")
+ }
+ } catch {
+ print("Error: \(error)")
+ }
+ }
+ */

Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClient.swift

@@ -38,13 +38,15 @@ struct FirecrawlClient {
             )
 
             guard let decodedData = try? response.content.decode(FirecrawlScrapeResult.self) else {
+                let bodyData = response.body?.getData(at: 0, length: response.body?.readableBytes ?? 0)
+                let bodyString = bodyData
+                    .flatMap { String(data: $0, encoding: .utf8) } ?? "Unable to convert body to string"
+
                 logger.error(
                     "Invalid response from firecrawl microservice",
                     metadata: [
                         "url": .string(input.url),
-                        "description": .string(
-                            response.body?.debugDescription ?? response.description
-                        ),
+                        "response": .string(bodyString),
                         "to": .string("FirecrawlClient.scrapeMarkdown"),
                     ]
                 )

Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClientTypes.swift

@@ -11,6 +11,7 @@ enum FirecrawlFormats: String, Content {
 
 struct ScrapeMarkdownInput: Content {
     let url: String
+    let timeout: Int = 60000
 }
 
 struct FirecrawlDataResult: Content {

Backend/Sources/App/Clients/RSSFeedClient/RSSFeedReaderClient.swift

@@ -11,16 +11,14 @@ import Retry
 import Vapor
 
 struct RSSFeedReaderClient {
-    func read(from url: URL) async -> RssFeedResponse {
-        var feed: Feed?
+    func read(from url: URL) async throws -> RssFeedResponse {
+        var feed: Feed? = nil
 
-        do {
-            try await retry(
-                maxAttempts: 3
-            ) {
-                feed = try! await Feed(url: url)
-            }
-        } catch {}
+        try await retry(
+            maxAttempts: 3
+        ) {
+            feed = try! await Feed(url: url)
+        }
 
         switch feed {
         case let .rss(rSSFeed):
@@ -31,7 +29,7 @@ struct RSSFeedReaderClient {
         }
     }
 
-    func convertRSSToGenericFeedItems(from feedItems: [RSSFeedItem]) -> [GenericRSSFeedItem] {
+    private func convertRSSToGenericFeedItems(from feedItems: [RSSFeedItem]) -> [GenericRSSFeedItem] {
         let items = feedItems.compactMap { feedItem -> GenericRSSFeedItem? in
             guard
                 let title = feedItem.title,

Backend/Sources/App/Controllers/FeedTesterController/FeedTesterController.swift

@@ -12,12 +12,26 @@ struct FeedTesterController: RouteCollection {
         let feedTesterRoute = routes.grouped("Feed-Tester")
 
         feedTesterRoute.get("request", use: request)
+        feedTesterRoute.get("add-to-user", use: addToUser)
     }
 
     @Sendable
     func request(req _: Request) async throws -> RssFeedResponse {
         let feedService: RSSFeedReaderClient = .init()
-        let response = await feedService.read(from: URL(string: "https://news.ycombinator.com/rss")!)
+        let response = try! await feedService.read(from: URL(string: "https://news.ycombinator.com/rss")!)
         return response
     }
+
+    @Sendable
+    func addToUser(req: Request) async throws -> HTTPStatus {
+        let feedService = RSSFeedService(database: req.dbWrite, client: req.client, logger: req.logger)
+        if let url = URL(string: "https://news.ycombinator.com/rss") {
+            let success = try await feedService.addFeedToUser(
+                userId: UUID(uuidString: "562771bf-98a5-4cc4-83ba-5f618dc79546")!,
+                feedUrl: url
+            )
+            return success ? .ok : .internalServerError
+        }
+        return .badRequest
+    }
 }

Backend/Sources/App/Migrations/RSSFeedMigration1741359416.swift

@@ -0,0 +1,22 @@
+// RSSFeedMigration1741359416.swift
+// Copyright (c) 2025 GetAutomaApp
+// All source code and related assets are the property of GetAutomaApp.
+// All rights reserved.
+
+import Fluent
+
+struct RSSFeedMigration1741359416: AsyncMigration {
+    func prepare(on database: Database) async throws {
+        try await database.schema("RSS-Feed")
+            .id()
+            .field("link", .string)
+            .field("updated_at", .datetime)
+            .field("created_at", .datetime)
+            .field("deleted_at", .datetime)
+            .create()
+    }
+
+    func revert(on database: Database) async throws {
+        try await database.schema("RSS-Feed").delete()
+    }
+}

Backend/Sources/App/Migrations/RssFeedItemMigration1741876963.swift

@@ -0,0 +1,29 @@
+// RssFeedItemMigration1741876963.swift
+// Copyright (c) 2025 GetAutomaApp
+// All source code and related assets are the property of GetAutomaApp.
+// All rights reserved.
+
+import Fluent
+
+struct RssFeedItemMigration1741876963: AsyncMigration {
+    func prepare(on database: Database) async throws {
+        try await database.schema("RSS-Feed-Item")
+            .id()
+            .field(
+                "rss_feed_id",
+                .uuid,
+                .required,
+                .references("RSS-Feed", "id")
+            )
+            .field("content", .string, .required)
+            .field("link", .string, .required)
+            .field("updated_at", .datetime)
+            .field("created_at", .datetime)
+            .field("deleted_at", .datetime)
+            .create()
+    }
+
+    func revert(on database: Database) async throws {
+        try await database.schema("Rss-Feed-Item").delete()
+    }
+}