diff --git a/.vscode/launch.json b/.vscode/launch.json index 8bffbadd..0fe4b936 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -17,6 +17,24 @@ "name": "Release App (Backend)", "program": "${workspaceFolder:AutomaInfraCore}/Backend/.build/release/App", "preLaunchTask": "swift: Build Release App (Backend)" + }, + { + "type": "lldb", + "request": "launch", + "args": [], + "cwd": "${workspaceFolder:AutomaInfraCore}/CLI", + "name": "Debug AutomaCLI (CLI)", + "program": "${workspaceFolder:AutomaInfraCore}/CLI/.build/debug/AutomaCLI", + "preLaunchTask": "swift: Build Debug AutomaCLI (CLI)" + }, + { + "type": "lldb", + "request": "launch", + "args": [], + "cwd": "${workspaceFolder:AutomaInfraCore}/CLI", + "name": "Release AutomaCLI (CLI)", + "program": "${workspaceFolder:AutomaInfraCore}/CLI/.build/release/AutomaCLI", + "preLaunchTask": "swift: Build Release AutomaCLI (CLI)" } ] } \ No newline at end of file diff --git a/Backend/DataTypes/Sources/DataTypes/RSSFeedDTO.swift b/Backend/DataTypes/Sources/DataTypes/RSSFeedDTO.swift new file mode 100644 index 00000000..f8775b54 --- /dev/null +++ b/Backend/DataTypes/Sources/DataTypes/RSSFeedDTO.swift @@ -0,0 +1,26 @@ +// RSSFeedDTO.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Vapor + +public struct RSSFeedDTO: Content { + public var id: UUID + public var link: URL + public var createdAt: Date + public var updatedAt: Date + public var deletedAt: Date? + + public init(id: UUID, link: String, createdAt: Date, updatedAt: Date, deletedAt: Date?) throws { + guard let link = URL(string: link) else { + throw GenericErrors.invalidUrl + } + + self.id = id + self.link = link + self.createdAt = createdAt + self.updatedAt = updatedAt + self.deletedAt = deletedAt + } +} diff --git a/Backend/DataTypes/Sources/DataTypes/RssFeedItemDTO.swift b/Backend/DataTypes/Sources/DataTypes/RssFeedItemDTO.swift new file mode 100644 index 00000000..b22e08b3 --- /dev/null +++ b/Backend/DataTypes/Sources/DataTypes/RssFeedItemDTO.swift @@ -0,0 +1,34 @@ +// RssFeedItemDTO.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Vapor + +public struct RssFeedItemDTO: Content { + public var id: UUID? + public var rssFeedId: UUID + public var content: String + public var link: String + public var createdAt: Date? + public var updatedAt: Date? + public var deletedAt: Date? + + public init( + id: UUID?, + rssFeedId: UUID, + content: String, + link: String, + createdAt: Date?, + updatedAt: Date?, + deletedAt: Date? + ) { + self.id = id + self.rssFeedId = rssFeedId + self.content = content + self.link = link + self.createdAt = createdAt + self.updatedAt = updatedAt + self.deletedAt = deletedAt + } +} diff --git a/Backend/DataTypes/Sources/DataTypes/UserFeedMappingDTO.swift b/Backend/DataTypes/Sources/DataTypes/UserFeedMappingDTO.swift new file mode 100644 index 00000000..4436790d --- /dev/null +++ b/Backend/DataTypes/Sources/DataTypes/UserFeedMappingDTO.swift @@ -0,0 +1,24 @@ +// UserFeedMappingDTO.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Vapor + +public struct UserFeedMappingDTO: Content { + public var id: UUID + public var feedId: UUID + public var userId: UUID + public var createdAt: Date + public var updatedAt: Date + public var deletedAt: Date? + + public init(id: UUID, feedId: UUID, userId: UUID, createdAt: Date, updatedAt: Date, deletedAt: Date?) { + self.id = id + self.userId = userId + self.feedId = feedId + self.createdAt = createdAt + self.updatedAt = updatedAt + self.deletedAt = deletedAt + } +} diff --git a/Backend/Package.resolved b/Backend/Package.resolved index 6915cfba..395c7452 100644 --- a/Backend/Package.resolved +++ b/Backend/Package.resolved @@ -10,6 +10,15 @@ "version" : "5.10.2" } }, + { + "identity" : "anycodable", + "kind" : "remoteSourceControl", + "location" : "https://github.com/Flight-School/AnyCodable", + "state" : { + "revision" : "862808b2070cd908cb04f9aafe7de83d35f81b05", + "version" : "0.6.7" + } + }, { "identity" : "async-http-client", "kind" : "remoteSourceControl", diff --git a/Backend/Public/favicon.ico b/Backend/Public/favicon.ico new file mode 100644 index 00000000..49cbb6bd Binary files /dev/null and b/Backend/Public/favicon.ico differ diff --git a/Backend/Sources/App/Clients/Crawl4AIClient.swift b/Backend/Sources/App/Clients/Crawl4AIClient.swift new file mode 100644 index 00000000..3eeeb214 --- /dev/null +++ b/Backend/Sources/App/Clients/Crawl4AIClient.swift @@ -0,0 +1,119 @@ +// Crawl4AIClient.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Foundation + +struct CrawlRequest: Codable { + let urls: [String] + let browserConfig: BrowserConfig? + let crawlerConfig: CrawlerConfig? + + enum CodingKeys: String, CodingKey { + case urls + case browserConfig = "browser_config" + case crawlerConfig = "crawler_config" + } +} + +struct BrowserConfig: Codable { + let headless: Bool + + init(headless: Bool = true) { + self.headless = headless + } +} + +struct CrawlerConfig: Codable { + let stream: Bool + + init(stream: Bool = false) { + self.stream = stream + } +} + +struct CrawlTaskId: Codable { + let taskId: UUID + + enum CodingKeys: String, CodingKey { + case taskId = "task_id" + } +} + +struct CrawlResult: Codable { + let results: [Result] + + struct Result: Codable { + let url: String + let extractedContent: String? + let markdown: String? + + enum CodingKeys: String, CodingKey { + case url + case extractedContent = "extracted_content" + case markdown + } + } +} + +class Crawl4AIClient { + private let baseURL: URL + private let apiKey: String + private let session: URLSession + + init(baseURL: String = "http://localhost:11235", apiKey: String) { + self.baseURL = URL(string: baseURL)! + self.apiKey = apiKey + session = URLSession.shared + } + + func startCrawl(urls: [String], browserConfig: BrowserConfig? = nil, + crawlerConfig: CrawlerConfig? = nil) async throws -> CrawlTaskId + { + let request = CrawlRequest(urls: urls, browserConfig: browserConfig, crawlerConfig: crawlerConfig) + + var urlRequest = URLRequest(url: baseURL.appendingPathComponent("crawl")) + urlRequest.httpMethod = "POST" + urlRequest.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization") + urlRequest.setValue("application/json", forHTTPHeaderField: "Content-Type") + + let encoder = JSONEncoder() + urlRequest.httpBody = try encoder.encode(request) + + let (data, response) = try await session.data(for: urlRequest) + + guard let httpResponse = response as? HTTPURLResponse else { + throw URLError(.badServerResponse) + } + + guard httpResponse.statusCode == 200 else { + throw URLError(.badServerResponse) + } + + let decoder = JSONDecoder() + let decodedResponse = try decoder.decode(CrawlTaskId.self, from: data) + return decodedResponse + } + + func getCrawl +} + +// Example usage: +/* + let client = Crawl4AIClient(apiKey: "YOUR_API_KEY") + + Task { + do { + let result = try await client.crawl(urls: ["https://example.com"]) + print("Crawl success: \(result.success)") + for item in result.results { + print("URL: \(item.url)") + print("Content: \(item.extractedContent ?? "")") + print("Markdown: \(item.markdown ?? "")") + } + } catch { + print("Error: \(error)") + } + } + */ diff --git a/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClient.swift b/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClient.swift index b157b05d..1ebdc168 100644 --- a/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClient.swift +++ b/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClient.swift @@ -38,13 +38,15 @@ struct FirecrawlClient { ) guard let decodedData = try? response.content.decode(FirecrawlScrapeResult.self) else { + let bodyData = response.body?.getData(at: 0, length: response.body?.readableBytes ?? 0) + let bodyString = bodyData + .flatMap { String(data: $0, encoding: .utf8) } ?? "Unable to convert body to string" + logger.error( "Invalid response from firecrawl microservice", metadata: [ "url": .string(input.url), - "description": .string( - response.body?.debugDescription ?? response.description - ), + "response": .string(bodyString), "to": .string("FirecrawlClient.scrapeMarkdown"), ] ) diff --git a/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClientTypes.swift b/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClientTypes.swift index 315f77a9..9c174d7b 100644 --- a/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClientTypes.swift +++ b/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClientTypes.swift @@ -11,6 +11,7 @@ enum FirecrawlFormats: String, Content { struct ScrapeMarkdownInput: Content { let url: String + let timeout: Int = 60000 } struct FirecrawlDataResult: Content { diff --git a/Backend/Sources/App/Clients/RSSFeedClient/RSSFeedReaderClient.swift b/Backend/Sources/App/Clients/RSSFeedClient/RSSFeedReaderClient.swift index 049ba7c3..fd4385fd 100644 --- a/Backend/Sources/App/Clients/RSSFeedClient/RSSFeedReaderClient.swift +++ b/Backend/Sources/App/Clients/RSSFeedClient/RSSFeedReaderClient.swift @@ -11,16 +11,14 @@ import Retry import Vapor struct RSSFeedReaderClient { - func read(from url: URL) async -> RssFeedResponse { - var feed: Feed? + func read(from url: URL) async throws -> RssFeedResponse { + var feed: Feed? = nil - do { - try await retry( - maxAttempts: 3 - ) { - feed = try! await Feed(url: url) - } - } catch {} + try await retry( + maxAttempts: 3 + ) { + feed = try! await Feed(url: url) + } switch feed { case let .rss(rSSFeed): @@ -31,7 +29,7 @@ struct RSSFeedReaderClient { } } - func convertRSSToGenericFeedItems(from feedItems: [RSSFeedItem]) -> [GenericRSSFeedItem] { + private func convertRSSToGenericFeedItems(from feedItems: [RSSFeedItem]) -> [GenericRSSFeedItem] { let items = feedItems.compactMap { feedItem -> GenericRSSFeedItem? in guard let title = feedItem.title, diff --git a/Backend/Sources/App/Controllers/FeedTesterController/FeedTesterController.swift b/Backend/Sources/App/Controllers/FeedTesterController/FeedTesterController.swift index 4a67b410..da6b9c5f 100644 --- a/Backend/Sources/App/Controllers/FeedTesterController/FeedTesterController.swift +++ b/Backend/Sources/App/Controllers/FeedTesterController/FeedTesterController.swift @@ -12,12 +12,26 @@ struct FeedTesterController: RouteCollection { let feedTesterRoute = routes.grouped("Feed-Tester") feedTesterRoute.get("request", use: request) + feedTesterRoute.get("add-to-user", use: addToUser) } @Sendable func request(req _: Request) async throws -> RssFeedResponse { let feedService: RSSFeedReaderClient = .init() - let response = await feedService.read(from: URL(string: "https://news.ycombinator.com/rss")!) + let response = try! await feedService.read(from: URL(string: "https://news.ycombinator.com/rss")!) return response } + + @Sendable + func addToUser(req: Request) async throws -> HTTPStatus { + let feedService = RSSFeedService(database: req.dbWrite, client: req.client, logger: req.logger) + if let url = URL(string: "https://news.ycombinator.com/rss") { + let success = try await feedService.addFeedToUser( + userId: UUID(uuidString: "562771bf-98a5-4cc4-83ba-5f618dc79546")!, + feedUrl: url + ) + return success ? .ok : .internalServerError + } + return .badRequest + } } diff --git a/Backend/Sources/App/Migrations/RSSFeedMigration1741359416.swift b/Backend/Sources/App/Migrations/RSSFeedMigration1741359416.swift new file mode 100644 index 00000000..74cd8d60 --- /dev/null +++ b/Backend/Sources/App/Migrations/RSSFeedMigration1741359416.swift @@ -0,0 +1,22 @@ +// RSSFeedMigration1741359416.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Fluent + +struct RSSFeedMigration1741359416: AsyncMigration { + func prepare(on database: Database) async throws { + try await database.schema("RSS-Feed") + .id() + .field("link", .string) + .field("updated_at", .datetime) + .field("created_at", .datetime) + .field("deleted_at", .datetime) + .create() + } + + func revert(on database: Database) async throws { + try await database.schema("RSS-Feed").delete() + } +} diff --git a/Backend/Sources/App/Migrations/RssFeedItemMigration1741876963.swift b/Backend/Sources/App/Migrations/RssFeedItemMigration1741876963.swift new file mode 100644 index 00000000..c84c9d25 --- /dev/null +++ b/Backend/Sources/App/Migrations/RssFeedItemMigration1741876963.swift @@ -0,0 +1,29 @@ +// RssFeedItemMigration1741876963.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Fluent + +struct RssFeedItemMigration1741876963: AsyncMigration { + func prepare(on database: Database) async throws { + try await database.schema("RSS-Feed-Item") + .id() + .field( + "rss_feed_id", + .uuid, + .required, + .references("RSS-Feed", "id") + ) + .field("content", .string, .required) + .field("link", .string, .required) + .field("updated_at", .datetime) + .field("created_at", .datetime) + .field("deleted_at", .datetime) + .create() + } + + func revert(on database: Database) async throws { + try await database.schema("Rss-Feed-Item").delete() + } +} diff --git a/Backend/Sources/App/Migrations/UserFeedMappingMigration1741429746.swift b/Backend/Sources/App/Migrations/UserFeedMappingMigration1741429746.swift new file mode 100644 index 00000000..b0abfff6 --- /dev/null +++ b/Backend/Sources/App/Migrations/UserFeedMappingMigration1741429746.swift @@ -0,0 +1,23 @@ +// UserFeedMappingMigration1741429746.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Fluent + +struct UserFeedMappingMigration1741429746: AsyncMigration { + func prepare(on database: Database) async throws { + try await database.schema("User-Feed-Mapping") + .id() + .field("feed_id", .uuid, .required, .references("RSS-Feed", "id")) + .field("user_id", .uuid, .required, .references("User", "id")) + .field("updated_at", .datetime) + .field("created_at", .datetime) + .field("deleted_at", .datetime) + .create() + } + + func revert(on database: Database) async throws { + try await database.schema("User-Feed-Mapping").delete() + } +} diff --git a/Backend/Sources/App/Models/RSSFeedItemModel.swift b/Backend/Sources/App/Models/RSSFeedItemModel.swift new file mode 100644 index 00000000..b17f197d --- /dev/null +++ b/Backend/Sources/App/Models/RSSFeedItemModel.swift @@ -0,0 +1,73 @@ +// RSSFeedItemModel.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import DataTypes +import Fluent +import Vapor + +final class RSSFeedItemModel: Model, @unchecked Sendable { + static let schema = "RSS-Feed-Item" + + @ID(key: .id) + var id: UUID? + + @Field(key: "rss_feed_id") + var rssFeedId: UUID + + @Field(key: "content") + var content: String + + @Field(key: "link") + var link: String + + @Timestamp(key: "created_at", on: .create) + var createdAt: Date? + + @Timestamp(key: "updated_at", on: .update) + var updatedAt: Date? + + @Timestamp(key: "deleted_at", on: .delete) + var deletedAt: Date? + + init() {} + + init( + id: UUID? = nil, + rssFeedId: UUID, + content: String, + link: String, + createdAt: Date? = nil, + updatedAt: Date? = nil, + deletedAt: Date? = nil + ) { + self.id = id + self.rssFeedId = rssFeedId + self.content = content + self.link = link + self.createdAt = createdAt + self.updatedAt = updatedAt + self.deletedAt = deletedAt + } + + func toDTO() -> RssFeedItemDTO { + .init( + id: id, + rssFeedId: rssFeedId, + content: content, + link: link, + createdAt: createdAt, + updatedAt: updatedAt, + deletedAt: deletedAt + ) + } + + static func fromDTO(dto: RssFeedItemDTO) -> RSSFeedItemModel { + let model = RSSFeedItemModel( + id: dto.id, rssFeedId: dto.rssFeedId, content: dto.content, link: dto.link, createdAt: dto.createdAt, + updatedAt: dto.updatedAt, deletedAt: dto.deletedAt + ) + return model + } +} diff --git a/Backend/Sources/App/Models/RSSFeedModel.swift b/Backend/Sources/App/Models/RSSFeedModel.swift new file mode 100644 index 00000000..44efefd1 --- /dev/null +++ b/Backend/Sources/App/Models/RSSFeedModel.swift @@ -0,0 +1,61 @@ +// RSSFeedModel.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import DataTypes +import Fluent +import Vapor + +final class RSSFeedModel: Model, @unchecked Sendable { + static let schema = "RSS-Feed" + + @ID(key: .id) + var id: UUID? + + @Field(key: "link") + var link: String + + @Timestamp(key: "created_at", on: .create) + var createdAt: Date? + + @Timestamp(key: "updated_at", on: .update) + var updatedAt: Date? + + @Timestamp(key: "deleted_at", on: .delete) + var deletedAt: Date? + + init() {} + + init( + id: UUID? = nil, + link: String, + createdAt: Date? = nil, + updatedAt: Date? = nil, + deletedAt: Date? = nil + ) { + self.id = id + self.link = link + self.createdAt = createdAt + self.updatedAt = updatedAt + self.deletedAt = deletedAt + } + + func toDTO() throws -> RSSFeedDTO { + try .init( + id: id!, + link: link, + createdAt: createdAt!, + updatedAt: updatedAt!, + deletedAt: deletedAt + ) + } + + static func fromDTO(dto: RSSFeedDTO) -> RSSFeedModel { + let model = RSSFeedModel( + id: dto.id, link: dto.link.absoluteString, createdAt: dto.createdAt, updatedAt: dto.updatedAt, + deletedAt: dto.deletedAt + ) + return model + } +} diff --git a/Backend/Sources/App/Models/UserFeedMappingModel.swift b/Backend/Sources/App/Models/UserFeedMappingModel.swift new file mode 100644 index 00000000..c2513851 --- /dev/null +++ b/Backend/Sources/App/Models/UserFeedMappingModel.swift @@ -0,0 +1,67 @@ +// UserFeedMappingModel.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import DataTypes +import Fluent +import Vapor + +final class UserFeedMappingModel: Model, @unchecked Sendable { + static let schema = "User-Feed-Mapping" + + @ID(key: .id) + var id: UUID? + + @Field(key: "user_id") + var userId: UUID + + @Field(key: "feed_id") + var feedId: UUID + + @Timestamp(key: "created_at", on: .create) + var createdAt: Date? + + @Timestamp(key: "updated_at", on: .update) + var updatedAt: Date? + + @Timestamp(key: "deleted_at", on: .delete) + var deletedAt: Date? + + init() {} + + init( + id: UUID? = nil, + userId: UUID, + feedId: UUID, + createdAt: Date? = nil, + updatedAt: Date? = nil, + deletedAt: Date? = nil + ) { + self.id = id + self.userId = userId + self.feedId = feedId + self.createdAt = createdAt + self.updatedAt = updatedAt + self.deletedAt = deletedAt + } + + func toDTO() -> UserFeedMappingDTO { + .init( + id: id!, + feedId: feedId, + userId: userId, + createdAt: createdAt!, + updatedAt: updatedAt!, + deletedAt: deletedAt + ) + } + + static func fromDTO(dto: UserFeedMappingDTO) -> UserFeedMappingModel { + let model = UserFeedMappingModel( + id: dto.id, userId: dto.userId, feedId: dto.feedId, createdAt: dto.createdAt, updatedAt: dto.updatedAt, + deletedAt: dto.deletedAt + ) + return model + } +} diff --git a/Backend/Sources/App/Procs/CronJobs/ScrapeRSSFeedCronJob.swift b/Backend/Sources/App/Procs/CronJobs/ScrapeRSSFeedCronJob.swift new file mode 100644 index 00000000..e4cf0f65 --- /dev/null +++ b/Backend/Sources/App/Procs/CronJobs/ScrapeRSSFeedCronJob.swift @@ -0,0 +1,19 @@ +// ScrapeRSSFeedCronJob.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Queues +import Vapor + +struct ScrapeRSSFeedCronJob: AsyncScheduledJob { + func run(context: QueueContext) async throws { + let rssFeedService = RSSFeedService( + database: context.dbWrite, + client: context.application.client, + logger: context.logger + ) + + try await rssFeedService.scrapeAndInsertLatestForAllFeeds() + } +} diff --git a/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift b/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift new file mode 100644 index 00000000..61295eef --- /dev/null +++ b/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift @@ -0,0 +1,128 @@ +// RSSFeedService.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import DataTypes +import Fluent +import Vapor + +struct RSSFeedService { + let database: Database + let rssFeedClient = RSSFeedReaderClient() + let client: Client + let logger: Logger + + func addFeedToUser(userId: UUID, feedUrl: URL) async throws -> Bool { + do { + if try await !rssFeedClient.read(from: feedUrl).isRssFeed { + print("isn't rss feed") + return false + } + + let (feed, _) = try await createFeedInDBIfNotExist(feedUrl: feedUrl) + try await addUserFeedMapping(feedId: feed.id, userId: userId) + return true + } catch { + // TODO: Log error + print("unknown error \(error.localizedDescription)") + return false + } + } + + // If the model already exists we return true + func createFeedInDBIfNotExist(feedUrl: URL) async throws -> (RSSFeedDTO, Bool) { + let feedUrlString = feedUrl.absoluteString + if let feed = try await RSSFeedModel.query(on: database).filter(\.$link == feedUrlString).first() { + // The feed exists, so we just return it as a DTO + return try (feed.toDTO(), true) + } + + // Create the new feed model + let feed = RSSFeedModel( + id: UUID(), + link: feedUrlString + ) + + // Save the newly created feed + try await feed.save(on: database) + + // Now that the feed is saved, return the DTO + return try (feed.toDTO(), false) + } + + // TODO: Discardable feed mapping id return + @discardableResult + func addUserFeedMapping(feedId: UUID, userId: UUID) async throws -> UserFeedMappingDTO { + if let existingMapping = try await UserFeedMappingModel.query(on: database).filter(\.$userId == userId) + .filter(\.$feedId == feedId).first() + { + return existingMapping.toDTO() + } + + let newMapping = UserFeedMappingModel(userId: userId, feedId: feedId) + + try await newMapping.save(on: database) + + return newMapping.toDTO() + } + + func scrapeAndInsertLatestForAllFeeds() async throws { + let feeds = try await RSSFeedModel.query(on: database).all() + + for feed in feeds { + do { + try await scrapeAndInsertLatestRSSFeedItems(feedDTO: feed.toDTO()) + } catch { + // Fix log + logger.error("Failed to process feed \(feed.link): \(error.localizedDescription)") + } + } + } + + func scrapeAndInsertLatestRSSFeedItems(feedDTO: RSSFeedDTO) async throws { + let firecrawlClient = try FirecrawlClient(client: client, logger: logger) + + let url = URL(string: feedDTO.link.absoluteString) + + guard let url else { + return + } + + let latestFeedItems = try await rssFeedClient.read(from: url).items + let latestFeedItemLinks = latestFeedItems.map(\.link) + + let existingFeedItems = try await RSSFeedItemModel.query(on: database) + .filter(\.$link ~~ latestFeedItemLinks) + .all() + + let existingFeedItemLinks = Set(existingFeedItems.map(\.link)) + + let unprocessedLinks = Set(latestFeedItemLinks).subtracting(existingFeedItemLinks) + + var feedItems: [RssFeedItemDTO] = [] + + for link in unprocessedLinks { + do { + let content = try await firecrawlClient.scrapeMarkdown( + from: .init(url: link) + ) + + let article = RSSFeedItemModel( + rssFeedId: feedDTO.id, + content: content.markdown, + link: link + ) + + try await article.save(on: database) + + feedItems.append(article.toDTO()) + } catch { + // Log the error if scraping fails + logger.error("Failed to scrape content for link \(link): \(error.localizedDescription)") + } + } + + // TODO: Log feedItems if necessary + } +} diff --git a/Backend/Sources/App/configure.swift b/Backend/Sources/App/configure.swift index ecfeb920..7df0e442 100644 --- a/Backend/Sources/App/configure.swift +++ b/Backend/Sources/App/configure.swift @@ -13,7 +13,7 @@ import Vapor // Configures your application public func configure(_ app: Application) async throws { // This is file middleware - // app.middleware.use(FileMiddleware(publicDirectory: app.directory.publicDirectory)) + app.middleware.use(FileMiddleware(publicDirectory: app.directory.publicDirectory)) app.middleware.use(ErrorStringMiddleware()) // Errors are getting thrown locally, this prevents run App & ./App execution diffs @@ -53,6 +53,9 @@ public func configure(_ app: Application) async throws { app.migrations.add(JobMetadataMigrate()) app.migrations.add(RemoveUserStorageMigration1739456565()) app.migrations.add(AddAcceptedColumnMigration1740658649()) + app.migrations.add(RSSFeedMigration1741359416()) + app.migrations.add(UserFeedMappingMigration1741429746()) + app.migrations.add(RssFeedItemMigration1741876963()) try await app.autoMigrate() @@ -75,9 +78,13 @@ public func configure(_ app: Application) async throws { // Jobs app.queues.add(TransactionalMessageAsyncJob()) app.queues.add(ProfilePictureAsyncJob()) +// app.queues.scheduleEvery(ScrapeRSSFeedCronJob(), minutes: 5) // Http Server Config app.http.server.configuration.responseCompression = .enabled + + let response = try await Crawl4AIClient(apiKey: "your_secret_token").crawl(urls: ["https://simonferns.com"]) + print("\(response)") } } @@ -95,3 +102,21 @@ extension Request { db(.readOnly) } } + +extension QueueContext { + var dbWrite: Database { + application.db(.readOnly) + } + + var dbReadOnly: Database { + application.db(.readOnly) + } +} + +extension Application.Queues { + func scheduleEvery(_ job: ScheduledJob, minutes: Int) { + for minuteOffset in stride(from: 0, to: 60, by: minutes) { + schedule(job).hourly().at(.init(integerLiteral: minuteOffset)) + } + } +} diff --git a/Backend/docker-compose.yml b/Backend/docker-compose.yml index 573a3cf7..85e56543 100644 --- a/Backend/docker-compose.yml +++ b/Backend/docker-compose.yml @@ -34,6 +34,23 @@ services: RUN_COMMAND: "queues" network_mode: host + + scheduled_worker_dev: + volumes: + - ./:/app/ + - /app/.build + build: + context: . + dockerfile: ./DevDockerfile + depends_on: + postgres: + condition: service_healthy + localstack: + condition: service_healthy + environment: + RUN_COMMAND: "queues --scheduled" + network_mode: host + postgres: image: postgres:latest container_name: postgres-container diff --git a/CLI/Sources/AutomaCLI/Commands/generate.swift b/CLI/Sources/AutomaCLI/Commands/generate.swift index f0af8863..a0944a00 100644 --- a/CLI/Sources/AutomaCLI/Commands/generate.swift +++ b/CLI/Sources/AutomaCLI/Commands/generate.swift @@ -91,7 +91,7 @@ let fileTypes: [FileType] = [ configurations: [ FileConfig( fromDirectory: "model/", - toDirectory: "./Sources/App/Models/", + toDirectory: "\(baseBackendAppPath)Models/", nestToDirectory: "", templates: [ "__CAPNAME__Model.swift.template", @@ -107,7 +107,7 @@ let fileTypes: [FileType] = [ ), FileConfig( fromDirectory: "migration/", - toDirectory: "./Sources/App/Migrations/", + toDirectory: "\(baseBackendAppPath)Migrations/", nestToDirectory: "", templates: [ "__CAPNAME__Migration__TIMESTAMP__.swift.template", @@ -146,7 +146,7 @@ let fileTypes: [FileType] = [ configurations: [ FileConfig( fromDirectory: "backend-service/", - toDirectory: "Sources/App/Services/", + toDirectory: "\(baseBackendAppPath)Services/", nestToDirectory: "__CAPNAME__Service/", templates: [ "__CAPNAME__Service.swift.template",