From 8792a722ee9206ad6c5f9b1847735e93ed503496 Mon Sep 17 00:00:00 2001 From: AdonisCodes Date: Fri, 7 Mar 2025 08:13:57 -0700 Subject: [PATCH 1/5] init: initialize rss feed service --- .vscode/launch.json | 18 ++++++ .../Sources/DataTypes/RSSFeedDTO.swift | 22 +++++++ Backend/Package.resolved | 2 +- .../RSSFeedClient/RSSFeedReaderClient.swift | 14 ++--- .../FeedTesterController.swift | 2 +- .../RSSFeedMigration1741359416.swift | 22 +++++++ Backend/Sources/App/Models/RSSFeedModel.swift | 59 +++++++++++++++++++ .../RSSFeedService/RSSFeedService.swift | 36 +++++++++++ CLI/Sources/AutomaCLI/Commands/generate.swift | 6 +- 9 files changed, 168 insertions(+), 13 deletions(-) create mode 100644 Backend/DataTypes/Sources/DataTypes/RSSFeedDTO.swift create mode 100644 Backend/Sources/App/Migrations/RSSFeedMigration1741359416.swift create mode 100644 Backend/Sources/App/Models/RSSFeedModel.swift create mode 100644 Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift diff --git a/.vscode/launch.json b/.vscode/launch.json index 8bffbadd..0fe4b936 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -17,6 +17,24 @@ "name": "Release App (Backend)", "program": "${workspaceFolder:AutomaInfraCore}/Backend/.build/release/App", "preLaunchTask": "swift: Build Release App (Backend)" + }, + { + "type": "lldb", + "request": "launch", + "args": [], + "cwd": "${workspaceFolder:AutomaInfraCore}/CLI", + "name": "Debug AutomaCLI (CLI)", + "program": "${workspaceFolder:AutomaInfraCore}/CLI/.build/debug/AutomaCLI", + "preLaunchTask": "swift: Build Debug AutomaCLI (CLI)" + }, + { + "type": "lldb", + "request": "launch", + "args": [], + "cwd": "${workspaceFolder:AutomaInfraCore}/CLI", + "name": "Release AutomaCLI (CLI)", + "program": "${workspaceFolder:AutomaInfraCore}/CLI/.build/release/AutomaCLI", + "preLaunchTask": "swift: Build Release AutomaCLI (CLI)" } ] } \ No newline at end of file diff --git a/Backend/DataTypes/Sources/DataTypes/RSSFeedDTO.swift b/Backend/DataTypes/Sources/DataTypes/RSSFeedDTO.swift new file mode 100644 index 00000000..3f3b34b6 --- /dev/null +++ b/Backend/DataTypes/Sources/DataTypes/RSSFeedDTO.swift @@ -0,0 +1,22 @@ +// RSSFeedDTO.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Vapor + +public struct RSSFeedDTO: Content { + public var id: UUID + public var link: URL + public var createdAt: Date + public var updatedAt: Date + public var deletedAt: Date + + public init(id: UUID, link: URL, createdAt: Date, updatedAt: Date, deletedAt: Date) { + self.id = id + self.link = link + self.createdAt = createdAt + self.updatedAt = updatedAt + self.deletedAt = deletedAt + } +} diff --git a/Backend/Package.resolved b/Backend/Package.resolved index 30ab962e..01a21c23 100644 --- a/Backend/Package.resolved +++ b/Backend/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "21a553d6b5237f7874318bb07a63ae91e30ff0407150f777c0f307ea5cf4dcc5", + "originHash" : "4e9475b2d5d9dafa8f3989be227cbdebdadd0a62dfb122a5ac3325094c4573ff", "pins" : [ { "identity" : "alamofire", diff --git a/Backend/Sources/App/Clients/RSSFeedClient/RSSFeedReaderClient.swift b/Backend/Sources/App/Clients/RSSFeedClient/RSSFeedReaderClient.swift index 5aa53a81..56daebec 100644 --- a/Backend/Sources/App/Clients/RSSFeedClient/RSSFeedReaderClient.swift +++ b/Backend/Sources/App/Clients/RSSFeedClient/RSSFeedReaderClient.swift @@ -11,16 +11,14 @@ import Retry import Vapor struct RSSFeedReaderClient { - func read(from url: URL) async -> RssFeedResponse { + func read(from url: URL) async throws -> RssFeedResponse { var feed: Feed? = nil - do { - try await retry( - maxAttempts: 3 - ) { - feed = try! await Feed(url: url) - } - } catch {} + try await retry( + maxAttempts: 3 + ) { + feed = try! await Feed(url: url) + } switch feed { case let .rss(rSSFeed): diff --git a/Backend/Sources/App/Controllers/FeedTesterController/FeedTesterController.swift b/Backend/Sources/App/Controllers/FeedTesterController/FeedTesterController.swift index 4a67b410..c75eb743 100644 --- a/Backend/Sources/App/Controllers/FeedTesterController/FeedTesterController.swift +++ b/Backend/Sources/App/Controllers/FeedTesterController/FeedTesterController.swift @@ -17,7 +17,7 @@ struct FeedTesterController: RouteCollection { @Sendable func request(req _: Request) async throws -> RssFeedResponse { let feedService: RSSFeedReaderClient = .init() - let response = await feedService.read(from: URL(string: "https://news.ycombinator.com/rss")!) + let response = try! await feedService.read(from: URL(string: "https://news.ycombinator.com/rss")!) return response } } diff --git a/Backend/Sources/App/Migrations/RSSFeedMigration1741359416.swift b/Backend/Sources/App/Migrations/RSSFeedMigration1741359416.swift new file mode 100644 index 00000000..74cd8d60 --- /dev/null +++ b/Backend/Sources/App/Migrations/RSSFeedMigration1741359416.swift @@ -0,0 +1,22 @@ +// RSSFeedMigration1741359416.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Fluent + +struct RSSFeedMigration1741359416: AsyncMigration { + func prepare(on database: Database) async throws { + try await database.schema("RSS-Feed") + .id() + .field("link", .string) + .field("updated_at", .datetime) + .field("created_at", .datetime) + .field("deleted_at", .datetime) + .create() + } + + func revert(on database: Database) async throws { + try await database.schema("RSS-Feed").delete() + } +} diff --git a/Backend/Sources/App/Models/RSSFeedModel.swift b/Backend/Sources/App/Models/RSSFeedModel.swift new file mode 100644 index 00000000..7d9ed336 --- /dev/null +++ b/Backend/Sources/App/Models/RSSFeedModel.swift @@ -0,0 +1,59 @@ +// RSSFeedModel.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import DataTypes +import Fluent +import Vapor + +final class RSSFeedModel: Model, @unchecked Sendable { + static let schema = "RSS-Feed" + + @ID(key: .id) + var id: UUID? + + @Field(key: "link") + var link: URL? + + @Timestamp(key: "created_at", on: .create) + var createdAt: Date? + + @Timestamp(key: "updated_at", on: .update) + var updatedAt: Date? + + @Timestamp(key: "deleted_at", on: .delete) + var deletedAt: Date? + + init() {} + + init( + id: UUID? = nil, + link _: URL? = nil, + createdAt: Date? = nil, + updatedAt: Date? = nil, + deletedAt: Date? = nil + ) { + self.id = id + self.createdAt = createdAt + self.updatedAt = updatedAt + self.deletedAt = deletedAt + } + + func toDTO() -> RSSFeedDTO { + .init( + id: id!, + link: link!, + createdAt: createdAt!, + updatedAt: updatedAt!, + deletedAt: deletedAt! + ) + } + + static func fromDTO(dto: RSSFeedDTO) -> RSSFeedModel { + let model = RSSFeedModel( + id: dto.id, link: dto.link, createdAt: dto.createdAt, updatedAt: dto.updatedAt, deletedAt: dto.deletedAt + ) + return model + } +} diff --git a/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift b/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift new file mode 100644 index 00000000..0e8adc92 --- /dev/null +++ b/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift @@ -0,0 +1,36 @@ +// RSSFeedService.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import DataTypes +import Fluent +import Vapor + +struct RSSFeedService { + let database: Database + + func addFeedToUser(userId _: UUID, feedUrl _: URL) async throws {} + + // If the model already exists we return true + func createFeedIfNotExist(feedUrl: URL) async throws -> (RSSFeedDTO, Bool) { + if let feed = try await RSSFeedModel.query(on: database).filter(\.$link == feedUrl).first() { + return (feed.toDTO(), true) + } + + let feed = RSSFeedModel( + id: UUID(), + link: feedUrl + ) + + try await feed.save(on: database) + + return (feed.toDTO(), false) + } + + // TODO: Discardable feed mapping id return + func addUserFeedMapping(feedId _: UUID, userId _: UUID) {} + + // Scrape Feed Posts + Add to table + // Scrape Feed Content + Add to table +} diff --git a/CLI/Sources/AutomaCLI/Commands/generate.swift b/CLI/Sources/AutomaCLI/Commands/generate.swift index f0af8863..a0944a00 100644 --- a/CLI/Sources/AutomaCLI/Commands/generate.swift +++ b/CLI/Sources/AutomaCLI/Commands/generate.swift @@ -91,7 +91,7 @@ let fileTypes: [FileType] = [ configurations: [ FileConfig( fromDirectory: "model/", - toDirectory: "./Sources/App/Models/", + toDirectory: "\(baseBackendAppPath)Models/", nestToDirectory: "", templates: [ "__CAPNAME__Model.swift.template", @@ -107,7 +107,7 @@ let fileTypes: [FileType] = [ ), FileConfig( fromDirectory: "migration/", - toDirectory: "./Sources/App/Migrations/", + toDirectory: "\(baseBackendAppPath)Migrations/", nestToDirectory: "", templates: [ "__CAPNAME__Migration__TIMESTAMP__.swift.template", @@ -146,7 +146,7 @@ let fileTypes: [FileType] = [ configurations: [ FileConfig( fromDirectory: "backend-service/", - toDirectory: "Sources/App/Services/", + toDirectory: "\(baseBackendAppPath)Services/", nestToDirectory: "__CAPNAME__Service/", templates: [ "__CAPNAME__Service.swift.template", From bd2bd4765308530f5cfb6b40b64c3ab4c923c2c4 Mon Sep 17 00:00:00 2001 From: AdonisCodes Date: Sat, 8 Mar 2025 07:48:42 -0700 Subject: [PATCH 2/5] feat: implement basics --- .../Sources/DataTypes/RSSFeedDTO.swift | 8 ++- .../DataTypes/UserFeedMappingDTO.swift | 24 +++++++ Backend/Public/favicon.ico | Bin 0 -> 1148 bytes .../FeedTesterController.swift | 14 ++++ .../UserFeedMappingMigration1741429746.swift | 23 ++++++ Backend/Sources/App/Models/RSSFeedModel.swift | 16 +++-- .../App/Models/UserFeedMappingModel.swift | 67 ++++++++++++++++++ .../RSSFeedService/RSSFeedService.swift | 42 +++++++++-- Backend/Sources/App/configure.swift | 4 +- 9 files changed, 181 insertions(+), 17 deletions(-) create mode 100644 Backend/DataTypes/Sources/DataTypes/UserFeedMappingDTO.swift create mode 100644 Backend/Public/favicon.ico create mode 100644 Backend/Sources/App/Migrations/UserFeedMappingMigration1741429746.swift create mode 100644 Backend/Sources/App/Models/UserFeedMappingModel.swift diff --git a/Backend/DataTypes/Sources/DataTypes/RSSFeedDTO.swift b/Backend/DataTypes/Sources/DataTypes/RSSFeedDTO.swift index 3f3b34b6..f8775b54 100644 --- a/Backend/DataTypes/Sources/DataTypes/RSSFeedDTO.swift +++ b/Backend/DataTypes/Sources/DataTypes/RSSFeedDTO.swift @@ -10,9 +10,13 @@ public struct RSSFeedDTO: Content { public var link: URL public var createdAt: Date public var updatedAt: Date - public var deletedAt: Date + public var deletedAt: Date? + + public init(id: UUID, link: String, createdAt: Date, updatedAt: Date, deletedAt: Date?) throws { + guard let link = URL(string: link) else { + throw GenericErrors.invalidUrl + } - public init(id: UUID, link: URL, createdAt: Date, updatedAt: Date, deletedAt: Date) { self.id = id self.link = link self.createdAt = createdAt diff --git a/Backend/DataTypes/Sources/DataTypes/UserFeedMappingDTO.swift b/Backend/DataTypes/Sources/DataTypes/UserFeedMappingDTO.swift new file mode 100644 index 00000000..4436790d --- /dev/null +++ b/Backend/DataTypes/Sources/DataTypes/UserFeedMappingDTO.swift @@ -0,0 +1,24 @@ +// UserFeedMappingDTO.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Vapor + +public struct UserFeedMappingDTO: Content { + public var id: UUID + public var feedId: UUID + public var userId: UUID + public var createdAt: Date + public var updatedAt: Date + public var deletedAt: Date? + + public init(id: UUID, feedId: UUID, userId: UUID, createdAt: Date, updatedAt: Date, deletedAt: Date?) { + self.id = id + self.userId = userId + self.feedId = feedId + self.createdAt = createdAt + self.updatedAt = updatedAt + self.deletedAt = deletedAt + } +} diff --git a/Backend/Public/favicon.ico b/Backend/Public/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..49cbb6bdfcb0425ddc4df6f286259fcf4bed0f6a GIT binary patch literal 1148 zcmZQzU<5)CU}R8WNMm7O5CgJ11N_{1xum#&OkPh9mmrWV2y?IjshFY_X&_#HkzWYo z`9V@;CTB?A!@_gCb8QGn|%K< zyNe%nb&U~~z0%j!Ew%dp!_V(FJ=?U0$NewQ{HF^lzu(*aJI-@m*y^iHAMS7BP+(|a zU|?e40J@Dj42}j@pg&j`1Q;AZ9-<0^dM3eLM1+wIue{mb2E@M(joWmIb8D55`TaNV zU$4IV^YNEiAG2>|o|%8XHE)IKtHQ@BH!U*L|HW+DTiVNgU{1FN=UsJkr>ZImw)t{l z*PkE0P+zI8a(eBGpU=$yf4cqm(dV?@$@An|zg^t=qBQs3t{5w~hj;&OD9MyC_Y<3N z@A7QLzd>fx4=3e+wk=!B?eJ1~u~FTTFDYj@zB&u0wl=C*vfy?DcxeNyIro8^A3FF*ZV&wAeb+%NUMOaj$arF2-%(zp>^Uc!m^SPWoKt+W=F)m@t^=#@W}fkSHtGFZ295?` z+F(*(U;@%GrUJvl|4Hg={z&>O2ZW|SHuzB?Fv-M@;irG?&+UIVAB+5H%{TXBSq;Ox zT(Rdi#d<%tf4ko=F4XYUyn1Q%bouk&!x&f+-knxiJNN6GMfbJl)l8i_H~#m7ERS2g zJH$(>&OR#t_o;B3bMo(`FT3<|*PV^O{P^`zYtD@y^WMg4%BzEXv+wx2Qc&7rIiN(H mREw1Qk(ksno90G7KW8tTSChEwS^I1TAn HTTPStatus { + let feedService = RSSFeedService(database: req.dbWrite) + if let url = URL(string: "https://news.ycombinator.com/rss") { + let success = try await feedService.addFeedToUser( + userId: UUID(uuidString: "562771bf-98a5-4cc4-83ba-5f618dc79546")!, + feedUrl: url + ) + return success ? .ok : .internalServerError + } + return .badRequest + } } diff --git a/Backend/Sources/App/Migrations/UserFeedMappingMigration1741429746.swift b/Backend/Sources/App/Migrations/UserFeedMappingMigration1741429746.swift new file mode 100644 index 00000000..b0abfff6 --- /dev/null +++ b/Backend/Sources/App/Migrations/UserFeedMappingMigration1741429746.swift @@ -0,0 +1,23 @@ +// UserFeedMappingMigration1741429746.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Fluent + +struct UserFeedMappingMigration1741429746: AsyncMigration { + func prepare(on database: Database) async throws { + try await database.schema("User-Feed-Mapping") + .id() + .field("feed_id", .uuid, .required, .references("RSS-Feed", "id")) + .field("user_id", .uuid, .required, .references("User", "id")) + .field("updated_at", .datetime) + .field("created_at", .datetime) + .field("deleted_at", .datetime) + .create() + } + + func revert(on database: Database) async throws { + try await database.schema("User-Feed-Mapping").delete() + } +} diff --git a/Backend/Sources/App/Models/RSSFeedModel.swift b/Backend/Sources/App/Models/RSSFeedModel.swift index 7d9ed336..44efefd1 100644 --- a/Backend/Sources/App/Models/RSSFeedModel.swift +++ b/Backend/Sources/App/Models/RSSFeedModel.swift @@ -14,7 +14,7 @@ final class RSSFeedModel: Model, @unchecked Sendable { var id: UUID? @Field(key: "link") - var link: URL? + var link: String @Timestamp(key: "created_at", on: .create) var createdAt: Date? @@ -29,30 +29,32 @@ final class RSSFeedModel: Model, @unchecked Sendable { init( id: UUID? = nil, - link _: URL? = nil, + link: String, createdAt: Date? = nil, updatedAt: Date? = nil, deletedAt: Date? = nil ) { self.id = id + self.link = link self.createdAt = createdAt self.updatedAt = updatedAt self.deletedAt = deletedAt } - func toDTO() -> RSSFeedDTO { - .init( + func toDTO() throws -> RSSFeedDTO { + try .init( id: id!, - link: link!, + link: link, createdAt: createdAt!, updatedAt: updatedAt!, - deletedAt: deletedAt! + deletedAt: deletedAt ) } static func fromDTO(dto: RSSFeedDTO) -> RSSFeedModel { let model = RSSFeedModel( - id: dto.id, link: dto.link, createdAt: dto.createdAt, updatedAt: dto.updatedAt, deletedAt: dto.deletedAt + id: dto.id, link: dto.link.absoluteString, createdAt: dto.createdAt, updatedAt: dto.updatedAt, + deletedAt: dto.deletedAt ) return model } diff --git a/Backend/Sources/App/Models/UserFeedMappingModel.swift b/Backend/Sources/App/Models/UserFeedMappingModel.swift new file mode 100644 index 00000000..c2513851 --- /dev/null +++ b/Backend/Sources/App/Models/UserFeedMappingModel.swift @@ -0,0 +1,67 @@ +// UserFeedMappingModel.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import DataTypes +import Fluent +import Vapor + +final class UserFeedMappingModel: Model, @unchecked Sendable { + static let schema = "User-Feed-Mapping" + + @ID(key: .id) + var id: UUID? + + @Field(key: "user_id") + var userId: UUID + + @Field(key: "feed_id") + var feedId: UUID + + @Timestamp(key: "created_at", on: .create) + var createdAt: Date? + + @Timestamp(key: "updated_at", on: .update) + var updatedAt: Date? + + @Timestamp(key: "deleted_at", on: .delete) + var deletedAt: Date? + + init() {} + + init( + id: UUID? = nil, + userId: UUID, + feedId: UUID, + createdAt: Date? = nil, + updatedAt: Date? = nil, + deletedAt: Date? = nil + ) { + self.id = id + self.userId = userId + self.feedId = feedId + self.createdAt = createdAt + self.updatedAt = updatedAt + self.deletedAt = deletedAt + } + + func toDTO() -> UserFeedMappingDTO { + .init( + id: id!, + feedId: feedId, + userId: userId, + createdAt: createdAt!, + updatedAt: updatedAt!, + deletedAt: deletedAt + ) + } + + static func fromDTO(dto: UserFeedMappingDTO) -> UserFeedMappingModel { + let model = UserFeedMappingModel( + id: dto.id, userId: dto.userId, feedId: dto.feedId, createdAt: dto.createdAt, updatedAt: dto.updatedAt, + deletedAt: dto.deletedAt + ) + return model + } +} diff --git a/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift b/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift index 0e8adc92..836dafe8 100644 --- a/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift +++ b/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift @@ -10,26 +10,54 @@ import Vapor struct RSSFeedService { let database: Database - func addFeedToUser(userId _: UUID, feedUrl _: URL) async throws {} + func addFeedToUser(userId: UUID, feedUrl: URL) async throws -> Bool { + do { + let (feed, _) = try await createFeedInDBIfNotExist(feedUrl: feedUrl) + try await addUserFeedMapping(feedId: feed.id, userId: userId) + return true + } catch { + // TODO: Log error + print("unknown error \(error.localizedDescription)") + return false + } + } // If the model already exists we return true - func createFeedIfNotExist(feedUrl: URL) async throws -> (RSSFeedDTO, Bool) { - if let feed = try await RSSFeedModel.query(on: database).filter(\.$link == feedUrl).first() { - return (feed.toDTO(), true) + func createFeedInDBIfNotExist(feedUrl: URL) async throws -> (RSSFeedDTO, Bool) { + let feedUrlString = feedUrl.absoluteString + if let feed = try await RSSFeedModel.query(on: database).filter(\.$link == feedUrlString).first() { + // The feed exists, so we just return it as a DTO + return try (feed.toDTO(), true) } + // Create the new feed model let feed = RSSFeedModel( id: UUID(), - link: feedUrl + link: feedUrlString ) + // Save the newly created feed try await feed.save(on: database) - return (feed.toDTO(), false) + // Now that the feed is saved, return the DTO + return try (feed.toDTO(), false) } // TODO: Discardable feed mapping id return - func addUserFeedMapping(feedId _: UUID, userId _: UUID) {} + @discardableResult + func addUserFeedMapping(feedId: UUID, userId: UUID) async throws -> UserFeedMappingDTO { + if let existingMapping = try await UserFeedMappingModel.query(on: database).filter(\.$userId == userId) + .filter(\.$feedId == feedId).first() + { + return existingMapping.toDTO() + } + + let newMapping = UserFeedMappingModel(userId: userId, feedId: feedId) + + try await newMapping.save(on: database) + + return newMapping.toDTO() + } // Scrape Feed Posts + Add to table // Scrape Feed Content + Add to table diff --git a/Backend/Sources/App/configure.swift b/Backend/Sources/App/configure.swift index dc5ba769..c6970ab7 100644 --- a/Backend/Sources/App/configure.swift +++ b/Backend/Sources/App/configure.swift @@ -13,7 +13,7 @@ import Vapor // Configures your application public func configure(_ app: Application) async throws { // This is file middleware - // app.middleware.use(FileMiddleware(publicDirectory: app.directory.publicDirectory)) + app.middleware.use(FileMiddleware(publicDirectory: app.directory.publicDirectory)) app.middleware.use(ErrorStringMiddleware()) // Errors are getting thrown locally, this prevents run App & ./App execution diffs @@ -48,6 +48,8 @@ public func configure(_ app: Application) async throws { app.migrations.add(JobMetadataMigrate()) app.migrations.add(RemoveUserStorageMigration1739456565()) app.migrations.add(AddAcceptedColumnMigration1740658649()) + app.migrations.add(RSSFeedMigration1741359416()) + app.migrations.add(UserFeedMappingMigration1741429746()) try await app.autoMigrate() From 5c612f2fb94738e53b246956e6af3a4a37c8045c Mon Sep 17 00:00:00 2001 From: AdonisCodes Date: Sun, 9 Mar 2025 08:33:44 -0600 Subject: [PATCH 3/5] feat: work on rss feed --- Backend/Package.resolved | 11 ++++++++++- Backend/Package.swift | 2 ++ .../App/Services/RSSFeedService/RSSFeedService.swift | 3 +++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Backend/Package.resolved b/Backend/Package.resolved index 01a21c23..8e5470cf 100644 --- a/Backend/Package.resolved +++ b/Backend/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "4e9475b2d5d9dafa8f3989be227cbdebdadd0a62dfb122a5ac3325094c4573ff", + "originHash" : "b810cd9246a77258ba53551bbd20d971d3aa5cf91046d2ef9addde0be4542884", "pins" : [ { "identity" : "alamofire", @@ -10,6 +10,15 @@ "version" : "5.10.2" } }, + { + "identity" : "anycodable", + "kind" : "remoteSourceControl", + "location" : "https://github.com/Flight-School/AnyCodable", + "state" : { + "revision" : "862808b2070cd908cb04f9aafe7de83d35f81b05", + "version" : "0.6.7" + } + }, { "identity" : "async-http-client", "kind" : "remoteSourceControl", diff --git a/Backend/Package.swift b/Backend/Package.swift index 6dd456ec..34626782 100644 --- a/Backend/Package.swift +++ b/Backend/Package.swift @@ -28,6 +28,7 @@ let package = Package( .package(url: "https://github.com/swhitty/FlyingFox.git", .upToNextMajor(from: "0.20.0")), .package(url: "https://github.com/nmdias/FeedKit.git", from: "10.0.0-rc.3"), .package(url: "https://github.com/GetAutomaApp/swift-retry.git", branch: "main"), + .package(path: "./FirecrawlSwift"), ], targets: [ .executableTarget( @@ -50,6 +51,7 @@ let package = Package( .product(name: "SotoTextract", package: "soto"), .product(name: "FeedKit", package: "FeedKit"), .product(name: "DMRetry", package: "swift-retry"), + .product(name: "FirecrawlSwift", package: "FirecrawlSwift"), ], exclude: [ "Documentation.md", diff --git a/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift b/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift index 836dafe8..8d9639b8 100644 --- a/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift +++ b/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift @@ -60,5 +60,8 @@ struct RSSFeedService { } // Scrape Feed Posts + Add to table + func scrapeRSSFeedPosts(feedId _: UUID) async throws { + let + } // Scrape Feed Content + Add to table } From 5844fc024a110266b75a80c9b901a93be87b5415 Mon Sep 17 00:00:00 2001 From: AdonisCodes Date: Fri, 14 Mar 2025 08:17:02 -0600 Subject: [PATCH 4/5] feat: implementations --- .../Sources/DataTypes/RssFeedItemDTO.swift | 34 +++++++++ Backend/Package.swift | 2 - .../RSSFeedClient/RSSFeedReaderClient.swift | 2 +- .../FeedTesterController.swift | 2 +- .../RssFeedItemMigration1741876963.swift | 29 ++++++++ .../Sources/App/Models/RSSFeedItemModel.swift | 73 +++++++++++++++++++ .../Procs/CronJobs/ScrapeRSSFeedCronJob.swift | 19 +++++ .../RSSFeedService/RSSFeedService.swift | 69 +++++++++++++++++- Backend/Sources/App/configure.swift | 20 +++++ Backend/docker-compose.yml | 17 +++++ 10 files changed, 259 insertions(+), 8 deletions(-) create mode 100644 Backend/DataTypes/Sources/DataTypes/RssFeedItemDTO.swift create mode 100644 Backend/Sources/App/Migrations/RssFeedItemMigration1741876963.swift create mode 100644 Backend/Sources/App/Models/RSSFeedItemModel.swift create mode 100644 Backend/Sources/App/Procs/CronJobs/ScrapeRSSFeedCronJob.swift diff --git a/Backend/DataTypes/Sources/DataTypes/RssFeedItemDTO.swift b/Backend/DataTypes/Sources/DataTypes/RssFeedItemDTO.swift new file mode 100644 index 00000000..b22e08b3 --- /dev/null +++ b/Backend/DataTypes/Sources/DataTypes/RssFeedItemDTO.swift @@ -0,0 +1,34 @@ +// RssFeedItemDTO.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Vapor + +public struct RssFeedItemDTO: Content { + public var id: UUID? + public var rssFeedId: UUID + public var content: String + public var link: String + public var createdAt: Date? + public var updatedAt: Date? + public var deletedAt: Date? + + public init( + id: UUID?, + rssFeedId: UUID, + content: String, + link: String, + createdAt: Date?, + updatedAt: Date?, + deletedAt: Date? + ) { + self.id = id + self.rssFeedId = rssFeedId + self.content = content + self.link = link + self.createdAt = createdAt + self.updatedAt = updatedAt + self.deletedAt = deletedAt + } +} diff --git a/Backend/Package.swift b/Backend/Package.swift index 28730f55..e166bdbd 100644 --- a/Backend/Package.swift +++ b/Backend/Package.swift @@ -29,7 +29,6 @@ let package = Package( .package(url: "https://github.com/swhitty/FlyingFox.git", .upToNextMajor(from: "0.20.0")), .package(url: "https://github.com/nmdias/FeedKit.git", from: "10.0.0-rc.3"), .package(url: "https://github.com/GetAutomaApp/swift-retry.git", branch: "main"), - .package(path: "./FirecrawlSwift"), ], targets: [ .executableTarget( @@ -52,7 +51,6 @@ let package = Package( .product(name: "SotoTextract", package: "soto"), .product(name: "FeedKit", package: "FeedKit"), .product(name: "DMRetry", package: "swift-retry"), - .product(name: "FirecrawlSwift", package: "FirecrawlSwift"), ], exclude: [ "Documentation.md", diff --git a/Backend/Sources/App/Clients/RSSFeedClient/RSSFeedReaderClient.swift b/Backend/Sources/App/Clients/RSSFeedClient/RSSFeedReaderClient.swift index 56daebec..fd4385fd 100644 --- a/Backend/Sources/App/Clients/RSSFeedClient/RSSFeedReaderClient.swift +++ b/Backend/Sources/App/Clients/RSSFeedClient/RSSFeedReaderClient.swift @@ -29,7 +29,7 @@ struct RSSFeedReaderClient { } } - func convertRSSToGenericFeedItems(from feedItems: [RSSFeedItem]) -> [GenericRSSFeedItem] { + private func convertRSSToGenericFeedItems(from feedItems: [RSSFeedItem]) -> [GenericRSSFeedItem] { let items = feedItems.compactMap { feedItem -> GenericRSSFeedItem? in guard let title = feedItem.title, diff --git a/Backend/Sources/App/Controllers/FeedTesterController/FeedTesterController.swift b/Backend/Sources/App/Controllers/FeedTesterController/FeedTesterController.swift index 681f98f1..da6b9c5f 100644 --- a/Backend/Sources/App/Controllers/FeedTesterController/FeedTesterController.swift +++ b/Backend/Sources/App/Controllers/FeedTesterController/FeedTesterController.swift @@ -24,7 +24,7 @@ struct FeedTesterController: RouteCollection { @Sendable func addToUser(req: Request) async throws -> HTTPStatus { - let feedService = RSSFeedService(database: req.dbWrite) + let feedService = RSSFeedService(database: req.dbWrite, client: req.client, logger: req.logger) if let url = URL(string: "https://news.ycombinator.com/rss") { let success = try await feedService.addFeedToUser( userId: UUID(uuidString: "562771bf-98a5-4cc4-83ba-5f618dc79546")!, diff --git a/Backend/Sources/App/Migrations/RssFeedItemMigration1741876963.swift b/Backend/Sources/App/Migrations/RssFeedItemMigration1741876963.swift new file mode 100644 index 00000000..c84c9d25 --- /dev/null +++ b/Backend/Sources/App/Migrations/RssFeedItemMigration1741876963.swift @@ -0,0 +1,29 @@ +// RssFeedItemMigration1741876963.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Fluent + +struct RssFeedItemMigration1741876963: AsyncMigration { + func prepare(on database: Database) async throws { + try await database.schema("RSS-Feed-Item") + .id() + .field( + "rss_feed_id", + .uuid, + .required, + .references("RSS-Feed", "id") + ) + .field("content", .string, .required) + .field("link", .string, .required) + .field("updated_at", .datetime) + .field("created_at", .datetime) + .field("deleted_at", .datetime) + .create() + } + + func revert(on database: Database) async throws { + try await database.schema("Rss-Feed-Item").delete() + } +} diff --git a/Backend/Sources/App/Models/RSSFeedItemModel.swift b/Backend/Sources/App/Models/RSSFeedItemModel.swift new file mode 100644 index 00000000..b17f197d --- /dev/null +++ b/Backend/Sources/App/Models/RSSFeedItemModel.swift @@ -0,0 +1,73 @@ +// RSSFeedItemModel.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import DataTypes +import Fluent +import Vapor + +final class RSSFeedItemModel: Model, @unchecked Sendable { + static let schema = "RSS-Feed-Item" + + @ID(key: .id) + var id: UUID? + + @Field(key: "rss_feed_id") + var rssFeedId: UUID + + @Field(key: "content") + var content: String + + @Field(key: "link") + var link: String + + @Timestamp(key: "created_at", on: .create) + var createdAt: Date? + + @Timestamp(key: "updated_at", on: .update) + var updatedAt: Date? + + @Timestamp(key: "deleted_at", on: .delete) + var deletedAt: Date? + + init() {} + + init( + id: UUID? = nil, + rssFeedId: UUID, + content: String, + link: String, + createdAt: Date? = nil, + updatedAt: Date? = nil, + deletedAt: Date? = nil + ) { + self.id = id + self.rssFeedId = rssFeedId + self.content = content + self.link = link + self.createdAt = createdAt + self.updatedAt = updatedAt + self.deletedAt = deletedAt + } + + func toDTO() -> RssFeedItemDTO { + .init( + id: id, + rssFeedId: rssFeedId, + content: content, + link: link, + createdAt: createdAt, + updatedAt: updatedAt, + deletedAt: deletedAt + ) + } + + static func fromDTO(dto: RssFeedItemDTO) -> RSSFeedItemModel { + let model = RSSFeedItemModel( + id: dto.id, rssFeedId: dto.rssFeedId, content: dto.content, link: dto.link, createdAt: dto.createdAt, + updatedAt: dto.updatedAt, deletedAt: dto.deletedAt + ) + return model + } +} diff --git a/Backend/Sources/App/Procs/CronJobs/ScrapeRSSFeedCronJob.swift b/Backend/Sources/App/Procs/CronJobs/ScrapeRSSFeedCronJob.swift new file mode 100644 index 00000000..e4cf0f65 --- /dev/null +++ b/Backend/Sources/App/Procs/CronJobs/ScrapeRSSFeedCronJob.swift @@ -0,0 +1,19 @@ +// ScrapeRSSFeedCronJob.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Queues +import Vapor + +struct ScrapeRSSFeedCronJob: AsyncScheduledJob { + func run(context: QueueContext) async throws { + let rssFeedService = RSSFeedService( + database: context.dbWrite, + client: context.application.client, + logger: context.logger + ) + + try await rssFeedService.scrapeAndInsertLatestForAllFeeds() + } +} diff --git a/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift b/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift index 8d9639b8..61295eef 100644 --- a/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift +++ b/Backend/Sources/App/Services/RSSFeedService/RSSFeedService.swift @@ -9,9 +9,17 @@ import Vapor struct RSSFeedService { let database: Database + let rssFeedClient = RSSFeedReaderClient() + let client: Client + let logger: Logger func addFeedToUser(userId: UUID, feedUrl: URL) async throws -> Bool { do { + if try await !rssFeedClient.read(from: feedUrl).isRssFeed { + print("isn't rss feed") + return false + } + let (feed, _) = try await createFeedInDBIfNotExist(feedUrl: feedUrl) try await addUserFeedMapping(feedId: feed.id, userId: userId) return true @@ -59,9 +67,62 @@ struct RSSFeedService { return newMapping.toDTO() } - // Scrape Feed Posts + Add to table - func scrapeRSSFeedPosts(feedId _: UUID) async throws { - let + func scrapeAndInsertLatestForAllFeeds() async throws { + let feeds = try await RSSFeedModel.query(on: database).all() + + for feed in feeds { + do { + try await scrapeAndInsertLatestRSSFeedItems(feedDTO: feed.toDTO()) + } catch { + // Fix log + logger.error("Failed to process feed \(feed.link): \(error.localizedDescription)") + } + } + } + + func scrapeAndInsertLatestRSSFeedItems(feedDTO: RSSFeedDTO) async throws { + let firecrawlClient = try FirecrawlClient(client: client, logger: logger) + + let url = URL(string: feedDTO.link.absoluteString) + + guard let url else { + return + } + + let latestFeedItems = try await rssFeedClient.read(from: url).items + let latestFeedItemLinks = latestFeedItems.map(\.link) + + let existingFeedItems = try await RSSFeedItemModel.query(on: database) + .filter(\.$link ~~ latestFeedItemLinks) + .all() + + let existingFeedItemLinks = Set(existingFeedItems.map(\.link)) + + let unprocessedLinks = Set(latestFeedItemLinks).subtracting(existingFeedItemLinks) + + var feedItems: [RssFeedItemDTO] = [] + + for link in unprocessedLinks { + do { + let content = try await firecrawlClient.scrapeMarkdown( + from: .init(url: link) + ) + + let article = RSSFeedItemModel( + rssFeedId: feedDTO.id, + content: content.markdown, + link: link + ) + + try await article.save(on: database) + + feedItems.append(article.toDTO()) + } catch { + // Log the error if scraping fails + logger.error("Failed to scrape content for link \(link): \(error.localizedDescription)") + } + } + + // TODO: Log feedItems if necessary } - // Scrape Feed Content + Add to table } diff --git a/Backend/Sources/App/configure.swift b/Backend/Sources/App/configure.swift index e9ffa72d..51bb1451 100644 --- a/Backend/Sources/App/configure.swift +++ b/Backend/Sources/App/configure.swift @@ -55,6 +55,7 @@ public func configure(_ app: Application) async throws { app.migrations.add(AddAcceptedColumnMigration1740658649()) app.migrations.add(RSSFeedMigration1741359416()) app.migrations.add(UserFeedMappingMigration1741429746()) + app.migrations.add(RssFeedItemMigration1741876963()) try await app.autoMigrate() @@ -77,6 +78,7 @@ public func configure(_ app: Application) async throws { // Jobs app.queues.add(TransactionalMessageAsyncJob()) app.queues.add(ProfilePictureAsyncJob()) + app.queues.scheduleEvery(ScrapeRSSFeedCronJob(), minutes: 5) // Http Server Config app.http.server.configuration.responseCompression = .enabled @@ -97,3 +99,21 @@ extension Request { db(.readOnly) } } + +extension QueueContext { + var dbWrite: Database { + application.db(.readOnly) + } + + var dbReadOnly: Database { + application.db(.readOnly) + } +} + +extension Application.Queues { + func scheduleEvery(_ job: ScheduledJob, minutes: Int) { + for minuteOffset in stride(from: 0, to: 60, by: minutes) { + schedule(job).hourly().at(.init(integerLiteral: minuteOffset)) + } + } +} diff --git a/Backend/docker-compose.yml b/Backend/docker-compose.yml index 573a3cf7..85e56543 100644 --- a/Backend/docker-compose.yml +++ b/Backend/docker-compose.yml @@ -34,6 +34,23 @@ services: RUN_COMMAND: "queues" network_mode: host + + scheduled_worker_dev: + volumes: + - ./:/app/ + - /app/.build + build: + context: . + dockerfile: ./DevDockerfile + depends_on: + postgres: + condition: service_healthy + localstack: + condition: service_healthy + environment: + RUN_COMMAND: "queues --scheduled" + network_mode: host + postgres: image: postgres:latest container_name: postgres-container From 98151b666eb00e64f6c27584aa94d2e0fe8f5706 Mon Sep 17 00:00:00 2001 From: AdonisCodes Date: Thu, 20 Mar 2025 07:10:39 -0600 Subject: [PATCH 5/5] git stash --- .../Sources/App/Clients/Crawl4AIClient.swift | 119 ++++++++++++++++++ .../FirecrawlClient/FirecrawlClient.swift | 8 +- .../FirecrawlClientTypes.swift | 1 + Backend/Sources/App/configure.swift | 5 +- 4 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 Backend/Sources/App/Clients/Crawl4AIClient.swift diff --git a/Backend/Sources/App/Clients/Crawl4AIClient.swift b/Backend/Sources/App/Clients/Crawl4AIClient.swift new file mode 100644 index 00000000..3eeeb214 --- /dev/null +++ b/Backend/Sources/App/Clients/Crawl4AIClient.swift @@ -0,0 +1,119 @@ +// Crawl4AIClient.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Foundation + +struct CrawlRequest: Codable { + let urls: [String] + let browserConfig: BrowserConfig? + let crawlerConfig: CrawlerConfig? + + enum CodingKeys: String, CodingKey { + case urls + case browserConfig = "browser_config" + case crawlerConfig = "crawler_config" + } +} + +struct BrowserConfig: Codable { + let headless: Bool + + init(headless: Bool = true) { + self.headless = headless + } +} + +struct CrawlerConfig: Codable { + let stream: Bool + + init(stream: Bool = false) { + self.stream = stream + } +} + +struct CrawlTaskId: Codable { + let taskId: UUID + + enum CodingKeys: String, CodingKey { + case taskId = "task_id" + } +} + +struct CrawlResult: Codable { + let results: [Result] + + struct Result: Codable { + let url: String + let extractedContent: String? + let markdown: String? + + enum CodingKeys: String, CodingKey { + case url + case extractedContent = "extracted_content" + case markdown + } + } +} + +class Crawl4AIClient { + private let baseURL: URL + private let apiKey: String + private let session: URLSession + + init(baseURL: String = "http://localhost:11235", apiKey: String) { + self.baseURL = URL(string: baseURL)! + self.apiKey = apiKey + session = URLSession.shared + } + + func startCrawl(urls: [String], browserConfig: BrowserConfig? = nil, + crawlerConfig: CrawlerConfig? = nil) async throws -> CrawlTaskId + { + let request = CrawlRequest(urls: urls, browserConfig: browserConfig, crawlerConfig: crawlerConfig) + + var urlRequest = URLRequest(url: baseURL.appendingPathComponent("crawl")) + urlRequest.httpMethod = "POST" + urlRequest.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization") + urlRequest.setValue("application/json", forHTTPHeaderField: "Content-Type") + + let encoder = JSONEncoder() + urlRequest.httpBody = try encoder.encode(request) + + let (data, response) = try await session.data(for: urlRequest) + + guard let httpResponse = response as? HTTPURLResponse else { + throw URLError(.badServerResponse) + } + + guard httpResponse.statusCode == 200 else { + throw URLError(.badServerResponse) + } + + let decoder = JSONDecoder() + let decodedResponse = try decoder.decode(CrawlTaskId.self, from: data) + return decodedResponse + } + + func getCrawl +} + +// Example usage: +/* + let client = Crawl4AIClient(apiKey: "YOUR_API_KEY") + + Task { + do { + let result = try await client.crawl(urls: ["https://example.com"]) + print("Crawl success: \(result.success)") + for item in result.results { + print("URL: \(item.url)") + print("Content: \(item.extractedContent ?? "")") + print("Markdown: \(item.markdown ?? "")") + } + } catch { + print("Error: \(error)") + } + } + */ diff --git a/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClient.swift b/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClient.swift index b157b05d..1ebdc168 100644 --- a/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClient.swift +++ b/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClient.swift @@ -38,13 +38,15 @@ struct FirecrawlClient { ) guard let decodedData = try? response.content.decode(FirecrawlScrapeResult.self) else { + let bodyData = response.body?.getData(at: 0, length: response.body?.readableBytes ?? 0) + let bodyString = bodyData + .flatMap { String(data: $0, encoding: .utf8) } ?? "Unable to convert body to string" + logger.error( "Invalid response from firecrawl microservice", metadata: [ "url": .string(input.url), - "description": .string( - response.body?.debugDescription ?? response.description - ), + "response": .string(bodyString), "to": .string("FirecrawlClient.scrapeMarkdown"), ] ) diff --git a/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClientTypes.swift b/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClientTypes.swift index 315f77a9..9c174d7b 100644 --- a/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClientTypes.swift +++ b/Backend/Sources/App/Clients/FirecrawlClient/FirecrawlClientTypes.swift @@ -11,6 +11,7 @@ enum FirecrawlFormats: String, Content { struct ScrapeMarkdownInput: Content { let url: String + let timeout: Int = 60000 } struct FirecrawlDataResult: Content { diff --git a/Backend/Sources/App/configure.swift b/Backend/Sources/App/configure.swift index 51bb1451..7df0e442 100644 --- a/Backend/Sources/App/configure.swift +++ b/Backend/Sources/App/configure.swift @@ -78,10 +78,13 @@ public func configure(_ app: Application) async throws { // Jobs app.queues.add(TransactionalMessageAsyncJob()) app.queues.add(ProfilePictureAsyncJob()) - app.queues.scheduleEvery(ScrapeRSSFeedCronJob(), minutes: 5) +// app.queues.scheduleEvery(ScrapeRSSFeedCronJob(), minutes: 5) // Http Server Config app.http.server.configuration.responseCompression = .enabled + + let response = try await Crawl4AIClient(apiKey: "your_secret_token").crawl(urls: ["https://simonferns.com"]) + print("\(response)") } }