diff --git a/.gitignore b/.gitignore index d2fef48..3d6ea8b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules -tasks \ No newline at end of file +tasks +.env \ No newline at end of file diff --git a/_tasks/02_add_remove_elements.json b/_tasks/02_add_remove_elements.json new file mode 100644 index 0000000..f39e2c0 --- /dev/null +++ b/_tasks/02_add_remove_elements.json @@ -0,0 +1,20 @@ +{ + "name": "add_remove_elements", + "tasks": [ + { + "action": "visit", + "url": "https://the-internet.herokuapp.com/add_remove_elements/" + }, + { + "action": "click", + "selector": "//button[text()='Add Element']", + "xpath": "True" + }, + { + "action": "click", + "selector": "//button[text()='Delete']", + "xpath": "True" + } + ], + "timestamp": 1695408306590 +} \ No newline at end of file diff --git a/_tasks/05_challenging_dom.json b/_tasks/05_challenging_dom.json new file mode 100644 index 0000000..c4a972a --- /dev/null +++ b/_tasks/05_challenging_dom.json @@ -0,0 +1,14 @@ +{ + "name": "challenging_dom", + "tasks": [ + { + "action": "visit", + "url": "https://the-internet.herokuapp.com/challenging_dom" + }, + { + "action": "evaluate", + "expression": "document.querySelector('.large-10 table tbody tr:nth-child(4) td:nth-child(5)').textContent" + } + ], + "timestamp": 1695234997171 +} \ No newline at end of file diff --git a/_tasks/06_checkboxes.json b/_tasks/06_checkboxes.json new file mode 100644 index 0000000..cae6bce --- /dev/null +++ b/_tasks/06_checkboxes.json @@ -0,0 +1,20 @@ +{ + "name": "checkboxes", + "tasks": [ + { + "action": "visit", + "url": "https://the-internet.herokuapp.com/checkboxes" + }, + { + "action": "click", + "selector": "//input[@type='checkbox'][2]", + "xpath": "True" + }, + { + "action": "click", + "selector": "//input[@type='checkbox'][1]", + "xpath": "True" + } + ], + "timestamp": 1695235445064 +} \ No newline at end of file diff --git a/_tasks/09_disappearing_elements.json b/_tasks/09_disappearing_elements.json new file mode 100644 index 0000000..53f183c --- /dev/null +++ b/_tasks/09_disappearing_elements.json @@ -0,0 +1,30 @@ +{ + "name": "disappearing_elements", + "tasks": [ + { + "action": "visit", + "url": "https://the-internet.herokuapp.com/disappearing_elements" + }, + { + "action": "hover", + "selector": "//ul/li[1]", + "xpath": "True" + }, + { + "action": "hover", + "selector": "//ul/li[2]", + "xpath": "True" + }, + { + "action": "hover", + "selector": "//ul/li[3]", + "xpath": "True" + }, + { + "action": "hover", + "selector": "//ul/li[4]", + "xpath": "True" + } + ], + "timestamp": 1695236242314 +} \ No newline at end of file diff --git a/_tasks/10_drag_and_drop.json b/_tasks/10_drag_and_drop.json new file mode 100644 index 0000000..93740bb --- /dev/null +++ b/_tasks/10_drag_and_drop.json @@ -0,0 +1,14 @@ +{ + "name": "drag_and_drop", + "tasks": [ + { + "action": "visit", + "url": "https://the-internet.herokuapp.com/drag_and_drop" + }, + { + "action": "evaluate", + "expression": "(() => { const columnA = document.querySelector('#column-a'); const columnB = document.querySelector('#column-b'); const dataTransfer = new DataTransfer(); columnA.dispatchEvent(new DragEvent('dragstart', { dataTransfer: dataTransfer })); columnB.dispatchEvent(new DragEvent('drop', { dataTransfer: dataTransfer })); return 'Drag and Drop simulated with DataTransfer'; })()" + } + ], + "timestamp": 1695407675816 +} \ No newline at end of file diff --git a/_tasks/11_dropdown.json b/_tasks/11_dropdown.json new file mode 100644 index 0000000..2b32ba1 --- /dev/null +++ b/_tasks/11_dropdown.json @@ -0,0 +1,19 @@ +{ + "name": "dropdown", + "tasks": [ + { + "action": "visit", + "url": "https://the-internet.herokuapp.com/dropdown" + }, + { + "action": "click", + "selector": "#dropdown", + "xpath": "False" + }, + { + "action": "evaluate", + "expression": "document.querySelector('#dropdown').value = '1';" + } + ], + "timestamp": 1695409162823 +} \ No newline at end of file diff --git a/actions/get.ts b/actions/get.ts index 08918f7..56306bd 100644 --- a/actions/get.ts +++ b/actions/get.ts @@ -14,6 +14,20 @@ export const click = { controller: controller.click } +export const hover = { + method: "get", + action: "hover", + path: "/hover", + controller: controller.hover +} + +export const select = { + method: "get", + action: "select", + path: "/select", + controller: controller.select +} + export const observe = { method: "get", action: "observe", @@ -52,6 +66,8 @@ export const exit = { export default [ visit, click, + hover, + select, observe, router, type, diff --git a/actions/index.ts b/actions/index.ts index 6848f3c..6e6fd58 100644 --- a/actions/index.ts +++ b/actions/index.ts @@ -1,7 +1,18 @@ import get from './get'; import post from './post'; +import { + visit, click, hover, observe, router, + type, wait, exit +} from './get'; +import { mouse, evaluate } from './post' + +export const actions = { + visit, click, hover, observe, router, + type, evaluate, wait, exit, mouse +} + export default { get, post -} +} \ No newline at end of file diff --git a/actions/middlewares.ts b/actions/middlewares.ts new file mode 100644 index 0000000..ec980ce --- /dev/null +++ b/actions/middlewares.ts @@ -0,0 +1,9 @@ +import express, { NextFunction, Request, Response } from 'express'; + +export const resolveJSON = (req: Request, res: Response, next: NextFunction) => { + express.json()(req, res, next); +} + +export const resolveRaw = (req: Request, res: Response, next: NextFunction) => { + express.raw({ type: "text/plain" })(req, res, next); +} \ No newline at end of file diff --git a/actions/post.ts b/actions/post.ts index e7836fb..1de02d5 100644 --- a/actions/post.ts +++ b/actions/post.ts @@ -1,22 +1,41 @@ import * as controller from '../controller'; +import { resolveRaw, resolveJSON } from './middlewares'; + +export const mouse = { + method: "post", + action: "mouse", + path: "/mouse", + middlewares: [resolveJSON], + controller: controller.mouse +} + +export const evaluate = { + method: "post", + action: "evaluate", + path: "/evaluate", + middlewares: [resolveJSON], + controller: controller.evaluate +} const savetask = { method: "post", action: "savetask", path: "/savetask", + middlewares: [resolveJSON], controller: controller.savetask } - const replay = { method: "get", action: "replay", path: "/replay", + middlewares: [], controller: controller.replay } - export default [ + mouse, + evaluate, savetask, replay ] \ No newline at end of file diff --git a/controller/index.ts b/controller/index.ts index ce0ec4e..df3f802 100644 --- a/controller/index.ts +++ b/controller/index.ts @@ -2,9 +2,10 @@ import { Request, Response } from 'express'; import { closePage, getPage } from '../model/browser'; import { waitForTimeout } from '../utils'; import fs from 'fs'; -import fetch from 'node-fetch'; +import fetch, { BodyInit, HeaderInit } from 'node-fetch'; -import * as actions from '../actions/get'; +import { actions } from '../actions'; +import { MouseClickOptions, MouseMoveOptions, MouseOptions, Point, Protocol } from 'puppeteer'; type VisitRequestQuery = { action: "visit", url: string; } export const visit = async (req: Request, res: Response) => { @@ -17,11 +18,81 @@ export const visit = async (req: Request, res: Response) => { await page.goto(url, { waitUntil: 'networkidle2' }); await waitForTimeout(2000); - const mapObject = await page.evaluate(() => { - // @ts-ignore - return parseDocument(document) + return res.status(200).json({ + success: true, + message: "Page visited" + }); + } catch (err) { + if (err instanceof Error) { + return res.status(400).json({ + error: true, + name: err.name, + message: err.message + }) + } + } +} + +type MouseMethods = { + move?: { x: number, y: number, options?: MouseMoveOptions }, + down?: { options: MouseOptions }, + up?: { options: MouseOptions }, + click?: { x: number, y: number, options?: MouseClickOptions }, + wheel?: { deltaX: number, deltaY: number }, + drag?: { start: Point, target: Point }, + dragAndDrop?: { source: Point, target: Point, options?: { delay: number } } +} +type MouseRequestQuery = { action: "mouse" } +export const mouse = async (req: Request, res: Response) => { + try { + const page = await getPage(); + + const mouseEvent = req.body as MouseMethods; + if (!mouseEvent) return res.status(400).send('Missing method parameter'); + + let eventResponse = []; + + if (mouseEvent.move) { + console.log("[MOVE]:", JSON.stringify(mouseEvent.move)) + await page.mouse.move(mouseEvent.move.x, mouseEvent.move.y, mouseEvent.move.options); + eventResponse.push({ message: "Mouse move triggered", data: mouseEvent.move }) + } + if (mouseEvent.down) { + console.log("[DOWN]:", JSON.stringify(mouseEvent.down)) + await page.mouse.down(mouseEvent.down.options); + eventResponse.push({ message: "Mouse down triggered", data: mouseEvent.down }) + } + if (mouseEvent.up) { + console.log("[UP]:", JSON.stringify(mouseEvent.up)) + await page.mouse.up(mouseEvent.up.options); + eventResponse.push({ message: "Mouse up triggered", data: mouseEvent.up }) + } + if (mouseEvent.click) { + console.log("[CLICK]:", JSON.stringify(mouseEvent.click)) + await page.mouse.click(mouseEvent.click.x, mouseEvent.click.y, mouseEvent.click.options); + eventResponse.push({ message: "Mouse click triggered", data: mouseEvent.click }) + } + if (mouseEvent.wheel) { + console.log("[WHEEL]:", JSON.stringify(mouseEvent.wheel)) + await page.mouse.wheel(mouseEvent.wheel); + eventResponse.push({ message: "Mouse wheel triggered", data: mouseEvent.wheel }) + } + if (mouseEvent.drag) { + console.log("[DRAG]:", JSON.stringify(mouseEvent.drag)) + await page.mouse.drag(mouseEvent.drag.start, mouseEvent.drag.target); + eventResponse.push({ message: "Mouse drag triggered", data: mouseEvent.drag }) + } + if (mouseEvent.dragAndDrop) { + console.log("[DRAG'N'DROP:]", JSON.stringify(mouseEvent.dragAndDrop)) + await page.mouse.dragAndDrop(mouseEvent.dragAndDrop.source, mouseEvent.dragAndDrop.target, mouseEvent.dragAndDrop.options); + eventResponse.push({ message: "Mouse dragAndDrop triggered", data: mouseEvent.dragAndDrop }) + } + + return res.status(200).json({ + success: true, + message: "Mouse event triggered", + response: eventResponse }); - return res.status(200).json(mapObject); } catch (err) { if (err instanceof Error) { return res.status(400).json({ @@ -35,7 +106,7 @@ export const visit = async (req: Request, res: Response) => { type ClickRequestQuery = { action: "click", selector: string, xpath?: "true" | "false" } export const click = async (req: Request, res: Response) => { - try{ + try { const page = await getPage(); const { selector, xpath = "true" } = req.query as unknown as ClickRequestQuery; @@ -45,19 +116,90 @@ export const click = async (req: Request, res: Response) => { } let element; - if (xpath === "true") element = await page.waitForXPath(selector); - else element = await page.waitForSelector(selector); + if (xpath.toLowerCase() === "true") { [element] = await page.$x(selector); } + else { [element] = await page.$$(selector); } + if (!element) return res.status(400).send('Element not found'); + const isIntersecting = await element.isIntersectingViewport(); + if (!isIntersecting) await element.scrollIntoView(); await element.click(); - await waitForTimeout(2000); - const mapObject = await page.evaluate(() => { - // @ts-ignore - return parseDocument(document) + return res.status(200).json({ + success: true, + message: "Element clicked" }); + } catch (err) { + if (err instanceof Error) { + return res.status(400).json({ + error: true, + name: err.name, + message: err.message + }) + } + } +} + +type HoverRequestQuery = { action: "hover", selector: string, xpath?: "true" | "false" } +export const hover = async (req: Request, res: Response) => { + try { + const page = await getPage(); + const { selector, xpath = "true" } = req.query as unknown as ClickRequestQuery; - return res.status(200).json(mapObject); + if (!selector) { + res.status(400).send('Missing selector parameter'); + return; + } + + let element; + if (xpath.toLowerCase() === "true") { [element] = await page.$x(selector); } + else { [element] = await page.$$(selector); } + + if (!element) return res.status(400).send('Element not found'); + + const isIntersecting = await element.isIntersectingViewport(); + if (!isIntersecting) await element.scrollIntoView(); + await element.hover(); + + + return res.status(200).json({ + success: true, + message: "Element hovered" + }); + } catch (err) { + if (err instanceof Error) { + return res.status(400).json({ + error: true, + name: err.name, + message: err.message + }) + } + } +} + +type SelectRequestQuery = { action: "select", selector: string, option: string } +export const select = async (req: Request, res: Response) => { + try { + const page = await getPage(); + const { selector, option } = req.query as unknown as SelectRequestQuery; + + if (!selector) { + res.status(400).send('Missing selector parameter'); + return; + } + + const element = await page.$(selector); + if (!element) return res.status(400).send('Element not found'); + + const isIntersecting = await element.isIntersectingViewport(); + if (!isIntersecting) await element.scrollIntoView(); + + await page.select(selector, option); + + return res.status(200).json({ + success: true, + message: "Dropdown option selected" + }); } catch (err) { if (err instanceof Error) { return res.status(400).json({ @@ -75,13 +217,6 @@ export const type = async (req: Request, res: Response) => { const page = await getPage(); const { selector, text, xpath = "true", pressEnter = "true" } = req.query as unknown as TypeRequestQuery; - console.log({ - selector, - text, - xpath, - pressEnter - }) - if (!selector) { res.status(400).send('Missing selector parameter'); return; @@ -93,20 +228,18 @@ export const type = async (req: Request, res: Response) => { } let element; - if (xpath === "true") { element = await page.waitForXPath(selector); } + if (xpath.toLowerCase() === "true") { element = await page.waitForXPath(selector); } else { element = await page.waitForSelector(selector); } if (!element) { return res.status(400).send('Element not found'); } await element.type(text, { delay: 100 }); - if (pressEnter === "true") await page.keyboard.press('Enter'); - await waitForTimeout(2000); + if (pressEnter?.toLowerCase() === "true") await page.keyboard.press('Enter'); - const mapObject = await page.evaluate(() => { - // @ts-ignore - return parseDocument(document) + return res.status(200).json({ + success: true, + message: "Text typed.", }); - return res.status(200).json(mapObject); } catch (err) { if (err instanceof Error) { return res.status(400).json({ @@ -127,14 +260,17 @@ export const wait = async (req: Request, res: Response) => { return; } - await waitForTimeout(2000); - return res.status(200).send('OK'); + await waitForTimeout(Number(time)); + return res.status(200).json({ + success: true, + message: "Waited for " + time + "ms" + }); } type ObserveRequestQuery = { action: "observe", selector: string, xpath?: "true" | "false" } export const observe = async (req: Request, res: Response) => { try { - const { selector, xpath } = req.query as unknown as ObserveRequestQuery; + const { selector, xpath = "true" } = req.query as unknown as ObserveRequestQuery; const page = await getPage(); if (!selector) { @@ -143,15 +279,19 @@ export const observe = async (req: Request, res: Response) => { } let element; - if (xpath === "true") element = await page.waitForXPath(selector); + if (xpath.toLowerCase() === "true") element = await page.waitForXPath(selector); else element = await page.waitForSelector(selector); if (!element) return res.status(400).send('Element not found'); return res.status(200).json({ - selector: selector, - exists: true, - clickable: await element.isIntersectingViewport(), - visible: await element.isIntersectingViewport() + success: true, + message: "Element found", + data: { + selector: selector, + exists: true, + clickable: await element.isIntersectingViewport(), + visible: await element.isIntersectingViewport() + } }) } catch (err) { if (err instanceof Error) { @@ -163,6 +303,7 @@ export const observe = async (req: Request, res: Response) => { } } } + type RouterRequestQuery = { action: "router", payload: "back" | "forward" | "reload"; } export const router = async (req: Request, res: Response) => { try { @@ -180,11 +321,10 @@ export const router = async (req: Request, res: Response) => { if (payload === 'forward') await page.goForward(); if (payload === 'reload') await page.reload(); - const mapObject = await page.evaluate(() => { - // @ts-ignore - return parseDocument(document) + return res.status(200).json({ + success: true, + message: "Page router action taken: " + payload }); - return res.status(200).json(mapObject); } catch (err) { if (err instanceof Error) { return res.status(400).json({ @@ -196,6 +336,50 @@ export const router = async (req: Request, res: Response) => { } } +type EvaluateRequestQuery = { action: "evaluate", code: string } +export const evaluate = async (req: Request, res: Response) => { + try { + const { code } = req.body as EvaluateRequestQuery; + const page = await getPage(); + + if (!code) { + res.status(400).send('Missing expression parameter'); + return; + } + + const evaluateResponse = await page.evaluate(code); + if (!evaluateResponse) return res.status(400).json({ + success: false, + message: "Expression failed", + response: evaluateResponse + }); + + return res.status(200).json({ + success: true, + message: "Evaluated expression", + response: evaluateResponse + }); + } catch (err) { + if (err instanceof Error) { + console.log(err) + return res.status(400).json({ + error: true, + title: err.name, + message: err.message + }) + } + } +} + +type ExitRequestQuery = { action: "exit" } +export const exit = async (req: Request, res: Response) => { + await closePage(); + return res.status(200).json({ + success: true, + message: "Browser closed" + }); +} + type SaveTaskRequestQuery = { action: "save" } export const savetask = async (req: Request, res: Response) => { const { name, tasks } = req.body; @@ -209,13 +393,10 @@ export const savetask = async (req: Request, res: Response) => { const macro = { name: name, tasks: tasks, timestamp: timestamp } fs.writeFileSync(`./tasks/${name}_${timestamp}.json`, JSON.stringify(macro, null, 2)); - return res.status(200).send('OK'); -} - -type ExitRequestQuery = { action: "exit" } -export const exit = async (req: Request, res: Response) => { - await closePage(); - return res.status(200).send('OK'); + return res.status(200).json({ + success: true, + message: "Task saved" + }); } type MacroTaskType = VisitRequestQuery | ClickRequestQuery | TypeRequestQuery | WaitRequestQuery | ObserveRequestQuery | RouterRequestQuery; @@ -230,14 +411,42 @@ export const replay = async (req: Request, res: Response) => { const macro = JSON.parse(fs.readFileSync(`./tasks/${name}.json`, 'utf8')) as { name: string, tasks: MacroTaskType[], timestamp: number } + let steps = []; for (let task of macro.tasks) { const { action, ...params } = task; - const searchParams = new URLSearchParams(params as any) - const url = new URL("http://localhost:8008" + actions[action].path + "?" + searchParams); - console.log("[TASK]", action, url.toString()) - await fetch(url, { method: actions[action].method }).then(res => res.text()) - await waitForTimeout(2000); + const method = actions[action].method; + + let url: URL | null = null; + let body: BodyInit | undefined = undefined; + let headers: HeaderInit | undefined = undefined; + + if (method.toUpperCase() === "GET") { + const searchParams = new URLSearchParams(params as any) + url = new URL("http://localhost:8008" + actions[action].path + "?" + searchParams); + } + + if (method.toUpperCase() === "POST") { + url = new URL("http://localhost:8008" + actions[action].path); + body = JSON.stringify(params); + headers = { 'Content-Type': 'application/json' }; + } + + if (!url) continue; + try { + const response = await fetch(url, { method, headers, body }).then(res => { + if (res.headers.get("content-type")?.includes("application/json")) return res.json(); + return res.text(); + }) + steps.push({ action, response }); + } catch (err) { + console.log(err) + steps.push({ action, response: err }); + } } - return res.status(200).send("OK"); + return res.status(200).json({ + success: true, + message: "Task replayed", + steps + }); } \ No newline at end of file diff --git a/index.ts b/index.ts index 1bf5b4c..0211a40 100644 --- a/index.ts +++ b/index.ts @@ -1,11 +1,11 @@ -import express, { NextFunction, Request, Response } from 'express'; +import express from 'express'; import morgan from 'morgan' import actions from './actions'; +import dotenv from 'dotenv'; +dotenv.config(); const app = express(); - app.use(morgan('combined')); -app.use(express.json()); app.use((req, res, next) => { res.setHeader('Access-Control-Allow-Origin', 'https://chat.openai.com'); @@ -15,9 +15,8 @@ app.use((req, res, next) => { next(); }); - actions.get.forEach((action) => app.get(action.path, action.controller)) -actions.post.forEach((action) => app.post(action.path, action.controller)) +actions.post.forEach((action) => app.post(action.path, ...(action.middlewares || []), action.controller)) app.use(express.static('static')); diff --git a/model/browser.ts b/model/browser.ts index e946521..a10deef 100644 --- a/model/browser.ts +++ b/model/browser.ts @@ -5,15 +5,9 @@ let browser: Browser | null; let page: Page | null; export const launchPage = async () => { - browser = await puppeteer.launch({ - headless: false, - args: [ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-dev-shm-usage' - ] - }); + browser = await puppeteer.connect({ browserWSEndpoint: process.env.BROWSER_WS_ENDPOINT }) page = await browser.newPage(); + page.setViewport({ width: 1680, height: 1050 }); const preloadFile = fs.readFileSync('./scripts/index.js', 'utf8'); await page.evaluateOnNewDocument(preloadFile); @@ -22,15 +16,14 @@ export const launchPage = async () => { } export const getPage = async () => { - if (!page) return launchPage(); + if (!page) { + return launchPage(); + } return page; } export const closePage = async () => { await page?.close(); - await browser?.close(); - page = null; - browser = null; } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 239e13d..4d13317 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,8 @@ "license": "ISC", "dependencies": { "@types/node-fetch": "^2.6.5", + "concurrently": "^8.2.1", + "dotenv": "^16.3.1", "express": "^4.18.2", "morgan": "^1.10.0", "node-fetch": "^2.7.0", @@ -52,6 +54,17 @@ "node": ">=6.9.0" } }, + "node_modules/@babel/runtime": { + "version": "7.22.15", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.22.15.tgz", + "integrity": "sha512-T0O+aa+4w0u06iNmapipJXMV4HoUir03hpx3/YqXXhu9xim3w+dVphjFWl1OH8NbZHw5Lbm9k45drDkgq2VNNA==", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@puppeteer/browsers": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-1.4.0.tgz", @@ -275,6 +288,11 @@ "node": ">=4" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -478,6 +496,138 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/concurrently": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-8.2.1.tgz", + "integrity": "sha512-nVraf3aXOpIcNud5pB9M82p1tynmZkrSGQ1p6X/VY8cJ+2LMVqAgXsJxYYefACSHbTYlm92O1xuhdGTjwoEvbQ==", + "dependencies": { + "chalk": "^4.1.2", + "date-fns": "^2.30.0", + "lodash": "^4.17.21", + "rxjs": "^7.8.1", + "shell-quote": "^1.8.1", + "spawn-command": "0.0.2", + "supports-color": "^8.1.1", + "tree-kill": "^1.2.2", + "yargs": "^17.7.2" + }, + "bin": { + "conc": "dist/bin/concurrently.js", + "concurrently": "dist/bin/concurrently.js" + }, + "engines": { + "node": "^14.13.0 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/open-cli-tools/concurrently?sponsor=1" + } + }, + "node_modules/concurrently/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/concurrently/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/concurrently/node_modules/chalk/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/concurrently/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/concurrently/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/concurrently/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/concurrently/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/concurrently/node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -543,6 +693,21 @@ "node": ">= 14" } }, + "node_modules/date-fns": { + "version": "2.30.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", + "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", + "dependencies": { + "@babel/runtime": "^7.21.0" + }, + "engines": { + "node": ">=0.11" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/date-fns" + } + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -578,6 +743,14 @@ "node": ">= 14" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -600,6 +773,17 @@ "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1120988.tgz", "integrity": "sha512-39fCpE3Z78IaIPChJsP6Lhmkbf4dWXOmzLk/KFTdRkNk/0JymRIfUynDVRndV9HoDz8PyalK1UH21ST/ivwW5Q==" }, + "node_modules/dotenv": { + "version": "16.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", + "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/motdotla/dotenv?sponsor=1" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -828,6 +1012,19 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -1124,6 +1321,11 @@ "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==" }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, "node_modules/lru-cache": { "version": "7.18.3", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", @@ -1575,6 +1777,11 @@ "node": ">= 6" } }, + "node_modules/regenerator-runtime": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz", + "integrity": "sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA==" + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -1591,6 +1798,14 @@ "node": ">=4" } }, + "node_modules/rxjs": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz", + "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==", + "dependencies": { + "tslib": "^2.1.0" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1675,6 +1890,14 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, + "node_modules/shell-quote": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz", + "integrity": "sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", @@ -1737,6 +1960,11 @@ "node": ">=0.10.0" } }, + "node_modules/spawn-command": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2.tgz", + "integrity": "sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -1832,6 +2060,14 @@ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, + "node_modules/tree-kill": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", + "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", + "bin": { + "tree-kill": "cli.js" + } + }, "node_modules/tslib": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.2.tgz", diff --git a/package.json b/package.json index 6e2e229..5537336 100644 --- a/package.json +++ b/package.json @@ -5,12 +5,16 @@ "main": "index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", - "start": "npx nodemon" + "start:express": "npx nodemon", + "start:chrome": "sh runchrome.sh", + "start": "concurrently npm:start:express npm:start:chrome" }, "author": "sametcodes", "license": "ISC", "dependencies": { "@types/node-fetch": "^2.6.5", + "concurrently": "^8.2.1", + "dotenv": "^16.3.1", "express": "^4.18.2", "morgan": "^1.10.0", "node-fetch": "^2.7.0", diff --git a/runchrome.sh b/runchrome.sh new file mode 100755 index 0000000..0b45812 --- /dev/null +++ b/runchrome.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +pkill -f "Google Chrome" +/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome https://chat.openai.com --remote-debugging-port=9222 --no-first-run --no-default-browser-check > output.log 2>&1 & + +while true; do + if grep -q 'DevTools listening on' output.log; then + WS_ENDPOINT=$(grep 'DevTools listening on' output.log | awk '{print $4}') + echo "BROWSER_WS_ENDPOINT=\"$WS_ENDPOINT\"" > .env + rm output.log + break + fi + sleep 1 +done diff --git a/static/openapi.yaml b/static/openapi.yaml index 8678830..16bbd2a 100644 --- a/static/openapi.yaml +++ b/static/openapi.yaml @@ -15,7 +15,6 @@ paths: name: url schema: type: string - enum: [back, forward, reload] required: true description: The url of the website to visit. responses: @@ -23,6 +22,133 @@ paths: description: OK "404": description: Not found + /mouse: + post: + operationId: mouse + summary: > + Handle mouse. + Example request body: + { "move": { "x": 100, "y": 100, "options": { "steps": 10 } } } + Another example: + { "dragAndDrop": { "source": { "x": 100, "y": 100 }, "target": { "x": 200, "y": 200 } } } + requestBody: + content: + application/json: + schema: + type: object + properties: + move: + type: object + description: > + The mousemove event is sent to an element when the mouse pointer moves inside the element. Use wheel for scrolling. + properties: + x: + type: number + y: + type: number + options: + type: object + properties: + steps: + type: number + down: + type: object + description: > + The mousedown event is sent to an element when the mouse pointer is over the element, and the mouse button is pressed. Any HTML element can receive this event. + properties: + options: + type: object + properties: + button: + type: string + enum: [back, forward, left, right, middle] + clickCount: + type: number + up: + type: object + description: > + The mouseup event is sent to an element when the mouse pointer is over the element, and the mouse button is released. Any HTML element can receive this event. + properties: + options: + type: object + properties: + button: + type: string + enum: [back, forward, left, right, middle] + clickCount: + type: number + click: + type: object + description: > + The click event is sent to an element when the mouse pointer is over the element, and the mouse button is pressed and released. Any HTML element can receive this event. + properties: + x: + type: number + y: + type: number + options: + type: object + properties: + delay: + type: number + count: + type: number + wheel: + type: object + description: > + The wheel event is sent to an element when the mouse wheel is rolled up or down over the element. This can be used for scrolling. + properties: + deltaX: + type: number + deltaY: + type: number + drag: + type: object + description: > + The drag event is sent to an element when the mouse pointer is moved while over the element during a drag operation. Any HTML element can receive this event. + properties: + start: + type: object + properties: + x: + type: number + y: + type: number + target: + type: object + properties: + x: + type: number + y: + type: number + dragAndDrop: + type: object + description: > + The drag and drop event is sent to an element when the mouse pointer is moved while over the element during a drag operation. Any HTML element can receive this event. + properties: + source: + type: object + properties: + x: + type: number + description: Use 1 decimal places for more accuracy. + y: + type: number + description: Use 1 decimal places for more accuracy. + target: + type: object + properties: + x: + type: number + description: Use 1 decimal places for more accuracy. + y: + type: number + description: Use 1 decimal places for more accuracy. + responses: + "200": + description: OK + "404": + description: Not found /click: get: operationId: click @@ -37,7 +163,32 @@ paths: - in: query name: xpath schema: - type: boolean + type: string + enum: [true, false] + default: true + required: true + description: Whether the selector is a xpath selector or not. Default is true. + responses: + "200": + description: OK + "404": + description: Not found + /hover: + get: + operationId: hover + summary: Hover on a node. + parameters: + - in: query + name: selector + schema: + type: string + required: true + description: The selector to hover on + - in: query + name: xpath + schema: + type: string + enum: [true, false] default: true required: true description: Whether the selector is a xpath selector or not. Default is true. @@ -46,17 +197,44 @@ paths: description: OK "404": description: Not found + /select: + get: + operationId: select + summary: Select an option of a dropdown. + parameters: + - in: query + name: selector + schema: + type: string + required: true + description: The selector to select + - in: query + name: option + schema: + type: string + description: The value option to select from the dropdown + responses: + "200": + description: OK + "404": + description: Not found /observe: get: operationId: observe - summary: Observe page body with XPath selector + summary: Observe page body with XPath selector. Returns if the selector is existing, clickable, or visible. parameters: - in: query name: xpath schema: - type: boolean + type: string + enum: [true, false] default: true description: Whether the selector is a xpath selector or not. Default is true. + - in: query + name: selector + schema: + type: string + required: true responses: "200": description: OK @@ -82,7 +260,8 @@ paths: - in: query name: xpath schema: - type: boolean + type: string + enum: [true, false] default: true required: true description: Whether the selector is a xpath selector or not. Default is true. @@ -115,6 +294,28 @@ paths: description: OK "404": description: Not found + /evaluate: + post: + operationId: evaluate + summary: Consider this is the console of the browser. + requestBody: + description: The JavaScript expression/code. + required: true + content: + application/json: + schema: + type: object + properties: + code: + type: string + description: The JavaScript expression to evaluate. + required: + - code + responses: + "200": + description: OK + "404": + description: Not found /wait: get: operationId: wait @@ -123,7 +324,7 @@ paths: - in: query name: time schema: - type: integer + type: number required: true description: Time to wait in seconds responses: @@ -144,20 +345,12 @@ paths: post: operationId: savetask summary: > - Saves the latest task to a file as JSON. - - Here is an example request body for the task format: + Saves the latest task to a file as JSON. Only use this if user says "save task" in the prompt. { "name": "visit_google", "tasks": [ - { - "action": "visit", - "url": "https://www.google.com" - }, - { - "action": "click", - "selector": "input[name='btnK']" - } + { "action": "visit", "url": "https://www.google.com" }, + { "action": "click", "selector": "input[name='btnK']" } ] } requestBody: @@ -176,13 +369,6 @@ paths: properties: action: type: string - url: - type: string - format: uri - selector: - type: string - text: - type: string description: The task to save. responses: "200":