diff --git a/README.md b/README.md index ccbfd38..db47b80 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,48 @@ Use CSS selectors to grab lists of stuff from various websites and list them altogether on a single page. + + +## API + +Accepts a `GET` request like this: + +``` +https://harvestor.particlesystem.com/api?url=https://example.com&itemSelector=.item +``` + +> [!NOTE] +> For readability the params in the example above have not been correctly encoded. + +And returns a JSON response like this: + +```json +{ + "items": [ + { + "title": "Item 1", + "link": "https://example.com/item1" + }, + { + "title": "Item 2", + "link": "https://example.com/item2" + } + ], + "debug": { + "itemsFound": 2, + "itemsAfterFilter": 2, + "firstTitle": "Item 1", + "firstLink": "https://example.com/item1" + } +} +``` + +### Params + +| Name | Description | +|------|-------------| +| `itemFilter` | A string an item's text content must contain to be included in the result. | +| `itemSelector` | **Required** A CSS selector used to identify the items on the page. | +| `linkSelector` | A CSS selector used to identify an `` element within each item that has a href attribute to be used as a link. | +| `titleSelector` | A CSS selector used to identify an element containing the title of an item. If omitted, the title will be all the text content of the item. | +| `url` | **Required** A URL of a page to scrape. | diff --git a/src/app/List.tsx b/src/app/List.tsx index 59c2167..765d12d 100644 --- a/src/app/List.tsx +++ b/src/app/List.tsx @@ -12,6 +12,8 @@ const List = ({ name, url, itemSelector, itemFilter, titleSelector, linkSelector const [requestUrl, setRequestUrl] = useState(null); useEffect(() => { + let timeoutId: NodeJS.Timeout; + if (window) { const result = new URL("/api", window.location.href); @@ -21,8 +23,14 @@ const List = ({ name, url, itemSelector, itemFilter, titleSelector, linkSelector result.searchParams.set("url", url); if (linkSelector) result.searchParams.set("linkSelector", linkSelector); - setRequestUrl(result.href); + timeoutId = setTimeout(() => { + setRequestUrl(result.href); + }, 500); } + + return () => { + if (timeoutId) clearTimeout(timeoutId); + }; }, [itemFilter, itemSelector, titleSelector, url, linkSelector]); const { data, error, isLoading, isValidating } = useSWR(requestUrl, fetcher); @@ -45,11 +53,10 @@ const List = ({ name, url, itemSelector, itemFilter, titleSelector, linkSelector ) : ( )} diff --git a/src/app/ListForm.tsx b/src/app/ListForm.tsx index b3d727b..6a489db 100644 --- a/src/app/ListForm.tsx +++ b/src/app/ListForm.tsx @@ -27,7 +27,7 @@ const ListForm = ({ list: initialList, onSubmit }: Props) => { setList({ ...list, name: event.target.value }); }} size={100} - placeholder="Example" + placeholder="Reddit" />
@@ -41,7 +41,7 @@ const ListForm = ({ list: initialList, onSubmit }: Props) => { setList({ ...list, url: event.target.value }); }} size={100} - placeholder="https://supabase.com/careers" + placeholder="https://www.reddit.com/" />
@@ -55,7 +55,7 @@ const ListForm = ({ list: initialList, onSubmit }: Props) => { setList({ ...list, itemSelector: event.target.value }); }} size={100} - placeholder="a[href^=https\:\/\/jobs\.ashbyhq\.com]" + placeholder="article" />
@@ -63,13 +63,12 @@ const ListForm = ({ list: initialList, onSubmit }: Props) => { { setList({ ...list, titleSelector: event.target.value }); }} size={100} - placeholder="h4" + placeholder="a" />
@@ -82,6 +81,7 @@ const ListForm = ({ list: initialList, onSubmit }: Props) => { setList({ ...list, itemFilter: event.target.value }); }} size={100} + placeholder="example" />
@@ -94,13 +94,11 @@ const ListForm = ({ list: initialList, onSubmit }: Props) => { setList({ ...list, linkSelector: event.target.value }); }} size={100} + placeholder="a" />
- {list.name.trim() === "" || - list.url.trim() === "" || - list.itemSelector.trim() === "" || - (list.titleSelector ?? "").trim() === "" ? ( -

Required fields: Name, URL, Item selector and Title selector

+ {list.name.trim() === "" || list.url.trim() === "" || list.itemSelector.trim() === "" ? ( +

Required fields: Name, URL and Item selector

) : ( { const linkSelector = requestUrl?.searchParams.get("linkSelector"); const url = requestUrl?.searchParams.get("url"); - if (itemSelector === null || itemSelector === undefined) { - return new Response("Missing itemSelector query param", { status: 400 }); - } - if (url === null || url === undefined) { return new Response("Missing URL query param", { status: 400 }); } + if (itemSelector === null || itemSelector === undefined || itemSelector.trim() === "") { + return new Response("Missing itemSelector query param", { status: 400 }); + } + const dom = await JSDOM.fromURL(url); - const items = dom.window.document.querySelectorAll(itemSelector); + const items = selectAll(dom.window.document, itemSelector); const result: ListResult = { items: [], debug: { - itemsFound: items.length, - itemsAfterFilter: 0, - firstTitle: undefined, firstLink: undefined, + firstTitle: undefined, + itemsAfterFilter: 0, + itemsFound: items.length, }, }; @@ -36,35 +36,53 @@ export const GET = async (request: Request) => { if ( itemFilter && itemFilter.trim() !== "" && - (item.textContent === null || item.textContent.includes(itemFilter) === false) + (item.textContent === null || + item.textContent.toLocaleLowerCase().includes(itemFilter) === false) ) { continue; } + result.debug.itemsAfterFilter += 1; + const title = titleSelector && titleSelector.trim() !== "" - ? item.querySelector(titleSelector)?.textContent + ? select(item, titleSelector)?.textContent : item.textContent; - if (result.debug.firstTitle === undefined && title) { + + if (result.debug.firstTitle === undefined) { result.debug.firstTitle = title; } - if (title) { - const link = - linkSelector && linkSelector.trim() !== "" ? item.querySelector(linkSelector) : item; + + if (title && result.items.every((item) => item.title !== title)) { + const link = linkSelector && linkSelector.trim() !== "" ? select(item, linkSelector) : item; const itemUrl = link?.tagName === "A" ? (link.getAttribute("href") ?? undefined) : undefined; const parsedUrl = itemUrl ? URL.parse(itemUrl, url)?.href : undefined; - if (parsedUrl === undefined) { - continue; - } - if (result.debug.firstLink === undefined && parsedUrl) { + + if (result.debug.firstLink === undefined) { result.debug.firstLink = parsedUrl; } - if (result.items.some((r) => r.url === parsedUrl)) { - continue; - } + result.items.push({ title, url: parsedUrl }); } } return Response.json(result); }; + +const select = (root: Document | Element, selector: string) => { + try { + return root.querySelector(selector); + } catch (error) { + console.error(error); + return null; + } +}; + +const selectAll = (root: Document | Element, selector: string) => { + try { + return root.querySelectorAll(selector); + } catch (error) { + console.error(error); + return []; + } +};