diff --git a/README.md b/README.md index 0317058..6eaf499 100644 --- a/README.md +++ b/README.md @@ -1,56 +1,99 @@ # Harvester -Use CSS selectors to grab lists of stuff from various websites and list them altogether on a single -page. +Aggregate data scrapped from web pages. ## API +### Params + +Required: + +- `url` — A URL of a page to scrape. +- `itemSelector` — A CSS selector used to identify the items on the page. + +Optional: + +- `debug` — A boolean indicating whether to include debug information in the response. +- `exclude` — A comma-separated list of words an item's text content must contain to be excluded in + the result. +- `include` — A comma-separated list of words an item's text content must contain to be included in + the result. +- `linkSelector` — A CSS selector used to identify an `` element within each item that has a href + attribute to be used as a link. If the item itself is an `` element, this param can be omitted. +- `titleSelector` — A CSS selector used to identify an element containing the title of an item. If + omitted, the title will be all the text content of the item. + +### Examples + +> [!NOTE] +> For readability the params in the following examples have not been correctly encoded. + +#### Basic + Request: ``` -GET https://harvestor.particlesystem.com/api?url=https://example.com&itemSelector=.item +GET https://harvestor.particlesystem.com/source?url=https://example.com&itemSelector=.item ``` -> [!NOTE] -> For readability the params in the example above have not been correctly encoded. - Response: ```json { + "fetchedAt": "2025-09-14T04:00:04Z", "items": [ { + "firstSeen": "2025-09-13T01:23:21Z", "title": "Item 1", - "link": "https://example.com/item1" + "url": "https://example.com/item1" }, { + "firstSeen": "2025-09-12T03:34:08Z", "title": "Item 2", - "link": "https://example.com/item2" + "url": "https://example.com/item2" } ], - "debug": { - "itemsFound": 2, - "itemsAfterFilter": 2, - "firstTitle": "Item 1", - "firstLink": "https://example.com/item1" - } + "ok": true } ``` -### Params +#### Debug Info -Required: +Request: -- `url` — A URL of a page to scrape. -- `itemSelector` — A CSS selector used to identify the items on the page. +``` +GET https://harvestor.particlesystem.com/source?url=https://example.com&itemSelector=.item&debug=true +``` -Optional: +> [!NOTE] +> For readability the params in the example above have not been correctly encoded. -- `exclude` — A comma-separated list of words an item's text content must contain to be excluded in - the result. -- `include` — A comma-separated list of words an item's text content must contain to be included in - the result. -- `linkSelector` — A CSS selector used to identify an `` element within each item that has a href - attribute to be used as a link. If the item itself is an `` element, this param can be omitted. -- `titleSelector` — A CSS selector used to identify an element containing the title of an item. If - omitted, the title will be all the text content of the item. +Response: + +```json +{ + "debug": { + "htmlSamples": [ + "
Item 1
", + "
Item 2
" + ], + "itemsMatched": 2, + "itemsRemovedByExcludeFilter": 3, + "itemsRemovedByIncludeFilter": 1 + }, + "fetchedAt": "2025-09-14T04:00:04Z", + "items": [ + { + "firstSeen": "2025-09-13T01:23:21Z", + "title": "Item 1", + "url": "https://example.com/item1" + }, + { + "firstSeen": "2025-09-12T03:34:08Z", + "title": "Item 2", + "url": "https://example.com/item2" + } + ], + "ok": true +} +``` diff --git a/package.json b/package.json index bdcfea6..fdbaf96 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "dependencies": { - "@colonydb/anthill": "0.2.0-33", + "@colonydb/anthill": "0.2.0-36", "@vercel/analytics": "^1.5.0", "jsdom": "^26.0.0", "next": "15.2.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7d6569c..61241ff 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@colonydb/anthill': - specifier: 0.2.0-33 - version: 0.2.0-33(@types/react@19.1.9)(react-dom@19.1.1(react@19.1.1))(react@19.1.1)(valibot@1.1.0(typescript@5.9.2)) + specifier: 0.2.0-36 + version: 0.2.0-36(@types/react@19.1.9)(react-dom@19.1.1(react@19.1.1))(react@19.1.1)(valibot@1.1.0(typescript@5.9.2)) '@vercel/analytics': specifier: ^1.5.0 version: 1.5.0(next@15.2.1(react-dom@19.1.1(react@19.1.1))(react@19.1.1))(react@19.1.1) @@ -122,8 +122,8 @@ packages: cpu: [x64] os: [win32] - '@colonydb/anthill@0.2.0-33': - resolution: {integrity: sha512-9OW3Px5Hbk1QZKnYcTyMp3vUGuKNOsXiqBBrRNpHZfFheoYOH5slPjxr5rVyaxGL9B67zO9TXCoqTeacw6aLZQ==} + '@colonydb/anthill@0.2.0-36': + resolution: {integrity: sha512-7+q1rMBQXssy7nag5PNnzI9LdiV4dgo1u2cxkwx4U9LrRGeFp0OExbfbkfeGN2R9XO7/lWBwjk0hg+dJpexc9g==} peerDependencies: react: ^19.0.0 react-dom: ^19.0.0 @@ -506,6 +506,15 @@ packages: supports-color: optional: true + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + decimal.js@10.6.0: resolution: {integrity: sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==} @@ -1011,7 +1020,7 @@ snapshots: '@biomejs/cli-win32-x64@2.1.4': optional: true - '@colonydb/anthill@0.2.0-33(@types/react@19.1.9)(react-dom@19.1.1(react@19.1.1))(react@19.1.1)(valibot@1.1.0(typescript@5.9.2))': + '@colonydb/anthill@0.2.0-36(@types/react@19.1.9)(react-dom@19.1.1(react@19.1.1))(react@19.1.1)(valibot@1.1.0(typescript@5.9.2))': dependencies: '@haydn/universal': 0.0.4 clsx: 2.1.1 @@ -1300,6 +1309,10 @@ snapshots: dependencies: ms: 2.1.3 + debug@4.4.3: + dependencies: + ms: 2.1.3 + decimal.js@10.6.0: {} decode-named-character-reference@1.2.0: @@ -1637,7 +1650,7 @@ snapshots: micromark@4.0.2: dependencies: '@types/debug': 4.1.12 - debug: 4.4.1 + debug: 4.4.3 decode-named-character-reference: 1.2.0 devlop: 1.1.0 micromark-core-commonmark: 2.0.3 diff --git a/src/app/SourceForm.tsx b/src/app/SourceForm.tsx index 19d2e77..151640b 100644 --- a/src/app/SourceForm.tsx +++ b/src/app/SourceForm.tsx @@ -1,15 +1,14 @@ "use client"; -import { Action } from "@colonydb/anthill/Action"; +import type { FormAction } from "@colonydb/anthill"; import { Block } from "@colonydb/anthill/Block"; import { Button } from "@colonydb/anthill/Button"; import { Card } from "@colonydb/anthill/Card"; import { CardContent } from "@colonydb/anthill/CardContent"; +import { DialogHeader } from "@colonydb/anthill/DialogHeader"; import { Form } from "@colonydb/anthill/Form"; import { FormFooter } from "@colonydb/anthill/FormFooter"; -import { Header } from "@colonydb/anthill/Header"; import { Heading } from "@colonydb/anthill/Heading"; -import { Icon } from "@colonydb/anthill/Icon"; import { Inline } from "@colonydb/anthill/Inline"; import { RegularField } from "@colonydb/anthill/RegularField"; import { Section } from "@colonydb/anthill/Section"; @@ -18,11 +17,14 @@ import { Stack } from "@colonydb/anthill/Stack"; import { StringInput } from "@colonydb/anthill/StringInput"; import { TabBlock } from "@colonydb/anthill/TabBlock"; import { useForm } from "@colonydb/anthill/useForm"; +import { useCallback } from "react"; import * as v from "valibot"; import type { SourceConfig } from "@/index"; -import { sourceSchema } from "@/schemas/sourceSchema"; -import List from "./List"; -import Source from "./Source"; +import { sourceConfigSchema } from "@/schemas/sourceConfigSchema"; +import SourceHtmlSamples from "./SourceHtmlSamples"; +import SourceItems from "./SourceItems"; +import SourceOverview from "./SourceOverview"; +import SourceStats from "./SourceStats"; type Props = { id: string; @@ -33,162 +35,153 @@ type Props = { title: string; }; -export const SourceForm = ({ id, initialData, onCancel, onSubmit, onSuccess, title }: Props) => ( -
{ +export const SourceForm = ({ id, initialData, onCancel, onSubmit, onSuccess, title }: Props) => { + const action = useCallback>( + async ({ data }) => { onSubmit(data); return { data, ok: true, }; - }} - onSuccess={onSuccess} - id={id} - initialData={initialData} - schema={sourceSchema} - > - } - title="close" - onClick={() => { - onCancel(); - }} - /> - } - > - {title} - - } - footer={ - { - onCancel(); - }} - > - Cancel - - } - /> - } + }, + [onSubmit], + ); + return ( + - -
-
Configuration}> - - - - - - - - - - CSS selector identifying the items on the page to be listed. - - } - label="Item Selector" - name="itemSelector" - required - > - - - - CSS selector applied to each item to identify its title. If not - provided, the text of the entire item will be used. - - } - label="Title Selector" - name="titleSelector" - > - - - - CSS selector applied to each item to identify its link. If not - provided, the items itself will be used. - - } - label="Link Selector" - name="linkSelector" - > - - - - ), - }, - { - key: "filters", - label: "Filters", - content: ( - - - A comma-separated list of phrases to filter the items by. If an item - does not contain any of the phrases, it will be excluded from the - list. - - } - label="Include" - name="include" - > - - - - A comma-separated list of phrases to filter the items by. If an item - contains any of the phrases, it will be excluded from the list. - - } - label="Exclude" - name="exclude" - > - - - - ), - }, - ]} - seamless - /> - -
- -
- - - + + {title} + + } + footer={ + Cancel} + /> + } + > + +
+ + +
+ + + + ); +}; + +const Configuration = () => ( +
Configuration}> + + + + + + + + + + CSS selector identifying the items on the page to be listed. + + } + label="Item Selector" + name="itemSelector" + required + > + + + + CSS selector applied to each item to identify its title. If not provided, the + text of the entire item will be used. + + } + label="Title Selector" + name="titleSelector" + > + + + + CSS selector applied to each item to identify its link. If not provided, the + items itself will be used. + + } + label="Link Selector" + name="linkSelector" + > + + + + ), + }, + { + key: "filters", + label: "Filters", + content: ( + + + A comma-separated list of phrases to filter the items by. If an item does not + contain any of the phrases, it will be excluded from the list. + + } + label="Include" + name="include" + > + + + + A comma-separated list of phrases to filter the items by. If an item contains + any of the phrases, it will be excluded from the list. + + } + label="Exclude" + name="exclude" + > + + + + ), + }, + ]} + seamless + /> + +
); const Preview = () => { - const { data } = useForm(sourceSchema); + const { data } = useForm(sourceConfigSchema); return (
Preview}> { { key: "overview", label: "Overview", - content: v.is(sourceSchema, data) ? ( + content: v.is(sourceConfigSchema, data) ? ( - - - - + + ) : ( @@ -214,10 +205,26 @@ const Preview = () => { { key: "items", label: "Items", + content: v.is(sourceConfigSchema, data) ? ( + + + + ) : ( + + Not configured + + ), }, { - key: "raw", - label: "Raw Data", + key: "matched", + label: "Sample HTML", + content: v.is(sourceConfigSchema, data) ? ( + + ) : ( + + Not configured + + ), }, ]} seamless diff --git a/src/app/SourceHtmlSamples.tsx b/src/app/SourceHtmlSamples.tsx new file mode 100644 index 0000000..164eb5a --- /dev/null +++ b/src/app/SourceHtmlSamples.tsx @@ -0,0 +1,41 @@ +"use client"; + +import { Badge } from "@colonydb/anthill/Badge"; +import { CodeBlock } from "@colonydb/anthill/CodeBlock"; +import { Icon } from "@colonydb/anthill/Icon"; +import { Inline } from "@colonydb/anthill/Inline"; +import { Stack } from "@colonydb/anthill/Stack"; +import { useSource } from "@/hooks/useSource"; +import type { SourceConfig } from "@/index"; + +type Props = { + config: SourceConfig; + sampleSet: "excludeFilter" | "includeFilter" | "matched"; +}; + +const SourceHtmlSamples = ({ config, sampleSet }: Props) => { + const { data, error, isLoading } = useSource(config, true); + + return isLoading ? ( + + ) : error ? ( + + + + ) : data?.debug?.htmlSamples[sampleSet].length !== undefined && + data.debug.htmlSamples[sampleSet].length > 0 ? ( + + {data.debug.htmlSamples[sampleSet].map((sample, index) => ( + + {sample} + + ))} + + ) : ( + + None + + ); +}; + +export default SourceHtmlSamples; diff --git a/src/app/List.tsx b/src/app/SourceItems.tsx similarity index 78% rename from src/app/List.tsx rename to src/app/SourceItems.tsx index a580934..0659fb3 100644 --- a/src/app/List.tsx +++ b/src/app/SourceItems.tsx @@ -1,6 +1,7 @@ "use client"; import { Badge } from "@colonydb/anthill/Badge"; +import { Icon } from "@colonydb/anthill/Icon"; import { Inline } from "@colonydb/anthill/Inline"; import { Link } from "@colonydb/anthill/Link"; import { Stack } from "@colonydb/anthill/Stack"; @@ -13,18 +14,21 @@ import { sourceUrlFromConfig } from "@/utils/sourceUrlFromConfig"; import type { SourceConfig } from ".."; type Props = { + debug?: boolean; sources: Array; }; -const List = ({ sources }: Props) => { +const SourceItems = ({ debug = false, sources }: Props) => { const location = useLocation(); const sourceUrls = useMemo( - () => sources.map((source) => sourceUrlFromConfig(source, location)), - [location, sources], + () => sources.map((source) => sourceUrlFromConfig(source, location, debug)), + [debug, location, sources], ); - const { data, isLoading } = useSWRList(sourceUrls, sourceFetcher); + const { data, error, isLoading } = useSWRList(sourceUrls, sourceFetcher, { + shouldRetryOnError: false, + }); const items = data .flatMap((result, resultIndex) => @@ -47,10 +51,19 @@ const List = ({ sources }: Props) => { return isLoading ? (

Loading…

- ) : data.length === 0 ? ( - - No results - + ) : items.length === 0 ? ( + error ? ( + + + + {" "} + {error.errors?.join(", ") ?? error.message} + + ) : ( + + No results + + ) ) : ( {items.map(({ key, source, ...item }) => ( @@ -82,4 +95,4 @@ const List = ({ sources }: Props) => { ); }; -export default List; +export default SourceItems; diff --git a/src/app/Source.tsx b/src/app/SourceOverview.tsx similarity index 70% rename from src/app/Source.tsx rename to src/app/SourceOverview.tsx index 3450de0..d474dfa 100644 --- a/src/app/Source.tsx +++ b/src/app/SourceOverview.tsx @@ -5,25 +5,19 @@ import { Header } from "@colonydb/anthill/Header"; import { Icon } from "@colonydb/anthill/Icon"; import { Inline } from "@colonydb/anthill/Inline"; import { Link } from "@colonydb/anthill/Link"; -import { type ReactNode, useMemo } from "react"; -import useSWR from "swr/immutable"; +import type { ReactNode } from "react"; import { Temporal } from "temporal-polyfill"; -import { sourceFetcher } from "@/fetchers/sourceFetcher"; -import { useLocation } from "@/hooks/useLocation"; -import type { SourceConfig, SourceResult } from "@/index"; -import { sourceUrlFromConfig } from "@/utils/sourceUrlFromConfig"; +import { useSource } from "@/hooks/useSource"; +import type { SourceConfig } from "@/index"; type Props = { actions?: ReactNode; config: SourceConfig; + debug?: boolean; }; -const Source = ({ actions, config }: Props) => { - const location = useLocation(); - - const sourceUrl = useMemo(() => sourceUrlFromConfig(config, location), [config, location]); - - const { data, error, isLoading } = useSWR(sourceUrl, sourceFetcher); +const SourceOverview = ({ actions, config, debug = false }: Props) => { + const { data, error, isLoading } = useSource(config, debug); return (
{ ); }; -export default Source; +export default SourceOverview; diff --git a/src/app/SourceStats.tsx b/src/app/SourceStats.tsx new file mode 100644 index 0000000..95bdf20 --- /dev/null +++ b/src/app/SourceStats.tsx @@ -0,0 +1,39 @@ +"use client"; + +import { Badge } from "@colonydb/anthill/Badge"; +import { Block } from "@colonydb/anthill/Block"; +import { Icon } from "@colonydb/anthill/Icon"; +import { Inline } from "@colonydb/anthill/Inline"; +import { useSource } from "@/hooks/useSource"; +import type { SourceConfig } from "@/index"; + +type Props = { + config: SourceConfig; +}; + +const SourceStats = ({ config }: Props) => { + const { data, error, isLoading } = useSource(config, true); + + return isLoading ? ( + + ) : error ? ( + + + + {" "} + {error.message} + + ) : data?.debug !== undefined ? ( + <> + Items matched: {data.debug.itemsMatched} + Items omitted by "Include" filter: {data.debug.itemsRemovedByIncludeFilter} + Items omitted by "Exclude" filter: {data.debug.itemsRemovedByExcludeFilter} + + ) : ( + + None + + ); +}; + +export default SourceStats; diff --git a/src/app/api/route.ts b/src/app/api/route.ts deleted file mode 100644 index 9e9cd87..0000000 --- a/src/app/api/route.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { JSDOM, VirtualConsole } from "jsdom"; -import { after } from "next/server"; -import { createClient } from "redis"; -import { toTemporalInstant } from "temporal-polyfill"; -import type { SourceResult } from "@/index"; - -export const GET = async (request: Request) => { - const redis = process.env.REDIS_URL ? createClient({ url: process.env.REDIS_URL }) : null; - - if (redis) await redis.connect(); - - after(() => { - if (redis) redis.destroy(); - }); - - const requestUrl = URL.parse(request.url); - - const include = requestUrl?.searchParams.get("include"); - const exclude = requestUrl?.searchParams.get("exclude"); - const itemSelector = requestUrl?.searchParams.get("itemSelector"); - const titleSelector = requestUrl?.searchParams.get("titleSelector"); - const linkSelector = requestUrl?.searchParams.get("linkSelector"); - const url = requestUrl?.searchParams.get("url"); - - if (url === null || url === undefined) { - return new Response(null, { status: 400, statusText: "Missing URL query param" }); - } - - if (itemSelector === null || itemSelector === undefined || itemSelector.trim() === "") { - return new Response(null, { status: 400, statusText: "Missing itemSelector query param" }); - } - - const response = await fetch(url, { - next: { - revalidate: process.env.NODE_ENV === "development" ? 60 : 3600, - }, - }); - - if (response.ok === false) throw Error("Failed to fetch URL"); - - const sourceDateHeader = response.headers.get("date"); - - const sourceUpdatedInstant = toTemporalInstant.apply( - sourceDateHeader ? new Date(sourceDateHeader) : new Date(Date.now()), - ); - - const html = await response.text(); - - const dom = await createDOM(html, url); - - if (!dom) { - return new Response(null, { status: 500, statusText: "Unable to load URL" }); - } - - const items = selectAll(dom.window.document, itemSelector); - - const result: SourceResult = { - debug: { - firstLink: undefined, - firstTitle: undefined, - itemsAfterFilter: 0, - itemsFound: items.length, - }, - fetchedAt: sourceUpdatedInstant.toString(), - items: [], - }; - - for (const item of items ?? []) { - if (include && include.trim() !== "" && item.textContent !== null) { - const words = include.split(",").map((word) => word.trim().toLocaleLowerCase()); - if (words.every((word) => item.textContent.toLocaleLowerCase().includes(word) === false)) { - continue; - } - } - - if (exclude && exclude.trim() !== "" && item.textContent !== null) { - const words = exclude.split(",").map((word) => word.trim().toLocaleLowerCase()); - if (words.some((word) => item.textContent.toLocaleLowerCase().includes(word) === true)) { - continue; - } - } - - result.debug.itemsAfterFilter += 1; - - const title = - titleSelector && titleSelector.trim() !== "" - ? select(item, titleSelector)?.textContent - : item.textContent; - - if (result.debug.firstTitle === undefined) { - result.debug.firstTitle = title; - } - - if (title && result.items.every((item) => item.title !== title)) { - const link = linkSelector && linkSelector.trim() !== "" ? select(item, linkSelector) : item; - const itemUrl = link?.tagName === "A" ? (link.getAttribute("href") ?? undefined) : undefined; - const parsedUrl = itemUrl ? URL.parse(itemUrl, url)?.href : undefined; - - if (result.debug.firstLink === undefined) { - result.debug.firstLink = parsedUrl; - } - - result.items.push({ - firstSeen: sourceUpdatedInstant.toString(), - title, - url: parsedUrl, - }); - } - } - - if (redis) { - const keys = result.items.map( - ({ title }) => `${url}:${itemSelector}:${titleSelector ?? ""}:${title.slice(0, 100)}`, - ); - - if (keys.length > 0) { - const firstSeenValues = await redis.HMGET("first_seen", keys); - - for (let i = 0; i < firstSeenValues.length; i++) { - const value = firstSeenValues[i]; - const item = result.items[i]; - - if (item && value) item.firstSeen = value; - } - - await redis - .multi() - .HSET( - "first_seen", - Object.fromEntries( - keys.map((key, index) => [ - key, - firstSeenValues[index] ?? sourceUpdatedInstant.toString(), - ]), - ), - ) - .HEXPIRE("first_seen", keys, 60 * 60 * 24 * 28) - .exec(); - } - } - - return Response.json(result, { - headers: { - "Vercel-CDN-Cache-Control": "max-age=10", - }, - }); -}; - -const select = (root: Document | Element, selector: string) => { - try { - return root.querySelector(selector); - } catch (error) { - console.error(error); - return null; - } -}; - -const selectAll = (root: Document | Element, selector: string) => { - try { - return root.querySelectorAll(selector); - } catch (error) { - console.error(error); - return []; - } -}; - -const createDOM = async (html: string, url: string) => { - const virtualConsole = new VirtualConsole(); - try { - const dom = new JSDOM(html, { url, virtualConsole }); - return dom; - } catch (error) { - console.error(error); - return null; - } -}; diff --git a/src/app/page.tsx b/src/app/page.tsx index a23c302..1b8bebc 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -10,17 +10,17 @@ import useSWR from "swr/immutable"; import { AddSourceDialog } from "@/dialogs/AddSourceDialog"; import { EditSourceDialog } from "@/dialogs/EditSourceDialog"; import { RemoveSourceDialog } from "@/dialogs/RemoveSourceDialog"; -import { configFetcher } from "@/fetchers/configFetcher"; +import { listFetcher } from "@/fetchers/listFetcher"; import type { SourceConfig } from "@/index"; -import List from "./List"; -import Source from "./Source"; +import SourceItems from "./SourceItems"; +import SourceOverview from "./SourceOverview"; const HomePage = () => { const { data: sources, error, isLoading, - } = useSWR, Error>("lists", configFetcher); + } = useSWR, Error>("lists", listFetcher); return (
@@ -49,7 +49,7 @@ const HomePage = () => { .sort((a, b) => a.name.localeCompare(b.name)) .map((source) => (
  • - {
    • - + )} diff --git a/src/app/source/route.ts b/src/app/source/route.ts new file mode 100644 index 0000000..6817aaa --- /dev/null +++ b/src/app/source/route.ts @@ -0,0 +1,192 @@ +import { after } from "next/server"; +import { createClient } from "redis"; +import { toTemporalInstant } from "temporal-polyfill"; +import * as v from "valibot"; +import { ResponseError } from "@/errors/ResponseError"; +import type { SourceResponse, SourceResponseDebugInfo, SourceResponseItem } from "@/index"; +import { sourceRequestSchema } from "@/schemas/sourceRequestSchema"; +import { createDOM } from "@/utils/createDOM"; +import { select } from "@/utils/select"; +import { selectAll } from "@/utils/selectAll"; + +export const GET = async (request: Request) => { + const debugInfo: SourceResponseDebugInfo = { + htmlSamples: { + excludeFilter: [], + includeFilter: [], + matched: [], + }, + itemsMatched: 0, + itemsRemovedByExcludeFilter: 0, + itemsRemovedByIncludeFilter: 0, + }; + + try { + const redis = process.env.REDIS_URL ? createClient({ url: process.env.REDIS_URL }) : null; + + if (redis) await redis.connect(); + + after(() => { + if (redis) redis.destroy(); + }); + + const items: Array = []; + + const requestUrl = new URL(request.url); + + const params: { [key: string]: string } = {}; + + for (const [key, value] of requestUrl.searchParams) { + if (key in params) { + throw new ResponseError(`Duplicate query params: ${key}`, 400); + } + params[key] = value; + } + + const parsedParams = v.safeParse(sourceRequestSchema, params); + + if (!parsedParams.success) { + throw new ResponseError( + "Invalid query params", + 400, + parsedParams.issues.map((issue) => + issue.path ? `${issue.path.join(".")}: ${issue.message}` : issue.message, + ), + ); + } + + const { debug, exclude, include, itemSelector, linkSelector, titleSelector, url } = + parsedParams.output; + + const sourceResponse = await fetch(url, { + next: { + revalidate: process.env.NODE_ENV === "development" ? 60 : 3600, + }, + }); + + if (sourceResponse.ok === false) { + throw new ResponseError("Failed to fetch from source", 502); + } + + const sourceDateHeader = sourceResponse.headers.get("date"); + + const sourceUpdatedInstant = toTemporalInstant.apply( + sourceDateHeader ? new Date(sourceDateHeader) : new Date(Date.now()), + ); + + const html = await sourceResponse.text(); + + const dom = await createDOM(html, url); + + if (!dom) { + throw new ResponseError("Unable to parse source"); + } + + const matchedItems = selectAll(dom.window.document, itemSelector); + + debugInfo.itemsMatched = matchedItems.length; + + for (const item of matchedItems) { + const lowercaseText = item.textContent.toLocaleLowerCase(); + + if (include.length > 0 && item.textContent !== null) { + if (include.every((phrase) => lowercaseText.includes(phrase) === false)) { + debugInfo.itemsRemovedByIncludeFilter += 1; + if (debugInfo.htmlSamples.includeFilter.length < 3) { + debugInfo.htmlSamples.includeFilter.push(item.outerHTML); + } + continue; + } + } + + if (exclude.length > 0 && item.textContent !== null) { + if (exclude.some((phrase) => lowercaseText.includes(phrase) === true)) { + debugInfo.itemsRemovedByExcludeFilter += 1; + if (debugInfo.htmlSamples.excludeFilter.length < 3) { + debugInfo.htmlSamples.excludeFilter.push(item.outerHTML); + } + continue; + } + } + + if (debugInfo.htmlSamples.matched.length < 3) { + debugInfo.htmlSamples.matched.push(item.outerHTML); + } + + const title = + titleSelector && titleSelector.trim() !== "" + ? select(item, titleSelector)?.textContent + : item.textContent; + + if (title && items.every((item) => item.title !== title)) { + const link = linkSelector && linkSelector.trim() !== "" ? select(item, linkSelector) : item; + const itemUrl = + link?.tagName === "A" ? (link.getAttribute("href") ?? undefined) : undefined; + const parsedUrl = itemUrl ? URL.parse(itemUrl, url)?.href : undefined; + + items.push({ + firstSeen: sourceUpdatedInstant.toString(), + title, + url: parsedUrl, + }); + } + } + + if (redis) { + const keys = items.map( + ({ title }) => `${url}:${itemSelector}:${titleSelector ?? ""}:${title.slice(0, 100)}`, + ); + + if (keys.length > 0) { + const firstSeenValues = await redis.HMGET("first_seen", keys); + + for (let i = 0; i < firstSeenValues.length; i++) { + const value = firstSeenValues[i]; + const item = items[i]; + + if (item && value) item.firstSeen = value; + } + + await redis + .multi() + .HSET( + "first_seen", + Object.fromEntries( + keys.map((key, index) => [ + key, + firstSeenValues[index] ?? sourceUpdatedInstant.toString(), + ]), + ), + ) + .HEXPIRE("first_seen", keys, 60 * 60 * 24 * 28) + .exec(); + } + } + + const response: SourceResponse = { + debug: debug ? debugInfo : undefined, + fetchedAt: sourceUpdatedInstant.toString(), + items, + ok: true, + }; + + return Response.json(response, { + headers: { + "Vercel-CDN-Cache-Control": "max-age=10", + }, + }); + } catch (error) { + const message = error instanceof Error ? error.message : `Unknown error: ${error}`; + + const response: SourceResponse = { + debug: debugInfo, + errors: error instanceof ResponseError ? error.errors : [message], + ok: false, + }; + + return Response.json(response, { + status: error instanceof ResponseError ? error.statusCode : 500, + statusText: message, + }); + } +}; diff --git a/src/dialogs/AddSourceDialog.tsx b/src/dialogs/AddSourceDialog.tsx index 198e352..cc31247 100644 --- a/src/dialogs/AddSourceDialog.tsx +++ b/src/dialogs/AddSourceDialog.tsx @@ -41,7 +41,7 @@ export const AddSourceDialog = () => { /> )} hue="gray" - width="medium" + width="large" > Add diff --git a/src/dialogs/EditSourceDialog.tsx b/src/dialogs/EditSourceDialog.tsx index 31fa8db..2284cbd 100644 --- a/src/dialogs/EditSourceDialog.tsx +++ b/src/dialogs/EditSourceDialog.tsx @@ -2,6 +2,7 @@ import { Dialog } from "@colonydb/anthill/Dialog"; import { Icon } from "@colonydb/anthill/Icon"; +import { useCallback } from "react"; import useSWRMutation from "swr/mutation"; import { SourceForm } from "@/app/SourceForm"; import type { SourceConfig } from "@/index"; @@ -13,31 +14,31 @@ type Props = { export const EditSourceDialog = ({ source }: Props) => { const { trigger } = useSWRMutation("lists", editSource); - return ( - } - padded - render={(closeDialog) => ( - { + + const render = useCallback( + (closeDialog: () => void) => ( + { + closeDialog(); + }} + onSubmit={async (source) => { + await trigger(source); + }} + onSuccess={() => { + setTimeout(() => { closeDialog(); - }} - onSubmit={async (source) => { - await trigger(source); - }} - onSuccess={() => { - setTimeout(() => { - closeDialog(); - }, 1000); - }} - title="Edit Source" - /> - )} - width="medium" - > + }, 1000); + }} + title="Edit Source" + /> + ), + [source, trigger], + ); + + return ( + } padded render={render} width="large"> Edit ); diff --git a/src/dialogs/RemoveSourceDialog.tsx b/src/dialogs/RemoveSourceDialog.tsx index f087e06..658635e 100644 --- a/src/dialogs/RemoveSourceDialog.tsx +++ b/src/dialogs/RemoveSourceDialog.tsx @@ -1,13 +1,12 @@ "use client"; -import { Action } from "@colonydb/anthill/Action"; import { Button } from "@colonydb/anthill/Button"; import { Card } from "@colonydb/anthill/Card"; import { CardContent } from "@colonydb/anthill/CardContent"; import { Dialog } from "@colonydb/anthill/Dialog"; +import { DialogHeader } from "@colonydb/anthill/DialogHeader"; import { Form } from "@colonydb/anthill/Form"; import { FormFooter } from "@colonydb/anthill/FormFooter"; -import { Header } from "@colonydb/anthill/Header"; import { Heading } from "@colonydb/anthill/Heading"; import { Icon } from "@colonydb/anthill/Icon"; import useSWRMutation from "swr/mutation"; @@ -44,20 +43,9 @@ export const RemoveSourceDialog = ({ source }: Props) => { > } - onClick={() => { - closeDialog(); - }} - title="close" - /> - } - > + Remove Source - + } footer={ { )} - width="narrow" + width="small" > Remove diff --git a/src/errors/ResponseError.ts b/src/errors/ResponseError.ts new file mode 100644 index 0000000..df8b571 --- /dev/null +++ b/src/errors/ResponseError.ts @@ -0,0 +1,9 @@ +export class ResponseError extends Error { + constructor( + message: string, + public statusCode: number = 500, + public errors: string[] = [], + ) { + super(message); + } +} diff --git a/src/fetchers/configFetcher.ts b/src/fetchers/configFetcher.ts deleted file mode 100644 index eab6ba7..0000000 --- a/src/fetchers/configFetcher.ts +++ /dev/null @@ -1,4 +0,0 @@ -export const configFetcher = (key: string) => { - const value = localStorage.getItem(key); - return value === null ? null : JSON.parse(value); -}; diff --git a/src/fetchers/listFetcher.ts b/src/fetchers/listFetcher.ts new file mode 100644 index 0000000..6eec791 --- /dev/null +++ b/src/fetchers/listFetcher.ts @@ -0,0 +1,9 @@ +import * as v from "valibot"; +import { listSchema } from "@/schemas/listSchema"; + +export const listFetcher = (key: string) => { + const value = localStorage.getItem(key); + const list = value === null ? [] : JSON.parse(value); + v.assert(listSchema, list); + return list; +}; diff --git a/src/fetchers/sourceFetcher.ts b/src/fetchers/sourceFetcher.ts index 5383610..d69e75a 100644 --- a/src/fetchers/sourceFetcher.ts +++ b/src/fetchers/sourceFetcher.ts @@ -1,10 +1,17 @@ -import type { SourceResult } from "@/index"; +import type { SourceResponse } from ".."; export const sourceFetcher = async (url: string) => { const response = await fetch(url); + if (!response.ok) { throw Error(`Failed to fetch data: ${response.status} ${response.statusText}`); } - const result = await response.json(); - return result as SourceResult; + + const result = (await response.json()) as SourceResponse; + + if (result.ok === false) { + throw Error(`Failed to fetch data: ${result.errors.join(", ")}`); + } + + return result; }; diff --git a/src/hooks/useSource.ts b/src/hooks/useSource.ts new file mode 100644 index 0000000..bf2e8ce --- /dev/null +++ b/src/hooks/useSource.ts @@ -0,0 +1,19 @@ +import { useMemo } from "react"; +import useSWR from "swr"; +import { sourceFetcher } from "@/fetchers/sourceFetcher"; +import { sourceUrlFromConfig } from "@/utils/sourceUrlFromConfig"; +import type { SourceConfig } from ".."; +import { useLocation } from "./useLocation"; + +export const useSource = (config: SourceConfig, debug: boolean = false) => { + const location = useLocation(); + + const sourceUrl = useMemo( + () => sourceUrlFromConfig(config, location, debug), + [config, debug, location], + ); + + return useSWR>, Error>(sourceUrl, sourceFetcher, { + shouldRetryOnError: false, + }); +}; diff --git a/src/index.ts b/src/index.ts index ed43f90..449a012 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,27 +1,37 @@ -export type Item = { - firstSeen: string; - title: string; - url: string | undefined; -}; +import type { InferOutput } from "valibot"; +import type { sourceConfigSchema } from "./schemas/sourceConfigSchema"; +import type { sourceRequestSchema } from "./schemas/sourceRequestSchema"; -export type SourceConfig = { - exclude?: string; - id: string; - include?: string; - itemSelector: string; - linkSelector?: string; - name: string; - titleSelector?: string; - url: string; -}; +export type SourceConfig = InferOutput; + +export type SourceRequest = InferOutput; -export type SourceResult = { - debug: { - itemsFound: number; - itemsAfterFilter: number; - firstTitle: string | undefined; - firstLink: string | undefined; +export type SourceResponse = + | { + debug?: SourceResponseDebugInfo; + fetchedAt: string; + items: Array; + ok: true; + } + | { + debug: SourceResponseDebugInfo; + errors: Array; + ok: false; + }; + +export type SourceResponseDebugInfo = { + htmlSamples: { + excludeFilter: Array; + includeFilter: Array; + matched: Array; }; - fetchedAt: string; - items: Array; + itemsMatched: number; + itemsRemovedByExcludeFilter: number; + itemsRemovedByIncludeFilter: number; +}; + +export type SourceResponseItem = { + firstSeen: string; + title: string; + url: string | undefined; }; diff --git a/src/mutations/addSource.ts b/src/mutations/addSource.ts index 66affbe..8598149 100644 --- a/src/mutations/addSource.ts +++ b/src/mutations/addSource.ts @@ -1,22 +1,11 @@ import * as v from "valibot"; +import { listSchema } from "@/schemas/listSchema"; import type { SourceConfig } from ".."; export const addSource = async (key: string, { arg: source }: { arg: SourceConfig }) => { const current = JSON.parse(localStorage.getItem(key) ?? "[]"); - v.parse( - v.object({ - exclude: v.optional(v.string()), - id: v.string(), - include: v.optional(v.string()), - itemSelector: v.pipe(v.string(), v.nonEmpty()), - linkSelector: v.optional(v.string()), - name: v.pipe(v.string(), v.nonEmpty()), - titleSelector: v.optional(v.string()), - url: v.pipe(v.string(), v.nonEmpty(), v.url()), - }), - source, - ); + v.assert(listSchema, current); const value = current ? [...current, source] : [source]; diff --git a/src/mutations/editSource.ts b/src/mutations/editSource.ts index 79742cd..1fc0635 100644 --- a/src/mutations/editSource.ts +++ b/src/mutations/editSource.ts @@ -1,26 +1,15 @@ import * as v from "valibot"; +import { listSchema } from "@/schemas/listSchema"; import type { SourceConfig } from ".."; export const editSource = async (key: string, { arg: source }: { arg: SourceConfig }) => { - const current = JSON.parse(localStorage.getItem(key) ?? "[]") as Array; + const current = JSON.parse(localStorage.getItem(key) ?? "[]"); - v.parse( - v.object({ - exclude: v.optional(v.string()), - id: v.string(), - include: v.optional(v.string()), - itemSelector: v.pipe(v.string(), v.nonEmpty()), - linkSelector: v.optional(v.string()), - name: v.pipe(v.string(), v.nonEmpty()), - titleSelector: v.optional(v.string()), - url: v.pipe(v.string(), v.nonEmpty(), v.url()), - }), - source, - ); + v.assert(listSchema, current); const value = current ? [...current.filter((x) => x.id !== source.id), source] : [source]; - localStorage.setItem("lists", JSON.stringify(value)); + localStorage.setItem(key, JSON.stringify(value)); return value; }; diff --git a/src/mutations/removeSource.ts b/src/mutations/removeSource.ts index 9bcf6ce..6aaf045 100644 --- a/src/mutations/removeSource.ts +++ b/src/mutations/removeSource.ts @@ -1,11 +1,15 @@ +import * as v from "valibot"; +import { listSchema } from "@/schemas/listSchema"; import type { SourceConfig } from ".."; export const removeSource = async (key: string, { arg: source }: { arg: SourceConfig }) => { - const current = JSON.parse(localStorage.getItem(key) ?? "[]") as Array; + const current = JSON.parse(localStorage.getItem(key) ?? "[]"); + + v.assert(listSchema, current); const value = current ? [...current.filter((x) => x.id !== source.id)] : []; - localStorage.setItem("lists", JSON.stringify(value)); + localStorage.setItem(key, JSON.stringify(value)); return value; }; diff --git a/src/schemas/listSchema.ts b/src/schemas/listSchema.ts new file mode 100644 index 0000000..08e19d6 --- /dev/null +++ b/src/schemas/listSchema.ts @@ -0,0 +1,4 @@ +import * as v from "valibot"; +import { sourceConfigSchema } from "./sourceConfigSchema"; + +export const listSchema = v.array(sourceConfigSchema); diff --git a/src/schemas/sourceSchema.ts b/src/schemas/sourceConfigSchema.ts similarity index 88% rename from src/schemas/sourceSchema.ts rename to src/schemas/sourceConfigSchema.ts index d7b9c2a..8c9a70b 100644 --- a/src/schemas/sourceSchema.ts +++ b/src/schemas/sourceConfigSchema.ts @@ -1,6 +1,6 @@ import * as v from "valibot"; -export const sourceSchema = v.object({ +export const sourceConfigSchema = v.object({ exclude: v.optional(v.string()), id: v.string(), include: v.optional(v.string()), diff --git a/src/schemas/sourceRequestSchema.ts b/src/schemas/sourceRequestSchema.ts new file mode 100644 index 0000000..abebd7c --- /dev/null +++ b/src/schemas/sourceRequestSchema.ts @@ -0,0 +1,34 @@ +import * as v from "valibot"; + +export const sourceRequestSchema = v.object({ + debug: v.pipe( + v.optional(v.string(), "false"), + v.toLowerCase(), + v.union([v.literal("true"), v.literal("false"), v.literal("t"), v.literal("f")]), + v.transform((value) => value === "true" || value === "t"), + ), + exclude: v.pipe( + v.optional(v.string(), ""), + v.toLowerCase(), + v.transform((value) => + value + .split(",") + .map((phrase) => phrase.trim()) + .filter((phrase) => phrase !== ""), + ), + ), + include: v.pipe( + v.optional(v.string(), ""), + v.toLowerCase(), + v.transform((value) => + value + .split(",") + .map((phrase) => phrase.trim()) + .filter((phrase) => phrase !== ""), + ), + ), + itemSelector: v.string(), + linkSelector: v.optional(v.string()), + titleSelector: v.optional(v.string()), + url: v.pipe(v.string(), v.url()), +}); diff --git a/src/utils/createDOM.ts b/src/utils/createDOM.ts new file mode 100644 index 0000000..2e7980b --- /dev/null +++ b/src/utils/createDOM.ts @@ -0,0 +1,10 @@ +import { JSDOM, VirtualConsole } from "jsdom"; + +export const createDOM = async (html: string, url: string) => { + const virtualConsole = new VirtualConsole(); + try { + return new JSDOM(html, { url, virtualConsole }); + } catch { + return null; + } +}; diff --git a/src/utils/select.ts b/src/utils/select.ts new file mode 100644 index 0000000..4c50307 --- /dev/null +++ b/src/utils/select.ts @@ -0,0 +1,7 @@ +export const select = (root: Document | Element, selector: string) => { + try { + return root.querySelector(selector); + } catch { + return null; + } +}; diff --git a/src/utils/selectAll.ts b/src/utils/selectAll.ts new file mode 100644 index 0000000..40dff3b --- /dev/null +++ b/src/utils/selectAll.ts @@ -0,0 +1,7 @@ +export const selectAll = (root: Document | Element, selector: string) => { + try { + return root.querySelectorAll(selector); + } catch { + return []; + } +}; diff --git a/src/utils/sourceUrlFromConfig.ts b/src/utils/sourceUrlFromConfig.ts index 811a09d..e82ecf6 100644 --- a/src/utils/sourceUrlFromConfig.ts +++ b/src/utils/sourceUrlFromConfig.ts @@ -3,10 +3,11 @@ import type { SourceConfig } from ".."; export const sourceUrlFromConfig = ( { exclude, include, itemSelector, linkSelector, titleSelector, url }: SourceConfig, location: Location | null, + debug?: boolean, ) => { if (!location) return undefined; - const result = new URL("/api", location.href); + const result = new URL("/source", location.href); if (exclude) result.searchParams.set("exclude", exclude); if (include) result.searchParams.set("include", include); @@ -18,5 +19,7 @@ export const sourceUrlFromConfig = ( result.searchParams.set("url", url); + if (debug) result.searchParams.set("debug", "true"); + return result.href; };