feat(katalog): set aus einem link uebernehmen
Amazon, idealo, geizhals, Hersteller-Shops und Set-Datenbanken: Produkt- daten aus der Seite (schema.org, Open Graph, Amazon-Produktseite), sonst aus dem Link selbst, ergaenzt ueber Rebrickable (Lego) und Merlins Bricks (BlueBrixx, Cobi). Fuellt das Formular vor, das Bild wird beim Speichern lokal abgelegt. Abrufe nur an oeffentliche Adressen (safeFetch). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
unknown
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
498a5c6226
commit
95a28640e1
@@ -5,8 +5,10 @@ import { and, eq, ne } from 'drizzle-orm';
|
||||
import { db } from '@/db';
|
||||
import { instruction, photo, set } from '@/db/schema';
|
||||
import { actionData, actionError, actionOk, formToObject, parseForm, type ActionResult } from '@/lib/action-result';
|
||||
import { env } from '@/env';
|
||||
import { safeFetch, SafeFetchError } from '@/lib/safe-fetch';
|
||||
import { requireUser } from '@/lib/session';
|
||||
import { deleteStoredFile, storeImage, storePdf, UploadError } from '@/lib/uploads';
|
||||
import { deleteStoredFile, storeImage, storeImageBytes, storePdf, UploadError } from '@/lib/uploads';
|
||||
import { optionalText, optionalUrl, setSchema } from '@/lib/validation';
|
||||
import { z } from 'zod';
|
||||
|
||||
@@ -20,6 +22,23 @@ function revalidateSet(id?: string) {
|
||||
if (id) revalidatePath(`/katalog/${id}`);
|
||||
}
|
||||
|
||||
async function cacheRemoteImage(url: string, setId: string): Promise<string | null> {
|
||||
try {
|
||||
const response = await safeFetch(url, {
|
||||
accept: 'image/webp,image/png,image/jpeg;q=0.9,*/*;q=0.5',
|
||||
maxBytes: env.MAX_UPLOAD_BYTES,
|
||||
});
|
||||
if (response.status !== 200) return null;
|
||||
return (await storeImageBytes(response.body, 'set-image', setId)).filePath;
|
||||
} catch (error) {
|
||||
if (error instanceof SafeFetchError || error instanceof UploadError) {
|
||||
console.error('[katalog] Bild konnte nicht übernommen werden', url, error.message);
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function fileFrom(formData: FormData, field: string): File | null {
|
||||
const value = formData.get(field);
|
||||
return value instanceof File && value.size > 0 ? value : null;
|
||||
@@ -60,12 +79,17 @@ export async function createSet(formData: FormData): Promise<ActionResult<{ id:
|
||||
if (error instanceof UploadError) return actionError(error.message);
|
||||
throw error;
|
||||
}
|
||||
} else if (formData.get('cacheImage') === 'on' && parsed.data.imageUrl) {
|
||||
// A picture found by the link import is kept locally, like the Lego
|
||||
// import does. If that fails, the URL stays as the fallback.
|
||||
imagePath = await cacheRemoteImage(parsed.data.imageUrl, id);
|
||||
}
|
||||
|
||||
db.insert(set)
|
||||
.values({
|
||||
id,
|
||||
...parsed.data,
|
||||
imageUrl: imagePath ? null : parsed.data.imageUrl,
|
||||
imagePath,
|
||||
createdAt: now(),
|
||||
updatedAt: now(),
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
'use server';
|
||||
|
||||
import { and, eq } from 'drizzle-orm';
|
||||
import { z } from 'zod';
|
||||
import { db } from '@/db';
|
||||
import { manufacturer, set } from '@/db/schema';
|
||||
import { actionData, actionError, type ActionResult } from '@/lib/action-result';
|
||||
import {
|
||||
cleanSetName,
|
||||
hintsFromHtml,
|
||||
hintsFromUrl,
|
||||
mergeHints,
|
||||
sourceLabel,
|
||||
type LinkHints,
|
||||
} from '@/lib/link-import';
|
||||
import { fetchMinifigCount, fetchSet, isRebrickableEnabled, RebrickableError } from '@/lib/rebrickable';
|
||||
import { safeFetch, SafeFetchError } from '@/lib/safe-fetch';
|
||||
import { requireUser } from '@/lib/session';
|
||||
import { uniqueSlug } from '@/lib/slug';
|
||||
|
||||
/**
|
||||
* "Aus einem Link übernehmen": reads a set out of a shop, price-comparison or
|
||||
* database link and returns a draft for the form. Nothing is saved here except
|
||||
* a missing manufacturer - the form needs it to be selectable.
|
||||
*
|
||||
* Amazon, idealo and geizhals usually refuse requests from a server; then the
|
||||
* link itself (brand, number, words in the address) is the starting point,
|
||||
* and Rebrickable (Lego) or Merlins Bricks (BlueBrixx, Cobi) fill in the rest.
|
||||
*/
|
||||
|
||||
export type LinkPreview = {
|
||||
manufacturerId: string | null;
|
||||
manufacturerName: string | null;
|
||||
setNumber: string;
|
||||
name: string;
|
||||
year: number | null;
|
||||
pieceCount: number | null;
|
||||
minifigCount: number | null;
|
||||
ean: string;
|
||||
imageUrl: string;
|
||||
sourceUrl: string;
|
||||
/** Where the data came from, for the confirmation message. */
|
||||
sources: string[];
|
||||
/** The shop did not let us read the page; only the address was used. */
|
||||
pageBlocked: boolean;
|
||||
/** The set is already in the catalog. */
|
||||
existingSetId: string | null;
|
||||
};
|
||||
|
||||
const inputSchema = z.object({
|
||||
url: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1, 'Bitte einen Link einfügen.')
|
||||
.max(2000, 'Der Link ist zu lang.')
|
||||
.url('Das ist kein gültiger Link.')
|
||||
.refine((value) => /^https?:\/\//i.test(value), 'Nur http- und https-Links.'),
|
||||
});
|
||||
|
||||
/** Manufacturer website for the ones we create on the fly. */
|
||||
const WEBSITES: Record<string, string> = {
|
||||
Lego: 'https://www.lego.com/de-de',
|
||||
Playmobil: 'https://www.playmobil.de',
|
||||
Cobi: 'https://cobi.pl',
|
||||
BlueBrixx: 'https://www.bluebrixx.com',
|
||||
'Mega Bloks': 'https://www.megabrands.com',
|
||||
};
|
||||
|
||||
/** Merlins Bricks keys its set pages by a brand prefix. */
|
||||
const MERLIN_PREFIX: Record<string, string> = { BlueBrixx: 'bb', Cobi: 'cobi' };
|
||||
|
||||
function key(value: string): string {
|
||||
return value.toLowerCase().replace(/[^a-z0-9]/g, '');
|
||||
}
|
||||
|
||||
function ensureManufacturer(name: string): string {
|
||||
const rows = db
|
||||
.select({ id: manufacturer.id, name: manufacturer.name, slug: manufacturer.slug })
|
||||
.from(manufacturer)
|
||||
.all();
|
||||
|
||||
const match = rows.find((row) => key(row.name) === key(name) || row.slug === key(name));
|
||||
if (match) return match.id;
|
||||
|
||||
const id = crypto.randomUUID();
|
||||
const now = new Date();
|
||||
db.insert(manufacturer)
|
||||
.values({
|
||||
id,
|
||||
name,
|
||||
slug: uniqueSlug(name, rows.map((row) => row.slug)),
|
||||
website: WEBSITES[name] ?? null,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.run();
|
||||
return id;
|
||||
}
|
||||
|
||||
async function readPage(url: URL): Promise<{ hints: LinkHints | null; finalUrl: URL; blocked: boolean }> {
|
||||
const response = await safeFetch(url);
|
||||
const finalUrl = new URL(response.url);
|
||||
const isHtml = /html|xml/i.test(response.contentType);
|
||||
|
||||
if (response.status !== 200 || !isHtml) return { hints: null, finalUrl, blocked: true };
|
||||
return { hints: hintsFromHtml(response.body.toString('utf8'), finalUrl), finalUrl, blocked: false };
|
||||
}
|
||||
|
||||
async function fromRebrickable(setNumber: string): Promise<(LinkHints & { minifigCount?: number }) | null> {
|
||||
if (!isRebrickableEnabled()) return null;
|
||||
try {
|
||||
const remote = await fetchSet(setNumber);
|
||||
if (!remote) return null;
|
||||
const minifigs = await fetchMinifigCount(remote.setNumber);
|
||||
return {
|
||||
name: remote.name,
|
||||
year: remote.year ?? undefined,
|
||||
pieceCount: remote.pieceCount ?? undefined,
|
||||
imageUrl: remote.imageUrl ?? undefined,
|
||||
minifigCount: minifigs ?? undefined,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof RebrickableError) return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function fromMerlinsBricks(brand: string, setNumber: string): Promise<LinkHints | null> {
|
||||
const prefix = MERLIN_PREFIX[brand];
|
||||
if (!prefix || !/^[\w-]+$/.test(setNumber)) return null;
|
||||
try {
|
||||
const page = await readPage(new URL(`https://www.merlinsbricks.com/sets/${prefix}-${setNumber}/`));
|
||||
return page.hints;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function previewSetFromLink(input: unknown): Promise<ActionResult<LinkPreview>> {
|
||||
await requireUser();
|
||||
|
||||
const parsed = inputSchema.safeParse(input);
|
||||
if (!parsed.success) return actionError(parsed.error.issues[0]?.message ?? 'Ungültiger Link.');
|
||||
|
||||
const url = new URL(parsed.data.url);
|
||||
const sources: string[] = [];
|
||||
|
||||
// 1. The page itself, if it lets us in. A short link (amzn.eu/...) still
|
||||
// tells us where it leads, even when the target then refuses.
|
||||
let pageHints: LinkHints | null = null;
|
||||
let finalUrl = url;
|
||||
let pageBlocked = true;
|
||||
try {
|
||||
const page = await readPage(url);
|
||||
pageHints = page.hints;
|
||||
finalUrl = page.finalUrl;
|
||||
pageBlocked = page.blocked;
|
||||
} catch (error) {
|
||||
if (!(error instanceof SafeFetchError)) throw error;
|
||||
if (/intern/i.test(error.message)) return actionError(error.message);
|
||||
}
|
||||
if (pageHints) sources.push(sourceLabel(finalUrl));
|
||||
|
||||
// 2. The address.
|
||||
const addressHints = mergeHints(hintsFromUrl(finalUrl), hintsFromUrl(url));
|
||||
if (!pageHints || pageBlocked) sources.push(`Link (${sourceLabel(finalUrl)})`);
|
||||
|
||||
let hints = mergeHints(pageHints, addressHints);
|
||||
let minifigCount: number | null = null;
|
||||
|
||||
// 3. A set database for the brand.
|
||||
if (hints.manufacturer === 'Lego' && hints.setNumber) {
|
||||
const remote = await fromRebrickable(hints.setNumber);
|
||||
if (remote) {
|
||||
// The shop's German name beats Rebrickable's English one; Rebrickable's
|
||||
// facts and clean picture beat the shop's.
|
||||
hints = mergeHints({ name: pageHints?.name ?? addressHints.name, ean: hints.ean }, remote, hints);
|
||||
minifigCount = remote.minifigCount ?? null;
|
||||
sources.push('Rebrickable');
|
||||
}
|
||||
} else if (hints.manufacturer && hints.setNumber && !/merlinsbricks\./.test(finalUrl.hostname)) {
|
||||
const merlin = await fromMerlinsBricks(hints.manufacturer, hints.setNumber);
|
||||
if (merlin) {
|
||||
hints = mergeHints(hints, merlin);
|
||||
sources.push('Merlins Bricks');
|
||||
}
|
||||
}
|
||||
|
||||
if (!hints.name && !hints.setNumber) {
|
||||
return actionError(
|
||||
pageBlocked
|
||||
? `${sourceLabel(finalUrl)} lässt keine Abfrage zu, und der Link selbst verrät nichts über das Set. Versuch es mit einem anderen Link oder trag das Set von Hand ein.`
|
||||
: 'Auf der Seite stehen keine Produktdaten, mit denen sich etwas anfangen lässt.',
|
||||
);
|
||||
}
|
||||
|
||||
const manufacturerName = hints.manufacturer ?? null;
|
||||
const manufacturerId = manufacturerName ? ensureManufacturer(manufacturerName) : null;
|
||||
const setNumber = hints.setNumber ?? '';
|
||||
|
||||
const existing =
|
||||
manufacturerId && setNumber
|
||||
? db
|
||||
.select({ id: set.id })
|
||||
.from(set)
|
||||
.where(and(eq(set.manufacturerId, manufacturerId), eq(set.setNumber, setNumber)))
|
||||
.get()
|
||||
: undefined;
|
||||
|
||||
return actionData({
|
||||
manufacturerId,
|
||||
manufacturerName,
|
||||
setNumber,
|
||||
name: hints.name ? cleanSetName(hints.name, manufacturerName ?? undefined, setNumber) : '',
|
||||
year: hints.year ?? null,
|
||||
pieceCount: hints.pieceCount ?? null,
|
||||
minifigCount,
|
||||
ean: hints.ean ?? '',
|
||||
imageUrl: hints.imageUrl ?? '',
|
||||
sourceUrl: url.toString(),
|
||||
sources,
|
||||
pageBlocked,
|
||||
existingSetId: existing?.id ?? null,
|
||||
});
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { Metadata } from 'next';
|
||||
import Link from 'next/link';
|
||||
import { NewSetEntry } from '@/components/katalog/link-import';
|
||||
import { RebrickableImport } from '@/components/katalog/rebrickable-import';
|
||||
import { SetForm } from '@/components/katalog/set-form';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { listManufacturers, listThemes } from '@/lib/queries/master-data';
|
||||
import { isRebrickableEnabled } from '@/lib/rebrickable';
|
||||
@@ -25,26 +25,8 @@ export default async function NeuesSetPage({ searchParams }: { searchParams: Sea
|
||||
|
||||
const importEnabled = isRebrickableEnabled();
|
||||
|
||||
if (manufacturers.length === 0) {
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<h1 className="text-2xl font-semibold tracking-tight">Neues Set</h1>
|
||||
|
||||
{/* The Lego import creates its manufacturer itself, so it works even
|
||||
on a completely empty instance. */}
|
||||
<RebrickableImport enabled={importEnabled} />
|
||||
|
||||
<p className="rounded-lg border border-dashed border-border p-6 text-sm text-muted-foreground">
|
||||
Für ein Set von Hand brauchst du zuerst einen Hersteller – Lego, Playmobil, Cobi oder
|
||||
was auch immer im Kinderzimmer steht.
|
||||
</p>
|
||||
<Button asChild size="lg">
|
||||
<Link href="/verwaltung/hersteller">Hersteller anlegen</Link>
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// One layout for both cases: the link import may create the first
|
||||
// manufacturer, and its draft must survive the refresh that follows.
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div>
|
||||
@@ -54,15 +36,19 @@ export default async function NeuesSetPage({ searchParams }: { searchParams: Sea
|
||||
Sammlung hinzufügen“ fest.
|
||||
</p>
|
||||
</div>
|
||||
{/* The Lego import creates its manufacturer itself, so it works even
|
||||
on a completely empty instance. */}
|
||||
<RebrickableImport enabled={importEnabled} />
|
||||
|
||||
<div className="flex items-center gap-3 text-xs text-muted-foreground">
|
||||
<span className="h-px flex-1 bg-border" />
|
||||
oder von Hand
|
||||
<span className="h-px flex-1 bg-border" />
|
||||
</div>
|
||||
|
||||
<SetForm manufacturers={manufacturers} themes={themes} defaultEan={ean} />
|
||||
<NewSetEntry manufacturers={manufacturers} themes={themes} defaultEan={ean}>
|
||||
<p className="rounded-lg border border-dashed border-border p-6 text-sm text-muted-foreground">
|
||||
Für ein Set von Hand brauchst du zuerst einen Hersteller – Lego, Playmobil, Cobi oder
|
||||
was auch immer im Kinderzimmer steht.
|
||||
</p>
|
||||
<Button asChild size="lg">
|
||||
<Link href="/verwaltung/hersteller">Hersteller anlegen</Link>
|
||||
</Button>
|
||||
</NewSetEntry>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
'use client';
|
||||
|
||||
import Link from 'next/link';
|
||||
import { useRouter } from 'next/navigation';
|
||||
import { useState, useTransition } from 'react';
|
||||
import { Link2, Loader2 } from 'lucide-react';
|
||||
import { toast } from 'sonner';
|
||||
import { previewSetFromLink, type LinkPreview } from '@/app/(app)/katalog/link-actions';
|
||||
import { SetForm, type SetPrefill } from '@/components/katalog/set-form';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card';
|
||||
import { Field } from '@/components/ui/field';
|
||||
import { Input } from '@/components/ui/input';
|
||||
|
||||
function toPrefill(preview: LinkPreview): SetPrefill {
|
||||
return {
|
||||
manufacturerId: preview.manufacturerId ?? undefined,
|
||||
setNumber: preview.setNumber || undefined,
|
||||
name: preview.name || undefined,
|
||||
year: preview.year,
|
||||
pieceCount: preview.pieceCount,
|
||||
minifigCount: preview.minifigCount,
|
||||
ean: preview.ean || null,
|
||||
imageUrl: preview.imageUrl || null,
|
||||
cacheImage: Boolean(preview.imageUrl),
|
||||
notes: `Übernommen aus: ${preview.sourceUrl}`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* "Neues Set" with the link import on top: paste a link from a shop, a price
|
||||
* comparison or a set database, check the prefilled form, save.
|
||||
*/
|
||||
export function NewSetEntry({
|
||||
manufacturers,
|
||||
themes,
|
||||
defaultEan,
|
||||
children,
|
||||
}: {
|
||||
manufacturers: Array<{ id: string; name: string }>;
|
||||
themes: Array<{ id: string; name: string; manufacturerId: string }>;
|
||||
defaultEan?: string;
|
||||
/** Shown instead of the form while there is no manufacturer yet. */
|
||||
children?: React.ReactNode;
|
||||
}) {
|
||||
const router = useRouter();
|
||||
const [url, setUrl] = useState('');
|
||||
const [prefill, setPrefill] = useState<SetPrefill | undefined>(undefined);
|
||||
const [version, setVersion] = useState(0);
|
||||
const [existing, setExisting] = useState<{ id: string; label: string } | null>(null);
|
||||
const [pending, startTransition] = useTransition();
|
||||
|
||||
function submit(event: React.FormEvent<HTMLFormElement>) {
|
||||
event.preventDefault();
|
||||
setExisting(null);
|
||||
|
||||
startTransition(async () => {
|
||||
const result = await previewSetFromLink({ url });
|
||||
if (!result.ok) {
|
||||
toast.error(result.error);
|
||||
return;
|
||||
}
|
||||
|
||||
const preview = result.data;
|
||||
if (preview.existingSetId) {
|
||||
setExisting({
|
||||
id: preview.existingSetId,
|
||||
label: [preview.setNumber, preview.name].filter(Boolean).join(' · '),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
setPrefill(toPrefill(preview));
|
||||
setVersion((value) => value + 1);
|
||||
// A manufacturer may just have been created for the draft.
|
||||
router.refresh();
|
||||
|
||||
const missing = [
|
||||
preview.manufacturerId ? null : 'Hersteller',
|
||||
preview.setNumber ? null : 'Setnummer',
|
||||
preview.name ? null : 'Name',
|
||||
].filter(Boolean);
|
||||
|
||||
toast.success(
|
||||
missing.length === 0
|
||||
? `Übernommen aus ${preview.sources.join(', ')}. Bitte kurz prüfen und speichern.`
|
||||
: `Übernommen aus ${preview.sources.join(', ')} – ${missing.join(', ')} bitte ergänzen.`,
|
||||
);
|
||||
if (preview.pageBlocked) {
|
||||
toast.info('Die Seite selbst ließ sich nicht lesen – die Angaben stammen aus dem Link.');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Aus einem Link übernehmen</CardTitle>
|
||||
<CardDescription>
|
||||
Link zu Amazon, idealo, geizhals, zum Shop des Herstellers oder zu einer Set-Datenbank
|
||||
einfügen. Die Angaben landen im Formular unten – gespeichert wird erst, wenn du es
|
||||
sagst.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-3">
|
||||
<form onSubmit={submit} className="flex items-end gap-2">
|
||||
<Field label="Link" htmlFor="import-link" className="flex-1">
|
||||
<Input
|
||||
id="import-link"
|
||||
type="url"
|
||||
inputMode="url"
|
||||
value={url}
|
||||
onChange={(event) => setUrl(event.target.value)}
|
||||
placeholder="https://…"
|
||||
maxLength={2000}
|
||||
/>
|
||||
</Field>
|
||||
<Button type="submit" size="lg" disabled={pending || url.trim().length === 0}>
|
||||
{pending ? <Loader2 className="animate-spin" /> : <Link2 />}
|
||||
Übernehmen
|
||||
</Button>
|
||||
</form>
|
||||
|
||||
{existing ? (
|
||||
<p className="rounded-lg bg-muted p-3 text-sm">
|
||||
Das Set ist schon im Katalog:{' '}
|
||||
<Link href={`/katalog/${existing.id}`} className="font-semibold underline">
|
||||
{existing.label}
|
||||
</Link>
|
||||
</p>
|
||||
) : null}
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<div className="flex items-center gap-3 text-xs text-muted-foreground">
|
||||
<span className="h-px flex-1 bg-border" />
|
||||
{prefill ? 'Bitte prüfen und speichern' : 'oder von Hand'}
|
||||
<span className="h-px flex-1 bg-border" />
|
||||
</div>
|
||||
|
||||
{manufacturers.length === 0 ? (
|
||||
children
|
||||
) : (
|
||||
<SetForm
|
||||
key={`${version}-${manufacturers.length}`}
|
||||
manufacturers={manufacturers}
|
||||
themes={themes}
|
||||
defaultEan={defaultEan}
|
||||
prefill={prefill}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -47,19 +47,26 @@ export type SetFormValues = {
|
||||
notes: string | null;
|
||||
};
|
||||
|
||||
/** A draft for a new set, e.g. from the link import. */
|
||||
export type SetPrefill = Partial<Omit<SetFormValues, 'id' | 'imagePath'>> & {
|
||||
/** Download imageUrl on save and keep it locally. */
|
||||
cacheImage?: boolean;
|
||||
};
|
||||
|
||||
type Props = {
|
||||
manufacturers: Array<{ id: string; name: string }>;
|
||||
themes: Array<{ id: string; name: string; manufacturerId: string }>;
|
||||
set?: SetFormValues;
|
||||
/** Prefilled EAN coming from the barcode scanner (phase 2). */
|
||||
defaultEan?: string;
|
||||
prefill?: SetPrefill;
|
||||
};
|
||||
|
||||
export function SetForm({ manufacturers, themes, set, defaultEan }: Props) {
|
||||
export function SetForm({ manufacturers, themes, set, defaultEan, prefill }: Props) {
|
||||
const router = useRouter();
|
||||
const [pending, startTransition] = useTransition();
|
||||
const [manufacturerId, setManufacturerId] = useState(
|
||||
set?.manufacturerId ?? manufacturers[0]?.id ?? '',
|
||||
set?.manufacturerId ?? prefill?.manufacturerId ?? manufacturers[0]?.id ?? '',
|
||||
);
|
||||
const [duplicates, setDuplicates] = useState<DuplicateCandidate[] | null>(null);
|
||||
const pendingForm = useRef<FormData | null>(null);
|
||||
@@ -167,7 +174,7 @@ export function SetForm({ manufacturers, themes, set, defaultEan }: Props) {
|
||||
<Input
|
||||
id="setNumber"
|
||||
name="setNumber"
|
||||
defaultValue={set?.setNumber ?? ''}
|
||||
defaultValue={set?.setNumber ?? prefill?.setNumber ?? ''}
|
||||
required
|
||||
maxLength={40}
|
||||
placeholder="60398"
|
||||
@@ -179,7 +186,7 @@ export function SetForm({ manufacturers, themes, set, defaultEan }: Props) {
|
||||
<Input
|
||||
id="name"
|
||||
name="name"
|
||||
defaultValue={set?.name ?? ''}
|
||||
defaultValue={set?.name ?? prefill?.name ?? ''}
|
||||
required
|
||||
maxLength={160}
|
||||
placeholder="Familienhaus und Elektroauto"
|
||||
@@ -192,7 +199,7 @@ export function SetForm({ manufacturers, themes, set, defaultEan }: Props) {
|
||||
name="year"
|
||||
type="number"
|
||||
inputMode="numeric"
|
||||
defaultValue={set?.year ?? ''}
|
||||
defaultValue={set?.year ?? prefill?.year ?? ''}
|
||||
min={1930}
|
||||
max={new Date().getFullYear() + 5}
|
||||
/>
|
||||
@@ -204,7 +211,7 @@ export function SetForm({ manufacturers, themes, set, defaultEan }: Props) {
|
||||
name="pieceCount"
|
||||
type="number"
|
||||
inputMode="numeric"
|
||||
defaultValue={set?.pieceCount ?? ''}
|
||||
defaultValue={set?.pieceCount ?? prefill?.pieceCount ?? ''}
|
||||
min={0}
|
||||
/>
|
||||
</Field>
|
||||
@@ -215,7 +222,7 @@ export function SetForm({ manufacturers, themes, set, defaultEan }: Props) {
|
||||
name="minifigCount"
|
||||
type="number"
|
||||
inputMode="numeric"
|
||||
defaultValue={set?.minifigCount ?? ''}
|
||||
defaultValue={set?.minifigCount ?? prefill?.minifigCount ?? ''}
|
||||
min={0}
|
||||
/>
|
||||
</Field>
|
||||
@@ -235,7 +242,7 @@ export function SetForm({ manufacturers, themes, set, defaultEan }: Props) {
|
||||
id="ean"
|
||||
name="ean"
|
||||
inputMode="numeric"
|
||||
defaultValue={set?.ean ?? defaultEan ?? ''}
|
||||
defaultValue={set?.ean ?? prefill?.ean ?? defaultEan ?? ''}
|
||||
maxLength={20}
|
||||
className="font-mono"
|
||||
/>
|
||||
@@ -256,10 +263,18 @@ export function SetForm({ manufacturers, themes, set, defaultEan }: Props) {
|
||||
name="imageUrl"
|
||||
type="url"
|
||||
inputMode="url"
|
||||
defaultValue={set?.imageUrl ?? ''}
|
||||
defaultValue={set?.imageUrl ?? prefill?.imageUrl ?? ''}
|
||||
placeholder="https://…"
|
||||
/>
|
||||
</Field>
|
||||
{prefill?.cacheImage && prefill.imageUrl ? (
|
||||
<>
|
||||
<input type="hidden" name="cacheImage" value="on" />
|
||||
<p className="-mt-4 text-xs text-muted-foreground">
|
||||
Das Bild aus dem Link wird beim Speichern heruntergeladen und lokal abgelegt.
|
||||
</p>
|
||||
</>
|
||||
) : null}
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="image">Bild hochladen</Label>
|
||||
@@ -286,7 +301,12 @@ export function SetForm({ manufacturers, themes, set, defaultEan }: Props) {
|
||||
</div>
|
||||
|
||||
<Field label="Notizen" htmlFor="notes">
|
||||
<Textarea id="notes" name="notes" defaultValue={set?.notes ?? ''} rows={4} />
|
||||
<Textarea
|
||||
id="notes"
|
||||
name="notes"
|
||||
defaultValue={set?.notes ?? prefill?.notes ?? ''}
|
||||
rows={4}
|
||||
/>
|
||||
</Field>
|
||||
|
||||
<div className="flex gap-3">
|
||||
|
||||
@@ -0,0 +1,380 @@
|
||||
import { isValidEan, normalizeEan } from './ean';
|
||||
|
||||
/**
|
||||
* Reads set data out of a pasted link and, where the page can be fetched, out
|
||||
* of its structured data (schema.org JSON-LD, Open Graph).
|
||||
*
|
||||
* Pure: no network, no database. The server action in katalog/link-actions.ts
|
||||
* does the fetching and decides what to trust.
|
||||
*/
|
||||
|
||||
export type LinkHints = {
|
||||
manufacturer?: string;
|
||||
setNumber?: string;
|
||||
name?: string;
|
||||
ean?: string;
|
||||
imageUrl?: string;
|
||||
pieceCount?: number;
|
||||
year?: number;
|
||||
};
|
||||
|
||||
/** Brands we recognise in links and page data, with how we spell them. */
|
||||
const BRANDS: Array<{ name: string; aliases: string[] }> = [
|
||||
{ name: 'Lego', aliases: ['lego'] },
|
||||
{ name: 'Playmobil', aliases: ['playmobil'] },
|
||||
{ name: 'Cobi', aliases: ['cobi'] },
|
||||
{ name: 'BlueBrixx', aliases: ['bluebrixx', 'bluebrixxpro', 'bb'] },
|
||||
{ name: 'Mega Bloks', aliases: ['megabloks', 'megaconstrux', 'mega'] },
|
||||
{ name: 'Mould King', aliases: ['mouldking'] },
|
||||
{ name: 'Cada', aliases: ['cada'] },
|
||||
{ name: 'Sluban', aliases: ['sluban'] },
|
||||
{ name: 'Qman', aliases: ['qman', 'keeppley'] },
|
||||
{ name: 'Xingbao', aliases: ['xingbao'] },
|
||||
];
|
||||
|
||||
function key(value: string): string {
|
||||
return value.toLowerCase().replace(/[^a-z0-9]/g, '');
|
||||
}
|
||||
|
||||
/** "LEGO®", "lego", "BlueBrixx Pro" -> "Lego", "BlueBrixx". */
|
||||
export function canonicalBrand(value: string | undefined | null): string | undefined {
|
||||
if (!value) return undefined;
|
||||
const wanted = key(value);
|
||||
if (!wanted) return undefined;
|
||||
for (const brand of BRANDS) {
|
||||
if (brand.aliases.some((alias) => wanted === alias || (alias.length > 3 && wanted.startsWith(alias)))) {
|
||||
return brand.name;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Where a link points, in words for the UI. */
|
||||
export function sourceLabel(url: URL): string {
|
||||
const host = url.hostname.replace(/^www\./, '');
|
||||
if (/(^|\.)amazon\./.test(host) || /^amzn\./.test(host)) return 'Amazon';
|
||||
if (/(^|\.)idealo\./.test(host)) return 'idealo';
|
||||
if (/(^|\.)geizhals\./.test(host)) return 'geizhals';
|
||||
if (/(^|\.)brickmerge\./.test(host)) return 'brickmerge';
|
||||
if (/(^|\.)brickfact\./.test(host)) return 'Brickfact';
|
||||
if (/(^|\.)merlinsbricks\./.test(host)) return 'Merlins Bricks';
|
||||
if (/(^|\.)ebay\./.test(host)) return 'eBay';
|
||||
return host;
|
||||
}
|
||||
|
||||
// Words that sit in shop slugs but never in a set name.
|
||||
const STOPWORDS = new Set([
|
||||
'set', 'sets', 'bausatz', 'bausteine', 'klemmbausteine', 'baustein', 'spielzeug', 'spielset',
|
||||
'kinder', 'jungen', 'maedchen', 'mädchen', 'ab', 'jahre', 'jahren', 'jahr', 'teile', 'teilig',
|
||||
'geschenk', 'geschenkidee', 'the', 'and', 'for', 'kids', 'boys',
|
||||
'girls', 'toy', 'toys', 'building', 'kit', 'bricks', 'brick', 'preisvergleich', 'dp', 'html',
|
||||
'neu', 'ovp', 'new', 'sealed',
|
||||
]);
|
||||
|
||||
const KEEP_LOWER = new Set(['und', 'mit', 'der', 'die', 'das', 'von', 'im', 'am', 'zum', 'zur']);
|
||||
|
||||
function looksLikeYear(value: string): boolean {
|
||||
const year = Number(value);
|
||||
return year >= 1950 && year <= 2040;
|
||||
}
|
||||
|
||||
/**
|
||||
* A set number in a list of slug tokens: 4 to 7 digits, longer ones first,
|
||||
* and four-digit years only as a last resort. Cobi's "2-30500" style is kept.
|
||||
*/
|
||||
function findSetNumber(tokens: string[]): string | undefined {
|
||||
const joined = tokens.join('-');
|
||||
const cobi = /(?:^|-)(\d-\d{5})(?:-|$)/.exec(joined);
|
||||
if (cobi) return cobi[1];
|
||||
|
||||
const numbers = tokens.filter((token) => /^\d{4,7}$/.test(token));
|
||||
const best =
|
||||
numbers.find((token) => token.length >= 5) ?? numbers.find((token) => !looksLikeYear(token));
|
||||
return best;
|
||||
}
|
||||
|
||||
function titleCase(word: string): string {
|
||||
if (KEEP_LOWER.has(word)) return word;
|
||||
return word.charAt(0).toUpperCase() + word.slice(1);
|
||||
}
|
||||
|
||||
/** The descriptive part of a shop URL, split into words. */
|
||||
function slugTokens(url: URL): string[] {
|
||||
const segments = decodeURIComponent(url.pathname)
|
||||
.split('/')
|
||||
.filter(Boolean)
|
||||
.map((segment) => segment.replace(/\.html?$/i, ''));
|
||||
|
||||
let slug = '';
|
||||
const host = url.hostname;
|
||||
|
||||
if (/amazon\./.test(host)) {
|
||||
// amazon.de/LEGO-60398-Familienhaus-Elektroauto/dp/B0...: the part before /dp/.
|
||||
const dp = segments.findIndex((segment) => segment === 'dp' || segment === 'gp');
|
||||
slug = dp > 0 ? segments[dp - 1]! : '';
|
||||
} else if (/idealo\./.test(host)) {
|
||||
// .../OffersOfProduct/202595377_-familienhaus-mit-elektroauto-60398-lego
|
||||
slug = (segments.at(-1) ?? '').replace(/^\d+_-?/, '');
|
||||
} else if (/geizhals\./.test(host)) {
|
||||
// geizhals.de/lego-city-familienhaus-...-60398-a2875000
|
||||
slug = (segments.at(-1) ?? '').replace(/-a\d+$/, '');
|
||||
} else {
|
||||
// Generic: the longest segment with dashes is usually the product slug.
|
||||
slug = [...segments].sort((a, b) => b.split(/[-_]/).length - a.split(/[-_]/).length)[0] ?? '';
|
||||
}
|
||||
|
||||
return slug
|
||||
.toLowerCase()
|
||||
.split(/[-_+\s]+/)
|
||||
.map((token) => token.replace(/[^\p{L}\p{N}]/gu, ''))
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
/** Everything the address alone tells us - useful when the page blocks us. */
|
||||
export function hintsFromUrl(url: URL): LinkHints {
|
||||
const host = url.hostname.replace(/^www\./, '');
|
||||
|
||||
// merlinsbricks.com/sets/bb-108782, /sets/cobi-2400: brand and number in one.
|
||||
const merlin = /merlinsbricks\./.test(host)
|
||||
? /\/sets\/([a-z]+)-([\w-]+?)\/?$/i.exec(url.pathname)
|
||||
: null;
|
||||
if (merlin) {
|
||||
return { manufacturer: canonicalBrand(merlin[1]), setNumber: merlin[2] };
|
||||
}
|
||||
|
||||
const tokens = slugTokens(url);
|
||||
const setNumber = findSetNumber(tokens);
|
||||
|
||||
let manufacturer: string | undefined;
|
||||
for (const token of tokens) {
|
||||
manufacturer = canonicalBrand(token);
|
||||
if (manufacturer) break;
|
||||
}
|
||||
manufacturer ??= canonicalBrand(host.split('.').at(-2));
|
||||
|
||||
const words = tokens.filter(
|
||||
(token) =>
|
||||
token !== setNumber &&
|
||||
!/^\d+$/.test(token) &&
|
||||
!STOPWORDS.has(token) &&
|
||||
!canonicalBrand(token) &&
|
||||
token.length > 1,
|
||||
);
|
||||
const name = words.length > 0 ? words.slice(0, 8).map(titleCase).join(' ') : undefined;
|
||||
|
||||
return { manufacturer, setNumber, name };
|
||||
}
|
||||
|
||||
// --- page data ------------------------------------------------------------
|
||||
|
||||
const ENTITIES: Record<string, string> = {
|
||||
amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', '#39': "'",
|
||||
auml: 'ä', ouml: 'ö', uuml: 'ü', Auml: 'Ä', Ouml: 'Ö', Uuml: 'Ü', szlig: 'ß',
|
||||
};
|
||||
|
||||
function decodeEntities(value: string): string {
|
||||
return value
|
||||
.replace(/&#(\d+);/g, (_, code: string) => String.fromCodePoint(Number(code)))
|
||||
.replace(/&#x([\da-f]+);/gi, (_, code: string) => String.fromCodePoint(parseInt(code, 16)))
|
||||
.replace(/&([a-z]+|#39);/gi, (match, name: string) => ENTITIES[name] ?? match);
|
||||
}
|
||||
|
||||
function metaContent(html: string, property: string): string | undefined {
|
||||
const escaped = property.replace(/[.:]/g, '\\$&');
|
||||
const pattern = new RegExp(
|
||||
`<meta[^>]+(?:property|name)=["']${escaped}["'][^>]*content=["']([^"']*)["']|<meta[^>]+content=["']([^"']*)["'][^>]*(?:property|name)=["']${escaped}["']`,
|
||||
'i',
|
||||
);
|
||||
const match = pattern.exec(html);
|
||||
const value = match?.[1] ?? match?.[2];
|
||||
return value ? decodeEntities(value).trim() : undefined;
|
||||
}
|
||||
|
||||
type JsonObject = Record<string, unknown>;
|
||||
|
||||
function isObject(value: unknown): value is JsonObject {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/** All schema.org Product objects on the page, however they are nested. */
|
||||
function jsonLdProducts(html: string): JsonObject[] {
|
||||
const products: JsonObject[] = [];
|
||||
const pattern = /<script[^>]+application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi;
|
||||
|
||||
const visit = (node: unknown) => {
|
||||
if (Array.isArray(node)) return node.forEach(visit);
|
||||
if (!isObject(node)) return;
|
||||
const type = node['@type'];
|
||||
const types = Array.isArray(type) ? type : [type];
|
||||
if (types.includes('Product') || types.includes('ProductGroup')) products.push(node);
|
||||
if (node['@graph']) visit(node['@graph']);
|
||||
};
|
||||
|
||||
for (const match of html.matchAll(pattern)) {
|
||||
try {
|
||||
visit(JSON.parse(match[1]!.trim()));
|
||||
} catch {
|
||||
// A broken block on someone else's page is not our problem.
|
||||
}
|
||||
}
|
||||
return products;
|
||||
}
|
||||
|
||||
function text(value: unknown): string | undefined {
|
||||
if (typeof value === 'string') return decodeEntities(value).trim() || undefined;
|
||||
if (typeof value === 'number') return String(value);
|
||||
if (isObject(value)) return text(value.name) ?? text(value.url);
|
||||
if (Array.isArray(value)) return text(value[0]);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function cleanEan(value: unknown): string | undefined {
|
||||
const raw = text(value);
|
||||
if (!raw) return undefined;
|
||||
const ean = normalizeEan(raw);
|
||||
return isValidEan(ean) ? ean : undefined;
|
||||
}
|
||||
|
||||
function absoluteUrl(value: string | undefined, base: URL): string | undefined {
|
||||
if (!value) return undefined;
|
||||
try {
|
||||
const url = new URL(value, base);
|
||||
return url.protocol === 'https:' || url.protocol === 'http:' ? url.toString() : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** "462 Teile", "450 pieces" in a description. */
|
||||
function piecesFrom(value: string | undefined): number | undefined {
|
||||
const match = value && /(\d{1,3}(?:[.,]\d{3})*|\d+)\s*(?:teilen?|pieces|pcs|steinen?|bausteinen?|parts)\b/i.exec(value);
|
||||
if (!match) return undefined;
|
||||
const pieces = Number(match[1]!.replace(/[.,]/g, ''));
|
||||
return pieces > 0 && pieces < 100_000 ? pieces : undefined;
|
||||
}
|
||||
|
||||
/** "from 2025", "aus 2023", "Jahr: 2021" in a description. */
|
||||
function yearFrom(value: string | undefined): number | undefined {
|
||||
const match = value && /\b(?:from|aus|jahr:?|released|erschienen)\s+((?:19|20)\d{2})\b/i.exec(value);
|
||||
return match ? Number(match[1]) : undefined;
|
||||
}
|
||||
|
||||
function stripTags(value: string): string {
|
||||
return decodeEntities(value.replace(/<[^>]+>/g, ' '))
|
||||
.replace(/[]/g, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Amazon has no schema.org data, but a stable page skeleton: the title in
|
||||
* #productTitle, the brand in the byline, pictures as "hiRes" in a script and
|
||||
* the facts in the "Produktinformation" table or bullet list.
|
||||
*/
|
||||
function hintsFromAmazon(html: string): LinkHints {
|
||||
const facts = new Map<string, string>();
|
||||
for (const match of html.matchAll(/<th[^>]*>([\s\S]*?)<\/th>\s*<td[^>]*>([\s\S]*?)<\/td>/gi)) {
|
||||
facts.set(stripTags(match[1]!).toLowerCase(), stripTags(match[2]!));
|
||||
}
|
||||
for (const match of html.matchAll(
|
||||
/<span class="a-text-bold">([\s\S]*?)<\/span>\s*<span[^>]*>([\s\S]*?)<\/span>/gi,
|
||||
)) {
|
||||
facts.set(stripTags(match[1]!).replace(/[:\s]+$/, '').toLowerCase(), stripTags(match[2]!));
|
||||
}
|
||||
const fact = (...labels: string[]) => {
|
||||
for (const [label, value] of facts) {
|
||||
if (labels.some((wanted) => label.includes(wanted))) return value;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const title = /id="productTitle"[^>]*>([\s\S]*?)<\/span>/i.exec(html);
|
||||
const byline = /id="bylineInfo"[^>]*>([\s\S]*?)<\/a>/i.exec(html);
|
||||
const brandText = byline
|
||||
? stripTags(byline[1]!).replace(/^(marke|brand):\s*/i, '').replace(/^besuche den\s+|-store$|\s+store$/gi, '')
|
||||
: fact('marke', 'hersteller');
|
||||
const image = /"hiRes":"(https:[^"]+)"/.exec(html) ?? /data-old-hires="(https:[^"]+)"/.exec(html);
|
||||
const pieces = Number((fact('anzahl der teile', 'teileanzahl', 'anzahl teile') ?? '').replace(/\D/g, ''));
|
||||
|
||||
return {
|
||||
manufacturer: canonicalBrand(brandText),
|
||||
setNumber: fact('modellnummer', 'herstellerreferenz')?.replace(/\s/g, ''),
|
||||
name: title ? stripTags(title[1]!) : undefined,
|
||||
ean: cleanEan(fact('ean', 'gtin')),
|
||||
imageUrl: image?.[1],
|
||||
pieceCount: pieces > 0 && pieces < 100_000 ? pieces : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function hintsFromHtml(html: string, pageUrl: URL): LinkHints {
|
||||
const hints = /(^|\.)amazon\./.test(pageUrl.hostname) ? hintsFromAmazon(html) : hintsFromStructuredData(html, pageUrl);
|
||||
|
||||
// A shop's own article number (an ASIN, a long SKU) is not a set number.
|
||||
if (hints.setNumber && !/^[\d-]{3,12}$/.test(hints.setNumber)) delete hints.setNumber;
|
||||
|
||||
// Titles usually carry the number: "LEGO City 60398 Familienhaus".
|
||||
if (!hints.setNumber && hints.name) {
|
||||
const fromName = findSetNumber(hints.name.toLowerCase().split(/[\s()[\]:,/|]+/).filter(Boolean));
|
||||
if (fromName) hints.setNumber = fromName;
|
||||
}
|
||||
return hints;
|
||||
}
|
||||
|
||||
function hintsFromStructuredData(html: string, pageUrl: URL): LinkHints {
|
||||
const product = jsonLdProducts(html)[0];
|
||||
const description = text(product?.description) ?? metaContent(html, 'og:description');
|
||||
|
||||
const hints: LinkHints = {
|
||||
manufacturer: canonicalBrand(text(product?.brand)) ?? canonicalBrand(metaContent(html, 'product:brand')),
|
||||
setNumber: text(product?.mpn) ?? text(product?.sku),
|
||||
name: text(product?.name) ?? metaContent(html, 'og:title'),
|
||||
ean: cleanEan(product?.gtin13) ?? cleanEan(product?.gtin) ?? cleanEan(product?.gtin12) ?? cleanEan(product?.gtin8),
|
||||
imageUrl: absoluteUrl(text(product?.image) ?? metaContent(html, 'og:image'), pageUrl),
|
||||
pieceCount: piecesFrom(description),
|
||||
year: yearFrom(description),
|
||||
};
|
||||
return hints;
|
||||
}
|
||||
|
||||
/** First value wins per field: pass the most trusted source first. */
|
||||
export function mergeHints(...sources: Array<LinkHints | null | undefined>): LinkHints {
|
||||
const merged: LinkHints = {};
|
||||
for (const source of sources) {
|
||||
if (!source) continue;
|
||||
for (const [field, value] of Object.entries(source) as Array<[keyof LinkHints, never]>) {
|
||||
if (merged[field] === undefined && value !== undefined && value !== '') merged[field] = value;
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shop titles carry everything: "LEGO® City Familienhaus mit Elektroauto
|
||||
* 60398", "BlueBrixx - Technisches Hilfswerk™ Vehicle | Set 108782". Keeps
|
||||
* the part that is the set's own name.
|
||||
*/
|
||||
export function cleanSetName(name: string, manufacturer?: string, setNumber?: string): string {
|
||||
let result = name.replace(/[®™©]/g, '');
|
||||
|
||||
// Site names and set references after a separator.
|
||||
result = result.split(/\s+[|•·]\s+/)[0]!;
|
||||
|
||||
// Price-comparison titles: "... ab € 4,20 (2026)", "... ab 52,99 €".
|
||||
result = result.replace(/\s+ab\s+(?:€\s*)?\d[\d.,]*\s*(?:€|EUR)?.*$/i, '');
|
||||
|
||||
if (setNumber) {
|
||||
const escaped = setNumber.replace(/[-]/g, '\\-');
|
||||
result = result.replace(new RegExp(`\\(?\\b${escaped}\\b\\)?`, 'g'), '');
|
||||
}
|
||||
|
||||
if (manufacturer) {
|
||||
for (const brand of BRANDS.filter((entry) => entry.name === manufacturer)) {
|
||||
for (const alias of [brand.name, ...brand.aliases]) {
|
||||
if (alias.length < 3) continue;
|
||||
result = result.replace(new RegExp(`^\\s*${alias}\\b\\s*[-–:]?\\s*`, 'i'), '');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result.replace(/\s{2,}/g, ' ').replace(/^[\s\-–:]+|[\s\-–:,]+$/g, '').trim();
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import { BlockList, isIP } from 'node:net';
|
||||
|
||||
/**
|
||||
* Which addresses the server may fetch on a user's behalf.
|
||||
*
|
||||
* The link import downloads pages and pictures from URLs people paste. Without
|
||||
* a guard, a link to http://192.168.1.1/ would make the server talk to the
|
||||
* router - or to any other device in the home network. Only public unicast
|
||||
* addresses pass. Pure, so it is unit tested.
|
||||
*/
|
||||
|
||||
const blocked = new BlockList();
|
||||
|
||||
for (const [network, prefix] of [
|
||||
['0.0.0.0', 8], // "this" network
|
||||
['10.0.0.0', 8], // private
|
||||
['100.64.0.0', 10], // carrier-grade NAT
|
||||
['127.0.0.0', 8], // loopback
|
||||
['169.254.0.0', 16], // link-local, cloud metadata
|
||||
['172.16.0.0', 12], // private
|
||||
['192.0.0.0', 24], // IETF protocol assignments
|
||||
['192.0.2.0', 24], // documentation
|
||||
['192.168.0.0', 16], // private
|
||||
['198.18.0.0', 15], // benchmarking
|
||||
['198.51.100.0', 24], // documentation
|
||||
['203.0.113.0', 24], // documentation
|
||||
['224.0.0.0', 4], // multicast
|
||||
['240.0.0.0', 4], // reserved, broadcast
|
||||
] as const) {
|
||||
blocked.addSubnet(network, prefix, 'ipv4');
|
||||
}
|
||||
|
||||
for (const [network, prefix] of [
|
||||
['::', 128], // unspecified
|
||||
['::1', 128], // loopback
|
||||
['64:ff9b::', 96], // NAT64 - would reach IPv4 space behind it
|
||||
['100::', 64], // discard
|
||||
['2001:db8::', 32], // documentation
|
||||
['fc00::', 7], // unique local
|
||||
['fe80::', 10], // link-local
|
||||
['ff00::', 8], // multicast
|
||||
] as const) {
|
||||
blocked.addSubnet(network, prefix, 'ipv6');
|
||||
}
|
||||
|
||||
export function isPublicAddress(address: string): boolean {
|
||||
const version = isIP(address);
|
||||
if (version === 0) return false;
|
||||
|
||||
if (version === 6) {
|
||||
// An IPv4 address in IPv6 clothing is judged as what it really is.
|
||||
const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(address);
|
||||
if (mapped) return isPublicAddress(mapped[1]!);
|
||||
return !blocked.check(address, 'ipv6');
|
||||
}
|
||||
|
||||
return !blocked.check(address, 'ipv4');
|
||||
}
|
||||
|
||||
/** Only plain web URLs on the standard ports; no credentials in the URL. */
|
||||
export function isAllowedWebUrl(url: URL): boolean {
|
||||
if (url.protocol !== 'https:' && url.protocol !== 'http:') return false;
|
||||
if (url.username || url.password) return false;
|
||||
if (url.port && url.port !== '80' && url.port !== '443') return false;
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
import 'server-only';
|
||||
import { lookup as dnsLookup, type LookupAddress } from 'node:dns';
|
||||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import { isIP, type LookupFunction } from 'node:net';
|
||||
import { Readable } from 'node:stream';
|
||||
import zlib from 'node:zlib';
|
||||
import { isAllowedWebUrl, isPublicAddress } from './net-guard';
|
||||
|
||||
/**
|
||||
* Fetches a URL a user pasted - and nothing else.
|
||||
*
|
||||
* - Every hostname is resolved first and refused if any of its addresses is
|
||||
* not public (see net-guard.ts). The check happens inside the socket's DNS
|
||||
* lookup, so a second resolution cannot swap in a private address.
|
||||
* - Redirects are followed by hand, and every hop passes the same checks.
|
||||
* - Size and time are capped, compressed bodies included.
|
||||
*/
|
||||
|
||||
export class SafeFetchError extends Error {}
|
||||
|
||||
export type SafeResponse = {
|
||||
status: number;
|
||||
url: string;
|
||||
contentType: string;
|
||||
body: Buffer;
|
||||
};
|
||||
|
||||
type Options = {
|
||||
maxBytes?: number;
|
||||
timeoutMs?: number;
|
||||
accept?: string;
|
||||
};
|
||||
|
||||
const MAX_REDIRECTS = 4;
|
||||
|
||||
// A regular browser identity: some shops answer anything else with an error
|
||||
// page. It is one request per paste, the same a browser preview would make.
|
||||
const USER_AGENT =
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0 Safari/537.36';
|
||||
|
||||
const guardedLookup = ((
|
||||
hostname: string,
|
||||
options: { all?: boolean; family?: number },
|
||||
callback: (error: Error | null, address: string | LookupAddress[], family?: number) => void,
|
||||
) => {
|
||||
dnsLookup(hostname, { family: options.family ?? 0, all: true }, (error, addresses) => {
|
||||
if (error) return callback(error, '', 4);
|
||||
if (addresses.length === 0 || addresses.some((entry) => !isPublicAddress(entry.address))) {
|
||||
return callback(new SafeFetchError(`${hostname} zeigt auf eine interne Adresse.`), '', 4);
|
||||
}
|
||||
if (options.all) return callback(null, addresses);
|
||||
callback(null, addresses[0]!.address, addresses[0]!.family);
|
||||
});
|
||||
}) as unknown as LookupFunction;
|
||||
|
||||
function checkUrl(url: URL) {
|
||||
if (!isAllowedWebUrl(url)) throw new SafeFetchError('Nur normale http- und https-Links.');
|
||||
|
||||
// Literal IPs never reach the DNS lookup, so they are checked here.
|
||||
const host = url.hostname.replace(/^\[|\]$/g, '');
|
||||
if (isIP(host) && !isPublicAddress(host)) {
|
||||
throw new SafeFetchError('Interne Adressen werden nicht abgerufen.');
|
||||
}
|
||||
}
|
||||
|
||||
function decode(response: http.IncomingMessage): Readable {
|
||||
switch ((response.headers['content-encoding'] ?? '').toLowerCase()) {
|
||||
case 'gzip':
|
||||
case 'x-gzip':
|
||||
return response.pipe(zlib.createGunzip());
|
||||
case 'deflate':
|
||||
return response.pipe(zlib.createInflate());
|
||||
case 'br':
|
||||
return response.pipe(zlib.createBrotliDecompress());
|
||||
default:
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
function requestOnce(url: URL, options: Required<Options>, signal: AbortSignal) {
|
||||
return new Promise<{ status: number; location: string | null; contentType: string; body: Buffer }>(
|
||||
(resolve, reject) => {
|
||||
const client = url.protocol === 'https:' ? https : http;
|
||||
const request = client.get(
|
||||
url,
|
||||
{
|
||||
lookup: guardedLookup,
|
||||
signal,
|
||||
headers: {
|
||||
'user-agent': USER_AGENT,
|
||||
accept: options.accept,
|
||||
'accept-language': 'de-DE,de;q=0.9,en;q=0.6',
|
||||
'accept-encoding': 'gzip, deflate, br',
|
||||
},
|
||||
},
|
||||
(response) => {
|
||||
const status = response.statusCode ?? 0;
|
||||
const location = response.headers.location ?? null;
|
||||
const contentType = response.headers['content-type'] ?? '';
|
||||
|
||||
if (status >= 300 && status < 400 && location) {
|
||||
response.resume();
|
||||
resolve({ status, location, contentType, body: Buffer.alloc(0) });
|
||||
return;
|
||||
}
|
||||
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
const stream = decode(response);
|
||||
stream.on('data', (chunk: Buffer) => {
|
||||
size += chunk.byteLength;
|
||||
if (size > options.maxBytes) {
|
||||
request.destroy();
|
||||
stream.destroy();
|
||||
reject(new SafeFetchError('Die Seite ist zu groß.'));
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
});
|
||||
stream.on('end', () =>
|
||||
resolve({ status, location: null, contentType, body: Buffer.concat(chunks) }),
|
||||
);
|
||||
stream.on('error', reject);
|
||||
},
|
||||
);
|
||||
request.on('error', reject);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export async function safeFetch(rawUrl: string | URL, options: Options = {}): Promise<SafeResponse> {
|
||||
const settings: Required<Options> = {
|
||||
maxBytes: options.maxBytes ?? 3 * 1024 * 1024,
|
||||
timeoutMs: options.timeoutMs ?? 10_000,
|
||||
accept: options.accept ?? 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.5',
|
||||
};
|
||||
const signal = AbortSignal.timeout(settings.timeoutMs);
|
||||
|
||||
let url = new URL(rawUrl);
|
||||
for (let hop = 0; hop <= MAX_REDIRECTS; hop += 1) {
|
||||
checkUrl(url);
|
||||
|
||||
let result;
|
||||
try {
|
||||
result = await requestOnce(url, settings, signal);
|
||||
} catch (error) {
|
||||
if (error instanceof SafeFetchError) throw error;
|
||||
if (signal.aborted) throw new SafeFetchError('Die Seite hat zu lange gebraucht.');
|
||||
throw new SafeFetchError('Die Seite war nicht erreichbar.');
|
||||
}
|
||||
|
||||
if (result.location) {
|
||||
url = new URL(result.location, url);
|
||||
continue;
|
||||
}
|
||||
|
||||
return {
|
||||
status: result.status,
|
||||
url: url.toString(),
|
||||
contentType: result.contentType,
|
||||
body: result.body,
|
||||
};
|
||||
}
|
||||
|
||||
throw new SafeFetchError('Zu viele Weiterleitungen.');
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
canonicalBrand,
|
||||
cleanSetName,
|
||||
hintsFromHtml,
|
||||
hintsFromUrl,
|
||||
mergeHints,
|
||||
sourceLabel,
|
||||
} from '@/lib/link-import';
|
||||
import { isAllowedWebUrl, isPublicAddress } from '@/lib/net-guard';
|
||||
|
||||
describe('Hersteller erkennen', () => {
|
||||
it('kennt die gaengigen Schreibweisen', () => {
|
||||
expect(canonicalBrand('LEGO®')).toBe('Lego');
|
||||
expect(canonicalBrand('Blue Brixx')).toBe('BlueBrixx');
|
||||
expect(canonicalBrand('BlueBrixx Pro')).toBe('BlueBrixx');
|
||||
expect(canonicalBrand('Mega Construx')).toBe('Mega Bloks');
|
||||
expect(canonicalBrand('COBI')).toBe('Cobi');
|
||||
});
|
||||
|
||||
it('raet nicht bei Unbekanntem', () => {
|
||||
expect(canonicalBrand('Ravensburger')).toBeUndefined();
|
||||
expect(canonicalBrand('')).toBeUndefined();
|
||||
expect(canonicalBrand('legoland')).toBe('Lego');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Daten aus dem Link', () => {
|
||||
it('liest Amazon-Links mit Produktnamen', () => {
|
||||
const url = new URL(
|
||||
'https://www.amazon.de/LEGO-60398-Familienhaus-Elektroauto-Spielzeug/dp/B0BYZJXGZQ/ref=sr_1_1?keywords=lego',
|
||||
);
|
||||
expect(sourceLabel(url)).toBe('Amazon');
|
||||
expect(hintsFromUrl(url)).toEqual({
|
||||
manufacturer: 'Lego',
|
||||
setNumber: '60398',
|
||||
name: 'Familienhaus Elektroauto',
|
||||
});
|
||||
});
|
||||
|
||||
it('nimmt aus einem Amazon-Link ohne Namen nicht die ASIN', () => {
|
||||
const hints = hintsFromUrl(new URL('https://www.amazon.de/dp/B0BYZJXGZQ'));
|
||||
expect(hints.setNumber).toBeUndefined();
|
||||
});
|
||||
|
||||
it('liest idealo und geizhals', () => {
|
||||
expect(
|
||||
hintsFromUrl(
|
||||
new URL(
|
||||
'https://www.idealo.de/preisvergleich/OffersOfProduct/202595377_-familienhaus-mit-elektroauto-60398-lego.html',
|
||||
),
|
||||
),
|
||||
).toEqual({ manufacturer: 'Lego', setNumber: '60398', name: 'Familienhaus mit Elektroauto' });
|
||||
|
||||
expect(
|
||||
hintsFromUrl(new URL('https://geizhals.de/bluebrixx-burg-blaustein-101400-a2101234.html')),
|
||||
).toEqual({ manufacturer: 'BlueBrixx', setNumber: '101400', name: 'Burg Blaustein' });
|
||||
});
|
||||
|
||||
it('erkennt Cobi-Nummern mit Bindestrich und Jahreszahlen nicht als Setnummer', () => {
|
||||
expect(hintsFromUrl(new URL('https://example.com/shop/cobi-2-30500-ford-mustang-gt')).setNumber).toBe(
|
||||
'2-30500',
|
||||
);
|
||||
expect(
|
||||
hintsFromUrl(new URL('https://example.com/p/playmobil-adventskalender-2024-71472')).setNumber,
|
||||
).toBe('71472');
|
||||
});
|
||||
|
||||
it('liest Marke und Nummer aus Merlins-Bricks-Adressen', () => {
|
||||
expect(hintsFromUrl(new URL('https://www.merlinsbricks.com/sets/bb-108782/'))).toEqual({
|
||||
manufacturer: 'BlueBrixx',
|
||||
setNumber: '108782',
|
||||
});
|
||||
expect(hintsFromUrl(new URL('https://www.merlinsbricks.com/sets/cobi-2400'))).toEqual({
|
||||
manufacturer: 'Cobi',
|
||||
setNumber: '2400',
|
||||
});
|
||||
});
|
||||
|
||||
it('erkennt den Hersteller am Shop', () => {
|
||||
expect(hintsFromUrl(new URL('https://www.bluebrixx.com/de/star-trek/108123/raumstation')).manufacturer).toBe(
|
||||
'BlueBrixx',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Daten aus der Seite', () => {
|
||||
const page = new URL('https://brickfact.com/de-de/sets/lego-city-familienhaus-60398');
|
||||
|
||||
it('nimmt schema.org-Produktdaten', () => {
|
||||
const html = `<html><head>
|
||||
<script type="application/ld+json">{"@context":"https://schema.org","@graph":[
|
||||
{"@type":"WebSite","name":"Brickfact"},
|
||||
{"@type":"Product","name":"LEGO® City Familienhaus mit Elektroauto 60398",
|
||||
"sku":"60398","mpn":"60398","gtin13":"5702017462363",
|
||||
"brand":{"@type":"Brand","name":"LEGO"},
|
||||
"image":["/img/60398.webp"],
|
||||
"description":"Ein Set mit 462 Teilen aus 2023."}]}
|
||||
</script></head></html>`;
|
||||
|
||||
expect(hintsFromHtml(html, page)).toEqual({
|
||||
manufacturer: 'Lego',
|
||||
setNumber: '60398',
|
||||
name: 'LEGO® City Familienhaus mit Elektroauto 60398',
|
||||
ean: '5702017462363',
|
||||
imageUrl: 'https://brickfact.com/img/60398.webp',
|
||||
pieceCount: 462,
|
||||
year: 2023,
|
||||
});
|
||||
});
|
||||
|
||||
it('faellt auf Open Graph zurueck und verwirft ungueltige EAN', () => {
|
||||
const html = `<meta property="og:title" content="Burg Blaustein & Zubehör">
|
||||
<meta content="https://cdn.example.com/burg.jpg" property="og:image">
|
||||
<script type="application/ld+json">{"@type":"Product","gtin13":"1234567890123"}</script>`;
|
||||
|
||||
const hints = hintsFromHtml(html, new URL('https://example.com/burg'));
|
||||
expect(hints.name).toBe('Burg Blaustein & Zubehör');
|
||||
expect(hints.imageUrl).toBe('https://cdn.example.com/burg.jpg');
|
||||
expect(hints.ean).toBeUndefined();
|
||||
});
|
||||
|
||||
it('haelt Shop-Artikelnummern nicht fuer Setnummern', () => {
|
||||
const html = '<script type="application/ld+json">{"@type":"Product","sku":"B0BYZJXGZQ"}</script>';
|
||||
expect(hintsFromHtml(html, page).setNumber).toBeUndefined();
|
||||
});
|
||||
|
||||
it('liest Amazon-Seiten ohne schema.org', () => {
|
||||
const html = `
|
||||
<span id="productTitle" class="a-size-large"> LEGO City Familienhaus mit Elektroauto, 60398 </span>
|
||||
<a id="bylineInfo" href="/stores/LEGO">Besuche den LEGO-Store</a>
|
||||
<script>var data = {"hiRes":"https://m.media-amazon.com/images/I/81abc.jpg","thumb":"x"};</script>
|
||||
<table><tr><th class="prodDetSectionEntry"> Modellnummer </th><td> ‎60398 </td></tr>
|
||||
<tr><th class="prodDetSectionEntry"> Anzahl der Teile </th><td> 462 </td></tr></table>`;
|
||||
|
||||
expect(hintsFromHtml(html, new URL('https://www.amazon.de/dp/B0BYZJXGZQ'))).toEqual({
|
||||
manufacturer: 'Lego',
|
||||
setNumber: '60398',
|
||||
name: 'LEGO City Familienhaus mit Elektroauto, 60398',
|
||||
ean: undefined,
|
||||
imageUrl: 'https://m.media-amazon.com/images/I/81abc.jpg',
|
||||
pieceCount: 462,
|
||||
});
|
||||
});
|
||||
|
||||
it('findet die Setnummer im Produktnamen', () => {
|
||||
const html = '<meta property="og:title" content="Cobi 2400 Ford GP ab € 12,90 (2026)">';
|
||||
const hints = hintsFromHtml(html, new URL('https://geizhals.de/x-a1.html'));
|
||||
expect(hints.setNumber).toBe('2400');
|
||||
expect(cleanSetName(hints.name!, 'Cobi', hints.setNumber)).toBe('Ford GP');
|
||||
});
|
||||
|
||||
it('uebersteht kaputtes JSON', () => {
|
||||
const html = '<script type="application/ld+json">{ kaputt </script><meta property="og:title" content="X">';
|
||||
expect(hintsFromHtml(html, page).name).toBe('X');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Zusammenfuehren und Namen putzen', () => {
|
||||
it('nimmt pro Feld die erste Quelle mit Wert', () => {
|
||||
expect(
|
||||
mergeHints({ name: 'Seite', pieceCount: undefined }, { name: 'Link', pieceCount: 450 }, null),
|
||||
).toEqual({ name: 'Seite', pieceCount: 450 });
|
||||
});
|
||||
|
||||
it('entfernt Marke, Nummer und Seitennamen', () => {
|
||||
expect(cleanSetName('LEGO® City Familienhaus mit Elektroauto 60398', 'Lego', '60398')).toBe(
|
||||
'City Familienhaus mit Elektroauto',
|
||||
);
|
||||
expect(
|
||||
cleanSetName('BlueBrixx - Technisches Hilfswerk™ Bi-Directional Vehicle | Set 108782', 'BlueBrixx', '108782'),
|
||||
).toBe('Technisches Hilfswerk Bi-Directional Vehicle');
|
||||
expect(cleanSetName('Cobi Ford GP • Set 2400 • SetDB • Merlins Bricks', 'Cobi', '2400')).toBe('Ford GP');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Nur oeffentliche Adressen', () => {
|
||||
it('sperrt Heimnetz, Loopback und Metadaten-Adressen', () => {
|
||||
for (const address of [
|
||||
'127.0.0.1',
|
||||
'10.0.0.5',
|
||||
'172.16.3.4',
|
||||
'192.168.178.1',
|
||||
'169.254.169.254',
|
||||
'100.64.0.1',
|
||||
'0.0.0.0',
|
||||
'::1',
|
||||
'fd00::1',
|
||||
'fe80::1',
|
||||
'::ffff:192.168.1.1',
|
||||
]) {
|
||||
expect(isPublicAddress(address), address).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it('laesst oeffentliche Adressen durch', () => {
|
||||
expect(isPublicAddress('93.184.216.34')).toBe(true);
|
||||
expect(isPublicAddress('2a00:1450:4001:80b::200e')).toBe(true);
|
||||
expect(isPublicAddress('::ffff:93.184.216.34')).toBe(true);
|
||||
});
|
||||
|
||||
it('erlaubt nur normale Web-Links', () => {
|
||||
expect(isAllowedWebUrl(new URL('https://example.com/a'))).toBe(true);
|
||||
expect(isAllowedWebUrl(new URL('http://example.com:80/a'))).toBe(true);
|
||||
expect(isAllowedWebUrl(new URL('file:///etc/passwd'))).toBe(false);
|
||||
expect(isAllowedWebUrl(new URL('https://example.com:8443/'))).toBe(false);
|
||||
expect(isAllowedWebUrl(new URL('https://user:pw@example.com/'))).toBe(false);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user