Skip to content

Commit e5e3f0e

Browse files
iknoxiknox
committed
test: add Readability extraction and metadata parsing tests
Adds 16 test cases covering the new extractMainContent and extractMetadata functions: - extractMainContent (6): strips nav/sidebar/footer, preserves headings, returns null for empty body, skips when false, defaults to on in fetchAndConvertToMarkdown, handles malformed HTML. - extractMetadata (7): prefers og:title over title, falls back, extracts author/date/description/siteName, returns empty object for no metadata, handles malformed HTML. - Integration (3): YAML block prepended with Readability body, extractMetadata=false skips block, extractMainContent+metadata both false preserves full HTML.
1 parent 37e4de9 commit e5e3f0e

1 file changed

Lines changed: 219 additions & 1 deletion

File tree

‎__tests__/unit/url-reader.test.ts‎

Lines changed: 219 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import * as net from 'node:net';
1616
import { createRequire, syncBuiltinESMExports } from 'node:module';
1717
import { fileURLToPath } from 'node:url';
1818
import * as zlib from 'node:zlib';
19-
import { checkContentLength, fetchAndConvertToMarkdown } from '../../src/url-reader.js';
19+
import { checkContentLength, extractMainContent, extractMetadata, fetchAndConvertToMarkdown } from '../../src/url-reader.js';
2020
import { createUrlReaderLookup } from '../../src/proxy.js';
2121
import { urlCache } from '../../src/cache.js';
2222
import { testFunction, createTestResults, printTestSummary } from '../helpers/test-utils.js';
@@ -1994,6 +1994,224 @@ async function runTests() {
19941994
}
19951995
}, results);
19961996

1997+
// ── Readability content extraction ────────────────────────────────────────
1998+
1999+
await testFunction('extractMainContent strips nav, sidebar, and footer', async () => {
2000+
const html = `<html><body>
2001+
<nav>Navigation</nav>
2002+
<article><h1>Main Article</h1><p>This is the article body.</p></article>
2003+
<aside>Sidebar</aside>
2004+
<footer>Footer</footer>
2005+
</body></html>`;
2006+
2007+
const result = extractMainContent(html, 'http://test.local');
2008+
assert.ok(result);
2009+
assert.ok(result.includes('Main Article'), 'Expected article heading');
2010+
assert.ok(!result.includes('Navigation'), 'Expected nav to be stripped');
2011+
assert.ok(!result.includes('Sidebar'), 'Expected aside to be stripped');
2012+
assert.ok(!result.includes('Footer'), 'Expected footer to be stripped');
2013+
}, results);
2014+
2015+
await testFunction('extractMainContent returns null for empty body', async () => {
2016+
const html = '<html><body></body></html>';
2017+
const result = extractMainContent(html, 'http://test.local');
2018+
assert.equal(result, null);
2019+
}, results);
2020+
2021+
await testFunction('extractMainContent preserves heading hierarchy', async () => {
2022+
const html = `<html><body>
2023+
<article><h1>Title</h1><h2>Subtitle</h2><p>Body text.</p></article>
2024+
</body></html>`;
2025+
2026+
const result = extractMainContent(html, 'http://test.local');
2027+
assert.ok(result);
2028+
assert.ok(result.includes('Title'));
2029+
assert.ok(result.includes('Subtitle'));
2030+
}, results);
2031+
2032+
await testFunction('extractMainContent is not called when extractMainContent is false', async () => {
2033+
const mockServer = createMockServer();
2034+
urlCache.clear();
2035+
2036+
const { url, close } = await startTestServer({
2037+
body: '<html><body><nav>Nav</nav><article><h1>Article</h1></article></body></html>',
2038+
});
2039+
2040+
try {
2041+
const result = await fetchAndConvertToMarkdown(mockServer as any, url, 10000, {
2042+
extractMainContent: false,
2043+
});
2044+
assert.ok(result.includes('Nav'), 'Expected full HTML when extractMainContent is false');
2045+
assert.ok(result.includes('Article'), 'Expected article content');
2046+
} finally {
2047+
await close();
2048+
urlCache.clear();
2049+
}
2050+
}, results);
2051+
2052+
await testFunction('extractMainContent defaults to on via fetchAndConvertToMarkdown', async () => {
2053+
const mockServer = createMockServer();
2054+
urlCache.clear();
2055+
2056+
const { url, close } = await startTestServer({
2057+
body: '<html><body><nav>Nav</nav><article><h1>Article</h1></article></body></html>',
2058+
});
2059+
2060+
try {
2061+
const result = await fetchAndConvertToMarkdown(mockServer as any, url);
2062+
assert.ok(!result.includes('Nav'), 'Expected nav stripped by default');
2063+
assert.ok(result.includes('Article'), 'Expected article content');
2064+
} finally {
2065+
await close();
2066+
urlCache.clear();
2067+
}
2068+
}, results);
2069+
2070+
await testFunction('extractMainContent handles malformed HTML gracefully', async () => {
2071+
const result = extractMainContent('<html><body><p>Hello', 'http://test.local');
2072+
assert.ok(result === null || result.includes('Hello'));
2073+
}, results);
2074+
2075+
// ── metadata extraction ────────────────────────────────────────────────────
2076+
2077+
await testFunction('extractMetadata pulls title from og:title', async () => {
2078+
const html = `<html><head>
2079+
<title>Fallback Title</title>
2080+
<meta property="og:title" content="OG Title">
2081+
</head></html>`;
2082+
2083+
const meta = extractMetadata(html, 'http://test.local');
2084+
assert.equal(meta.title, 'OG Title');
2085+
}, results);
2086+
2087+
await testFunction('extractMetadata falls back to title tag', async () => {
2088+
const html = `<html><head>
2089+
<title>Document Title</title>
2090+
</head></html>`;
2091+
2092+
const meta = extractMetadata(html, 'http://test.local');
2093+
assert.equal(meta.title, 'Document Title');
2094+
}, results);
2095+
2096+
await testFunction('extractMetadata extracts author', async () => {
2097+
const html = `<html><head>
2098+
<meta name="author" content="Test Author">
2099+
</head></html>`;
2100+
2101+
const meta = extractMetadata(html, 'http://test.local');
2102+
assert.equal(meta.author, 'Test Author');
2103+
}, results);
2104+
2105+
await testFunction('extractMetadata extracts publish date', async () => {
2106+
const html = `<html><head>
2107+
<meta property="article:published_time" content="2024-01-15">
2108+
</head></html>`;
2109+
2110+
const meta = extractMetadata(html, 'http://test.local');
2111+
assert.equal(meta.publishedDate, '2024-01-15');
2112+
}, results);
2113+
2114+
await testFunction('extractMetadata extracts description', async () => {
2115+
const html = `<html><head>
2116+
<meta property="og:description" content="Test description">
2117+
</head></html>`;
2118+
2119+
const meta = extractMetadata(html, 'http://test.local');
2120+
assert.equal(meta.description, 'Test description');
2121+
}, results);
2122+
2123+
await testFunction('extractMetadata extracts site name', async () => {
2124+
const html = `<html><head>
2125+
<meta property="og:site_name" content="Test Site">
2126+
</head></html>`;
2127+
2128+
const meta = extractMetadata(html, 'http://test.local');
2129+
assert.equal(meta.siteName, 'Test Site');
2130+
}, results);
2131+
2132+
await testFunction('extractMetadata returns empty object for no metadata', async () => {
2133+
const html = '<html><head></head><body><p>No metadata here.</p></body></html>';
2134+
const meta = extractMetadata(html, 'http://test.local');
2135+
assert.deepEqual(meta, {});
2136+
}, results);
2137+
2138+
await testFunction('extractMetadata handles malformed HTML gracefully', async () => {
2139+
const meta = extractMetadata('<html><head><meta', 'http://test.local');
2140+
assert.ok(typeof meta === 'object');
2141+
}, results);
2142+
2143+
// ── metadata block integration ─────────────────────────────────────────────
2144+
2145+
await testFunction('metadata is prepended as YAML block with Readability body', async () => {
2146+
const mockServer = createMockServer();
2147+
urlCache.clear();
2148+
2149+
const html = `<html><head>
2150+
<title>Test Page</title>
2151+
<meta name="author" content="Author Name">
2152+
<meta property="og:description" content="Page description">
2153+
</head><body>
2154+
<nav>Skip this</nav>
2155+
<article><h1>Hello World</h1><p>Article body here.</p></article>
2156+
</body></html>`;
2157+
2158+
const { url, close } = await startTestServer({ body: html });
2159+
2160+
try {
2161+
const result = await fetchAndConvertToMarkdown(mockServer as any, url);
2162+
assert.ok(result.startsWith('---'), 'Expected YAML frontmatter');
2163+
assert.ok(result.includes('title: Test Page'));
2164+
assert.ok(result.includes('author: Author Name'));
2165+
assert.ok(result.includes('description: Page description'));
2166+
assert.ok(!result.includes('Skip this'), 'Expected nav stripped');
2167+
} finally {
2168+
await close();
2169+
urlCache.clear();
2170+
}
2171+
}, results);
2172+
2173+
await testFunction('extractMetadata false skips metadata block', async () => {
2174+
const mockServer = createMockServer();
2175+
urlCache.clear();
2176+
2177+
const html = `<html><head>
2178+
<meta property="og:title" content="Title">
2179+
</head><body><h1>Hello</h1></body></html>`;
2180+
2181+
const { url, close } = await startTestServer({ body: html });
2182+
2183+
try {
2184+
const result = await fetchAndConvertToMarkdown(mockServer as any, url, 10000, {
2185+
extractMetadata: false,
2186+
});
2187+
assert.ok(!result.startsWith('---'), 'Expected no YAML frontmatter');
2188+
assert.ok(result.includes('Hello'));
2189+
} finally {
2190+
await close();
2191+
urlCache.clear();
2192+
}
2193+
}, results);
2194+
2195+
await testFunction('extractMainContent false preserves full HTML when metadata is also off', async () => {
2196+
const mockServer = createMockServer();
2197+
urlCache.clear();
2198+
2199+
const html = '<html><body><nav>Nav</nav><p>Just text.</p></body></html>';
2200+
const { url, close } = await startTestServer({ body: html });
2201+
2202+
try {
2203+
const result = await fetchAndConvertToMarkdown(mockServer as any, url, 10000, {
2204+
extractMainContent: false,
2205+
extractMetadata: false,
2206+
});
2207+
assert.ok(result.includes('Nav'), 'Expected Nav to be present');
2208+
assert.ok(result.includes('Just text'));
2209+
} finally {
2210+
await close();
2211+
urlCache.clear();
2212+
}
2213+
}, results);
2214+
19972215
if (originalAllowPrivateUrls === undefined) {
19982216
delete process.env.MCP_HTTP_ALLOW_PRIVATE_URLS;
19992217
} else {

0 commit comments

Comments
 (0)