From c462aa4ee8b56d4e977d927787ae9e19ccd0e1eb Mon Sep 17 00:00:00 2001 From: Lauren Tan Date: Thu, 24 Apr 2025 11:56:09 -0400 Subject: [PATCH] [mcp] Convert docs resource to tool Seems to work better as a tool. Also it now returns plaintext instead of markdown. --- .../packages/react-mcp-server/package.json | 4 +- .../packages/react-mcp-server/src/index.ts | 93 ++++++------------- .../react-mcp-server/src/utils/algolia.ts | 32 ++++++- compiler/yarn.lock | 81 ++++++++++++---- 4 files changed, 122 insertions(+), 88 deletions(-) diff --git a/compiler/packages/react-mcp-server/package.json b/compiler/packages/react-mcp-server/package.json index d4c270513a..faa2b7f5ed 100644 --- a/compiler/packages/react-mcp-server/package.json +++ b/compiler/packages/react-mcp-server/package.json @@ -19,12 +19,12 @@ "@modelcontextprotocol/sdk": "^1.9.0", "algoliasearch": "^5.23.3", "cheerio": "^1.0.0", + "html-to-text": "^9.0.5", "prettier": "^3.3.3", - "turndown": "^7.2.0", "zod": "^3.23.8" }, "devDependencies": { - "@types/turndown": "^5.0.5" + "@types/html-to-text": "^9.0.4" }, "license": "MIT", "repository": { diff --git a/compiler/packages/react-mcp-server/src/index.ts b/compiler/packages/react-mcp-server/src/index.ts index 6938c3ee64..8359bc218a 100644 --- a/compiler/packages/react-mcp-server/src/index.ts +++ b/compiler/packages/react-mcp-server/src/index.ts @@ -5,10 +5,7 @@ * LICENSE file in the root directory of this source tree. */ -import { - McpServer, - ResourceTemplate, -} from '@modelcontextprotocol/sdk/server/mcp.js'; +import {McpServer} from '@modelcontextprotocol/sdk/server/mcp.js'; import {StdioServerTransport} from '@modelcontextprotocol/sdk/server/stdio.js'; import {z} from 'zod'; import {compile, type PrintedCompilerPipelineValue} from './compiler'; @@ -20,81 +17,43 @@ import { SourceLocation, } from 'babel-plugin-react-compiler/src'; import * as cheerio from 'cheerio'; -import TurndownService from 'turndown'; import {queryAlgolia} from './utils/algolia'; import assertExhaustive from './utils/assertExhaustive'; +import {convert} from 'html-to-text'; -const turndownService = new TurndownService(); const server = new McpServer({ name: 'React', version: '0.0.0', }); -function slugify(heading: string): string { - return heading - .split(' ') - .map(w => w.toLowerCase()) - .join('-'); -} - -// TODO: how to verify this works? -server.resource( +server.tool( 'docs', - new ResourceTemplate('docs://{message}', {list: undefined}), - async (_uri, {message}) => { - const hits = await queryAlgolia(message); - const deduped = new Map(); - for (const hit of hits) { - // drop hashes to dedupe properly - const u = new URL(hit.url); - if (deduped.has(u.pathname)) { - continue; + 'Search docs from react.dev', + { + query: z.string(), + }, + async ({query}) => { + const pages = await queryAlgolia(query); + const content = pages.map(html => { + const $ = cheerio.load(html); + // react.dev should always have at least one
with the main content + const article = $('article').html(); + if (article != null) { + return { + type: 'text' as const, + text: convert(article), + }; + } else { + return { + type: 'text' as const, + // Fallback to converting the whole page to text. + text: convert($.html()), + }; } - deduped.set(u.pathname, hit); - } - const pages: Array = await Promise.all( - Array.from(deduped.values()).map(hit => { - return fetch(hit.url, { - headers: { - 'User-Agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36', - }, - }).then(res => { - if (res.ok === true) { - return res.text(); - } else { - console.error( - `Could not fetch docs: ${res.status} ${res.statusText}`, - ); - return null; - } - }); - }), - ); - - const resultsMarkdown = pages - .filter(html => html !== null) - .map(html => { - const $ = cheerio.load(html); - const title = encodeURIComponent(slugify($('h1').text())); - // react.dev should always have at least one
with the main content - const article = $('article').html(); - if (article != null) { - return { - uri: `docs://${title}`, - text: turndownService.turndown(article), - }; - } else { - return { - uri: `docs://${title}`, - // Fallback to converting the whole page to markdown - text: turndownService.turndown($.html()), - }; - } - }); + }); return { - contents: resultsMarkdown, + content, }; }, ); diff --git a/compiler/packages/react-mcp-server/src/utils/algolia.ts b/compiler/packages/react-mcp-server/src/utils/algolia.ts index 9baed23138..cfc08022db 100644 --- a/compiler/packages/react-mcp-server/src/utils/algolia.ts +++ b/compiler/packages/react-mcp-server/src/utils/algolia.ts @@ -44,7 +44,7 @@ export function printHierarchy( export async function queryAlgolia( message: string | Array, -): Promise[]> { +): Promise> { const {results} = await ALGOLIA_CLIENT.search({ requests: [ { @@ -87,5 +87,33 @@ export async function queryAlgolia( }); const firstResult = results[0] as SearchResponse; const {hits} = firstResult; - return hits; + const deduped = new Map(); + for (const hit of hits) { + // drop hashes to dedupe properly + const u = new URL(hit.url); + if (deduped.has(u.pathname)) { + continue; + } + deduped.set(u.pathname, hit); + } + const pages: Array = await Promise.all( + Array.from(deduped.values()).map(hit => { + return fetch(hit.url, { + headers: { + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36', + }, + }).then(res => { + if (res.ok === true) { + return res.text(); + } else { + console.error( + `Could not fetch docs: ${res.status} ${res.statusText}`, + ); + return null; + } + }); + }), + ); + return pages.filter(page => page !== null); } diff --git a/compiler/yarn.lock b/compiler/yarn.lock index 410651315d..a60ae668ed 100644 --- a/compiler/yarn.lock +++ b/compiler/yarn.lock @@ -2909,11 +2909,6 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" -"@mixmark-io/domino@^2.2.0": - version "2.2.0" - resolved "https://registry.yarnpkg.com/@mixmark-io/domino/-/domino-2.2.0.tgz#4e8ec69bf1afeb7a14f0628b7e2c0f35bdb336c3" - integrity sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw== - "@modelcontextprotocol/sdk@^1.9.0": version "1.9.0" resolved "https://registry.yarnpkg.com/@modelcontextprotocol/sdk/-/sdk-1.9.0.tgz#1bf7a4843870b81da26983b8e69bf398d87055f1" @@ -3061,6 +3056,14 @@ resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.34.9.tgz#1973871850856ae72bc678aeb066ab952330e923" integrity sha512-AyleYRPU7+rgkMWbEh71fQlrzRfeP6SyMnRf9XX4fCdDPAJumdSBqYEcWPMzVQ4ScAl7E4oFfK0GUVn77xSwbw== +"@selderee/plugin-htmlparser2@^0.11.0": + version "0.11.0" + resolved "https://registry.yarnpkg.com/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz#d5b5e29a7ba6d3958a1972c7be16f4b2c188c517" + integrity sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ== + dependencies: + domhandler "^5.0.3" + selderee "^0.11.0" + "@sideway/address@^4.1.5": version "4.1.5" resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.5.tgz#4bc149a0076623ced99ca8208ba780d65a99b9d5" @@ -3257,6 +3260,11 @@ dependencies: "@types/node" "*" +"@types/html-to-text@^9.0.4": + version "9.0.4" + resolved "https://registry.yarnpkg.com/@types/html-to-text/-/html-to-text-9.0.4.tgz#4a83dd8ae8bfa91457d0b1ffc26f4d0537eff58c" + integrity sha512-pUY3cKH/Nm2yYrEmDlPR1mR7yszjGx4DrwPjQ702C4/D5CwHuZTgZdIdwPkRbcuhs7BAh2L5rg3CL5cbRiGTCQ== + "@types/invariant@^2.2.35": version "2.2.35" resolved "https://registry.yarnpkg.com/@types/invariant/-/invariant-2.2.35.tgz#cd3ebf581a6557452735688d8daba6cf0bd5a3be" @@ -3397,11 +3405,6 @@ resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.5.tgz#74fef9ffbaa198eb8b588be029f38b00299caa2c" integrity sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw== -"@types/turndown@^5.0.5": - version "5.0.5" - resolved "https://registry.yarnpkg.com/@types/turndown/-/turndown-5.0.5.tgz#614de24fc9ace4d8c0d9483ba81dc8c1976dd26f" - integrity sha512-TL2IgGgc7B5j78rIccBtlYAnkuv8nUQqhQc+DSYV5j9Be9XOcm/SKOVRuA47xAVI3680Tk9B1d8flK2GWT2+4w== - "@types/vscode@^1.96.0": version "1.96.0" resolved "https://registry.yarnpkg.com/@types/vscode/-/vscode-1.96.0.tgz#3181004bf25d71677ae4aacdd7605a3fd7edf08e" @@ -4807,6 +4810,11 @@ deepmerge@^4.2.2: resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955" integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg== +deepmerge@^4.3.1: + version "4.3.1" + resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" + integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== + defaults@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/defaults/-/defaults-1.0.4.tgz#b0b02062c1e2aa62ff5d9528f0f98baa90978d7a" @@ -6051,6 +6059,27 @@ html-escaper@^2.0.0: resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg== +html-to-text@^9.0.5: + version "9.0.5" + resolved "https://registry.yarnpkg.com/html-to-text/-/html-to-text-9.0.5.tgz#6149a0f618ae7a0db8085dca9bbf96d32bb8368d" + integrity sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg== + dependencies: + "@selderee/plugin-htmlparser2" "^0.11.0" + deepmerge "^4.3.1" + dom-serializer "^2.0.0" + htmlparser2 "^8.0.2" + selderee "^0.11.0" + +htmlparser2@^8.0.2: + version "8.0.2" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-8.0.2.tgz#f002151705b383e62433b5cf466f5b716edaec21" + integrity sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.3" + domutils "^3.0.1" + entities "^4.4.0" + htmlparser2@^9.1.0: version "9.1.0" resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-9.1.0.tgz#cdb498d8a75a51f739b61d3f718136c369bc8c23" @@ -7687,6 +7716,11 @@ kuler@^2.0.0: resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A== +leac@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/leac/-/leac-0.6.0.tgz#dcf136e382e666bd2475f44a1096061b70dc0912" + integrity sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg== + leven@2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/leven/-/leven-2.1.0.tgz#c2e7a9f772094dee9d34202ae8acce4687875580" @@ -8411,6 +8445,14 @@ parse5@^7.1.2: dependencies: entities "^4.4.0" +parseley@^0.12.0: + version "0.12.1" + resolved "https://registry.yarnpkg.com/parseley/-/parseley-0.12.1.tgz#4afd561d50215ebe259e3e7a853e62f600683aef" + integrity sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw== + dependencies: + leac "^0.6.0" + peberminta "^0.9.0" + parseurl@^1.3.3: version "1.3.3" resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" @@ -8475,6 +8517,11 @@ path-type@^4.0.0: resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== +peberminta@^0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/peberminta/-/peberminta-0.9.0.tgz#8ec9bc0eb84b7d368126e71ce9033501dca2a352" + integrity sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ== + picocolors@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" @@ -9018,6 +9065,13 @@ scheduler@0.0.0-experimental-4beb1fd8-20241118: resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.0.0-experimental-4beb1fd8-20241118.tgz#3143baa23dfb4daed6a9d0bfd44a8050a0cdab93" integrity sha512-b7GQktevD5BPcS+R5qY5se5oX4b8AHQyebWswGZBdLCmElIwR3Q+RO5EgsLOA4t5D3/TDjLm58CQG16uEB5rMA== +selderee@^0.11.0: + version "0.11.0" + resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.11.0.tgz#6af0c7983e073ad3e35787ffe20cefd9daf0ec8a" + integrity sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA== + dependencies: + parseley "^0.12.0" + semver@7.x, semver@^7.3.5: version "7.3.7" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f" @@ -9638,13 +9692,6 @@ tsup@^8.4.0: tinyglobby "^0.2.11" tree-kill "^1.2.2" -turndown@^7.2.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/turndown/-/turndown-7.2.0.tgz#67d614fe8371fb511079a93345abfd156c0ffcf4" - integrity sha512-eCZGBN4nNNqM9Owkv9HAtWRYfLA4h909E/WGAWWBpmB275ehNhZyk87/Tpvjbp0jjNl9XwCsbe6bm6CqFsgD+A== - dependencies: - "@mixmark-io/domino" "^2.2.0" - type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1"