diff --git a/src/backends/contracts.ts b/src/backends/contracts.ts new file mode 100644 index 0000000..9e78172 --- /dev/null +++ b/src/backends/contracts.ts @@ -0,0 +1,50 @@ +export type BackendId = 'qianwen' | 'yuanbao' | (string & {}); + +export type BackendLoginStatus = 'logged-in' | 'logged-out' | 'unknown'; + +/** 主进程可安全使用的后端静态信息。 */ +export interface BackendDefinition { + readonly id: BackendId; + readonly label: string; + readonly url: string; + readonly origins: readonly string[]; + readonly partition: string; + readonly editorSelector: string; +} + +export interface BackendPageStatus { + readonly loginStatus: BackendLoginStatus; + /** 页面已经具备开始语音识别所需的编辑器和麦克风入口。 */ + readonly ready: boolean; + readonly message?: string; +} + +/** 仅在后端网页 preload 的渲染进程中调用。 */ +export interface BackendWebAdapter { + readonly definition: BackendDefinition; + readonly stopDelayMs: number; + findEditor(document: Document): HTMLElement | null; + findMicrophone(document: Document): HTMLElement | null; + serialize(editor: HTMLElement): string; + detectPageStatus(document: Document): BackendPageStatus; +} + +export interface BackendControlCommand { + readonly backend: BackendId; + readonly action: 'start' | 'stop' | 'cancel' | 'refresh-status'; +} + +export interface BackendTranscriptPayload { + readonly backend: BackendId; + readonly text: string; + readonly revision: number; +} + +export function backendOwnsUrl(definition: BackendDefinition, candidate: string): boolean { + try { + const origin = new URL(candidate).origin; + return definition.origins.includes(origin); + } catch { + return false; + } +} diff --git a/src/backends/definitions.ts b/src/backends/definitions.ts new file mode 100644 index 0000000..64c5201 --- /dev/null +++ b/src/backends/definitions.ts @@ -0,0 +1,24 @@ +import type { BackendDefinition } from './contracts'; + +export const qianwenDefinition: BackendDefinition = Object.freeze({ + id: 'qianwen', + label: '千问', + url: 'https://www.qianwen.com/', + origins: Object.freeze(['https://www.qianwen.com']), + partition: 'persist:meloasr-qianwen', + editorSelector: '[data-slate-editor="true"][contenteditable="true"], [role="textbox"][contenteditable="true"]' +}); + +export const yuanbaoDefinition: BackendDefinition = Object.freeze({ + id: 'yuanbao', + label: '元宝', + url: 'https://yuanbao.tencent.com/', + origins: Object.freeze(['https://yuanbao.tencent.com']), + partition: 'persist:meloasr-yuanbao', + editorSelector: '.ql-editor[contenteditable="true"], [contenteditable="true"][role="textbox"]' +}); + +export const backendDefinitions = Object.freeze([ + qianwenDefinition, + yuanbaoDefinition +] as const); diff --git a/src/backends/dom.ts b/src/backends/dom.ts new file mode 100644 index 0000000..18cd268 --- /dev/null +++ b/src/backends/dom.ts @@ -0,0 +1,58 @@ +import type { BackendLoginStatus, BackendPageStatus } from './contracts'; + +export function isVisible(element: Element | null): element is HTMLElement { + if (!(element instanceof HTMLElement)) return false; + const style = getComputedStyle(element); + const rect = element.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' && rect.width > 0 && rect.height > 0; +} + +export function isUsableControl(element: Element | null): element is HTMLElement { + return isVisible(element) && !element.matches('[disabled], [aria-disabled="true"]'); +} + +export function firstVisible(document: Document, selector: string): HTMLElement | null { + const elements = Array.from(document.querySelectorAll(selector)); + return elements.find(isVisible) ?? (elements[0] instanceof HTMLElement ? elements[0] : null); +} + +export function controlDescription(element: Element): string { + return [ + element.getAttribute('aria-label'), + element.getAttribute('title'), + element.getAttribute('data-tooltip'), + element.textContent, + element.getAttribute('class'), + element.querySelector('svg')?.getAttribute('class') + ].filter(Boolean).join(' '); +} + +export function findSemanticControl(document: Document, pattern: RegExp): HTMLElement | null { + const control = Array.from(document.querySelectorAll('button, a, [role="button"]')) + .find((element) => pattern.test(controlDescription(element)) && isUsableControl(element)); + return control instanceof HTMLElement ? control : null; +} + +export function operationalStatus( + document: Document, + editor: HTMLElement | null, + microphone: HTMLElement | null, + loginPattern: RegExp +): BackendPageStatus { + if (editor && microphone) return { loginStatus: 'logged-in', ready: true }; + if (editor) { + return { + loginStatus: 'logged-in', + ready: false, + message: '语音入口尚未就绪' + }; + } + + const loginControl = findSemanticControl(document, loginPattern); + const loginStatus: BackendLoginStatus = loginControl ? 'logged-out' : 'unknown'; + return { + loginStatus, + ready: false, + message: loginControl ? '尚未登录' : '页面尚未进入可语音输入状态' + }; +} diff --git a/src/backends/index.ts b/src/backends/index.ts new file mode 100644 index 0000000..d9281c1 --- /dev/null +++ b/src/backends/index.ts @@ -0,0 +1,15 @@ +export { + getBackend, + listBackends, + requireBackend +} from './registry'; +export type { + BackendControlCommand, + BackendDefinition, + BackendId, + BackendLoginStatus, + BackendPageStatus, + BackendTranscriptPayload, + BackendWebAdapter +} from './contracts'; +export { backendOwnsUrl } from './contracts'; diff --git a/src/backends/qianwen.ts b/src/backends/qianwen.ts new file mode 100644 index 0000000..4f85749 --- /dev/null +++ b/src/backends/qianwen.ts @@ -0,0 +1,40 @@ +import type { BackendWebAdapter } from './contracts'; +import { qianwenDefinition } from './definitions'; +import { firstVisible, isUsableControl, operationalStatus } from './dom'; + +export const qianwenAdapter: BackendWebAdapter = { + definition: qianwenDefinition, + stopDelayMs: 1_600, + + findEditor(document) { + return firstVisible(document, qianwenDefinition.editorSelector); + }, + + findMicrophone(document) { + const anchor = document.querySelector('[data-global-speaking-guide-anchor]'); + if (isUsableControl(anchor)) return anchor; + const icon = document.querySelector('[data-icon-type="qwpcicon-mic"]'); + const control = icon?.closest('button, [role="button"], [data-global-speaking-guide-anchor]') ?? icon?.parentElement; + return control instanceof HTMLElement && isUsableControl(control) ? control : null; + }, + + serialize(editor) { + return Array.from(editor.children) + .filter((node) => node.nodeType === Node.ELEMENT_NODE) + .map((block) => { + const clone = block.cloneNode(true) as HTMLElement; + clone.querySelectorAll('[data-slate-placeholder], [data-slate-zero-width]').forEach((node) => node.remove()); + return clone.textContent ?? ''; + }) + .join('\n'); + }, + + detectPageStatus(document) { + return operationalStatus( + document, + this.findEditor(document), + this.findMicrophone(document), + /登录|注册|log\s*in|sign\s*in/i + ); + } +}; diff --git a/src/backends/registry.ts b/src/backends/registry.ts new file mode 100644 index 0000000..e4205b7 --- /dev/null +++ b/src/backends/registry.ts @@ -0,0 +1,22 @@ +import type { BackendDefinition, BackendId } from './contracts'; +import { backendDefinitions } from './definitions'; + +export type { BackendDefinition, BackendId, BackendWebAdapter } from './contracts'; + +const definitionById = new Map( + backendDefinitions.map((definition) => [definition.id, definition] as const) +); + +export function listBackends(): readonly BackendDefinition[] { + return backendDefinitions; +} + +export function getBackend(id: string): BackendDefinition | undefined { + return definitionById.get(id); +} + +export function requireBackend(id: string): BackendDefinition { + const definition = getBackend(id); + if (!definition) throw new Error(`未知后端:${id}`); + return definition; +} diff --git a/src/backends/web-registry.ts b/src/backends/web-registry.ts new file mode 100644 index 0000000..84f0951 --- /dev/null +++ b/src/backends/web-registry.ts @@ -0,0 +1,13 @@ +import type { BackendId, BackendWebAdapter } from './contracts'; +import { qianwenAdapter } from './qianwen'; +import { yuanbaoAdapter } from './yuanbao'; + +const adapterById = new Map( + [qianwenAdapter, yuanbaoAdapter] + .map((adapter) => [adapter.definition.id, adapter] as const) +); + +/** 仅供后端网页 preload 使用。 */ +export function getBackendWebAdapter(id: string): BackendWebAdapter | undefined { + return adapterById.get(id); +} diff --git a/src/backends/yuanbao.ts b/src/backends/yuanbao.ts new file mode 100644 index 0000000..f025b42 --- /dev/null +++ b/src/backends/yuanbao.ts @@ -0,0 +1,49 @@ +import type { BackendWebAdapter } from './contracts'; +import { yuanbaoDefinition } from './definitions'; +import { controlDescription, firstVisible, isUsableControl, operationalStatus } from './dom'; + +export const yuanbaoAdapter: BackendWebAdapter = { + definition: yuanbaoDefinition, + stopDelayMs: 3_500, + + findEditor(document) { + return firstVisible(document, yuanbaoDefinition.editorSelector); + }, + + findMicrophone(document) { + const directSelectors = [ + '[data-testid*="voice-input" i]', + '[data-testid*="voice" i]', + '[class*="voice-input" i] button', + '[class*="voiceInput"] button', + 'button[class*="voice" i]', + '[role="button"][class*="voice" i]' + ]; + for (const selector of directSelectors) { + const element = document.querySelector(selector); + if (isUsableControl(element)) return element; + } + + const control = Array.from(document.querySelectorAll('button, [role="button"]')) + .find((element) => + /语音输入|开始语音|停止语音|voice.?input|microphone|mic\b/i.test(controlDescription(element)) && + isUsableControl(element) + ); + return control instanceof HTMLElement ? control : null; + }, + + serialize(editor) { + const clone = editor.cloneNode(true) as HTMLElement; + const text = clone.innerText ?? clone.textContent ?? ''; + return text.replace(/\n$/, ''); + }, + + detectPageStatus(document) { + return operationalStatus( + document, + this.findEditor(document), + this.findMicrophone(document), + /登录|微信登录|QQ登录|手机号登录|log\s*in|sign\s*in/i + ); + } +}; diff --git a/src/preload/backend.ts b/src/preload/backend.ts new file mode 100644 index 0000000..2cd7dad --- /dev/null +++ b/src/preload/backend.ts @@ -0,0 +1,175 @@ +import { ipcRenderer } from 'electron'; +import type { BackendControlCommand, BackendPageStatus, BackendTranscriptPayload } from '../backends/contracts'; +import { getBackendWebAdapter } from '../backends/web-registry'; + +const backendId = process.argv + .find((argument) => argument.startsWith('--meloasr-backend=')) + ?.slice('--meloasr-backend='.length) ?? ''; +const adapter = getBackendWebAdapter(backendId); + +let editorObserver: MutationObserver | undefined; +let pageObserver: MutationObserver | undefined; +let active = false; +let stopping = false; +let lastSpeechText = ''; +let emitScheduled = false; +let stopTimer: ReturnType | undefined; +let statusTimer: ReturnType | undefined; +let transcriptRevision = 0; +let lastStatusKey = ''; + +function emit(channel: string, value: Record = {}): void { + ipcRenderer.send(channel, { backend: backendId, ...value }); +} + +function statusKey(status: BackendPageStatus): string { + return JSON.stringify([status.loginStatus, status.ready, status.message ?? '']); +} + +function emitPageStatus(force = false): void { + if (!adapter) return; + const status = adapter.detectPageStatus(document); + const key = statusKey(status); + if (!force && key === lastStatusKey) return; + lastStatusKey = key; + emit('backend-status', { + ready: status.ready, + loggedIn: status.loginStatus === 'unknown' ? null : status.loginStatus === 'logged-in', + detail: status.message ?? '' + }); +} + +function schedulePageStatus(): void { + clearTimeout(statusTimer); + statusTimer = setTimeout(() => emitPageStatus(), 200); +} + +function emitCurrentEditorText(): void { + emitScheduled = false; + if (!active || !adapter) return; + const editor = adapter.findEditor(document); + if (!editor) { + fail(`${adapter.definition.label}输入框在录音期间消失,本轮已停止`); + return; + } + const text = adapter.serialize(editor); + if (text === lastSpeechText) return; + lastSpeechText = text; + transcriptRevision += 1; + const payload: Omit = { text, revision: transcriptRevision }; + emit('backend-transcript', payload); +} + +function scheduleTranscript(): void { + if (emitScheduled || !active) return; + emitScheduled = true; + requestAnimationFrame(emitCurrentEditorText); +} + +function observeEditor(editor: HTMLElement): void { + editorObserver?.disconnect(); + editorObserver = new MutationObserver(scheduleTranscript); + editorObserver.observe(editor, { + subtree: true, + childList: true, + characterData: true, + attributes: true, + attributeFilter: ['class'] + }); +} + +function placeCaretAtEnd(editor: HTMLElement): void { + editor.focus({ preventScroll: true }); + const selection = window.getSelection(); + if (!selection) return; + const range = document.createRange(); + range.selectNodeContents(editor); + range.collapse(false); + selection.removeAllRanges(); + selection.addRange(range); +} + +function resetSession(): void { + clearTimeout(stopTimer); + active = false; + stopping = false; + editorObserver?.disconnect(); +} + +function fail(message: string): void { + resetSession(); + emit('backend-start-error', { message }); + emitPageStatus(true); +} + +function startSpeech(): void { + if (!adapter) return; + const status = adapter.detectPageStatus(document); + if (!status.ready) { + fail(status.loginStatus === 'logged-out' + ? `${adapter.definition.label}尚未登录,请先打开登录页面` + : `${adapter.definition.label}页面尚未就绪,请打开后端页面并进入对话页`); + return; + } + + const editor = adapter.findEditor(document); + const microphone = adapter.findMicrophone(document); + if (!editor || !microphone) { + fail(`没有找到${adapter.definition.label}输入框或麦克风按钮`); + return; + } + + clearTimeout(stopTimer); + placeCaretAtEnd(editor); + lastSpeechText = adapter.serialize(editor); + transcriptRevision = 0; + active = true; + stopping = false; + observeEditor(editor); + microphone.click(); +} + +function stopSpeech(): void { + if (!active || stopping || !adapter) return; + stopping = true; + adapter.findMicrophone(document)?.click(); + clearTimeout(stopTimer); + stopTimer = setTimeout(() => { + emitCurrentEditorText(); + resetSession(); + emit('backend-session-ended', { text: lastSpeechText, revision: transcriptRevision }); + emitPageStatus(true); + }, adapter.stopDelayMs); +} + +function cancelSpeech(): void { + if (!active || !adapter) return; + adapter.findMicrophone(document)?.click(); + resetSession(); + lastSpeechText = ''; +} + +ipcRenderer.on('backend-control', (_event, command: BackendControlCommand) => { + if (!adapter || command.backend !== backendId) return; + if (command.action === 'start') startSpeech(); + if (command.action === 'stop') stopSpeech(); + if (command.action === 'cancel') cancelSpeech(); + if (command.action === 'refresh-status') emitPageStatus(true); +}); + +if (adapter) { + const startStatusObserver = (): void => { + emitPageStatus(true); + pageObserver?.disconnect(); + pageObserver = new MutationObserver(schedulePageStatus); + pageObserver.observe(document.documentElement, { subtree: true, childList: true }); + window.setInterval(() => emitPageStatus(), 5_000); + }; + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', startStatusObserver, { once: true }); + } else { + startStatusObserver(); + } +} else { + emit('backend-start-error', { message: `未知后端:${backendId || '未指定'}` }); +} diff --git a/test/backends.test.ts b/test/backends.test.ts new file mode 100644 index 0000000..57b068a --- /dev/null +++ b/test/backends.test.ts @@ -0,0 +1,20 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { backendOwnsUrl } from '../src/backends/contracts'; +import { getBackend, listBackends, requireBackend } from '../src/backends/registry'; + +test('后端注册表包含千问和元宝且 ID 唯一', () => { + const backends = listBackends(); + assert.deepEqual(backends.map(({ id }) => id), ['qianwen', 'yuanbao']); + assert.equal(new Set(backends.map(({ id }) => id)).size, backends.length); + assert.equal(getBackend('qianwen')?.label, '千问'); + assert.equal(getBackend('missing'), undefined); + assert.throws(() => requireBackend('missing'), /未知后端/); +}); + +test('后端 URL 校验使用精确 origin', () => { + const qianwen = requireBackend('qianwen'); + assert.equal(backendOwnsUrl(qianwen, 'https://www.qianwen.com/chat/1'), true); + assert.equal(backendOwnsUrl(qianwen, 'https://www.qianwen.com.evil.example/'), false); + assert.equal(backendOwnsUrl(qianwen, 'not a url'), false); +});