admin/kiss-translator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task pool

Browse Source
This commit is contained in:
Gabe Yuan
2023-08-04 16:05:14 +08:00
parent 3631219b92
commit 7d2fafcd0e
11 changed files with 13621 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13094
dist/chrome/content.js vendored
View File

File diff suppressed because one or more lines are too long

142
public/content.html
View File

@@ -1,7 +1,6 @@
<!DOCTYPE html>
<html lang="en">
<head>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>%REACT_APP_NAME%</title>
@@ -16,11 +15,143 @@
max-height: 1.2em;
}
</style>
</head>
</head>
<body>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root">
<h2>React is a JavaScript library for building user interfaces.</h2>
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<h2>React is a JavaScript library for building user interfaces.</h2>
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<h2>React is a JavaScript library for building user interfaces.</h2>
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<h2>React is a JavaScript library for building user interfaces.</h2>
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<br />
<div class="cont cont1">
<h2>React is a JavaScript library for building user interfaces.</h2>
<ul>
@@ -211,6 +342,5 @@
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</body>
</html>

56
src/apis/index.js
View File

@@ -36,7 +36,7 @@ export const apiSyncData = async (url, key, data) =>
* @param {*} from
* @returns
*/
const apiGoogleTranslate = async (text, to, from) => {
const apiGoogleTranslate = async (translator, text, to, from) => {
const params = {
client: "gtx",
dt: "t",
@@ -48,14 +48,15 @@ const apiGoogleTranslate = async (text, to, from) => {
};
const { googleUrl } = await getSetting();
const input = `${googleUrl}?${queryString.stringify(params)}`;
return fetchPolyfill(input, {
useCache: true,
usePool: true,
return fetchPolyfill(
input,
{
headers: {
"Content-type": "application/json",
"X-Translator": OPT_TRANS_GOOGLE,
},
});
},
{ useCache: true, translator }
);
};
/**
@@ -65,23 +66,25 @@ const apiGoogleTranslate = async (text, to, from) => {
* @param {*} from
* @returns
*/
const apiMicrosoftTranslate = (text, to, from) => {
const apiMicrosoftTranslate = (translator, text, to, from, token) => {
const params = {
from,
to,
"api-version": "3.0",
};
const input = `${URL_MICROSOFT_TRANS}?${queryString.stringify(params)}`;
return fetchPolyfill(input, {
useCache: true,
usePool: true,
return fetchPolyfill(
input,
{
headers: {
"Content-type": "application/json",
"X-Translator": OPT_TRANS_MICROSOFT,
Authorization: `Bearer ${token}`,
},
method: "POST",
body: JSON.stringify([{ Text: text }]),
});
},
{ useCache: true, translator }
);
};
/**
@@ -91,17 +94,19 @@ const apiMicrosoftTranslate = (text, to, from) => {
* @param {*} from
* @returns
*/
const apiOpenaiTranslate = async (text, to, from) => {
const { openaiUrl, openaiModel, openaiPrompt } = await getSetting();
const apiOpenaiTranslate = async (translator, text, to, from) => {
const { openaiUrl, openaiKey, openaiModel, openaiPrompt } =
await getSetting();
let prompt = openaiPrompt
.replaceAll(PROMPT_PLACE_FROM, from)
.replaceAll(PROMPT_PLACE_TO, to);
return fetchPolyfill(openaiUrl, {
useCache: true,
usePool: true,
return fetchPolyfill(
openaiUrl,
{
headers: {
"Content-type": "application/json",
"X-Translator": OPT_TRANS_OPENAI,
Authorization: `Bearer ${openaiKey}`, // OpenAI
"api-key": openaiKey, // Azure OpenAI
},
method: "POST",
body: JSON.stringify({
@@ -119,7 +124,9 @@ const apiOpenaiTranslate = async (text, to, from) => {
temperature: 0,
max_tokens: 256,
}),
});
},
{ useCache: true, translator }
);
};
/**
@@ -127,7 +134,10 @@ const apiOpenaiTranslate = async (text, to, from) => {
* @param {*} param0
* @returns
*/
export const apiTranslate = async ({ translator, q, fromLang, toLang }) => {
export const apiTranslate = async (
{ translator, q, fromLang, toLang },
{ token }
) => {
let trText = "";
let isSame = false;
@@ -135,15 +145,15 @@ export const apiTranslate = async ({ translator, q, fromLang, toLang }) => {
let to = OPT_LANGS_SPECIAL?.[translator]?.get(toLang) ?? toLang;
if (translator === OPT_TRANS_GOOGLE) {
const res = await apiGoogleTranslate(q, to, from);
const res = await apiGoogleTranslate(translator, q, to, from);
trText = res.sentences.map((item) => item.trans).join(" ");
isSame = to === res.src;
} else if (translator === OPT_TRANS_MICROSOFT) {
const res = await apiMicrosoftTranslate(q, to, from);
const res = await apiMicrosoftTranslate(translator, q, to, from, token);
trText = res[0].translations[0].text;
isSame = to === res[0].detectedLanguage.language;
} else if (translator === OPT_TRANS_OPENAI) {
const res = await apiOpenaiTranslate(q, to, from);
const res = await apiOpenaiTranslate(translator, q, to, from);
trText = res?.choices?.[0].message.content;
isSame = (await detectLang(q)) === (await detectLang(trText));
}

9
src/background.js
View File

@@ -1,7 +1,6 @@
import browser from "webextension-polyfill";
import {
MSG_FETCH,
MSG_FETCH_LIMIT,
DEFAULT_SETTING,
DEFAULT_RULES,
DEFAULT_SYNC,
@@ -10,7 +9,7 @@ import {
STOKEY_SYNC,
CACHE_NAME,
} from "./config";
import { fetchData, setFetchLimit } from "./libs/fetch";
import { fetchData } from "./libs/fetch";
import storage from "./libs/storage";
import { getSetting } from "./libs";
import { syncAll } from "./libs/sync";
@@ -48,7 +47,7 @@ browser.runtime.onMessage.addListener(
({ action, args }, sender, sendResponse) => {
switch (action) {
case MSG_FETCH:
fetchData(args.input, args.init)
fetchData(args.input, args.init, args.opts)
.then((data) => {
sendResponse({ data });
})
@@ -56,10 +55,6 @@ browser.runtime.onMessage.addListener(
sendResponse({ error: error.message });
});
break;
case MSG_FETCH_LIMIT:
setFetchLimit(args.limit);
sendResponse({ data: "ok" });
break;
default:
sendResponse({ error: `message action is unavailable: ${action}` });
}

8
src/config/i18n.js
View File

@@ -43,8 +43,12 @@ export const I18N = {
en: `Interface Language`,
},
fetch_limit: {
zh: `并发请求数量`,
en: `Concurrent Requests Limit`,
zh: `最大请求数量`,
en: `Maximum Number Of Request`,
},
fetch_interval: {
zh: `请求间隔时间(ms)`,
en: `Request Interval(ms)`,
},
translate_service: {
zh: `翻译服务`,

9
src/config/index.js
View File

@@ -109,8 +109,8 @@ export const OPT_STYLE_ALL = [
OPT_STYLE_HIGHTLIGHT,
];
export const DEFAULT_FETCH_LIMIT = 1; // 默认并发请求数
export const DEFAULT_FETCH_INTERVAL = 500; // 默认请求间隔时间
export const DEFAULT_FETCH_LIMIT = 10; // 默认最大任务数量
export const DEFAULT_FETCH_INTERVAL = 500; // 默认任务间隔时间
export const PROMPT_PLACE_FROM = "{{from}}"; // 占位符
export const PROMPT_PLACE_TO = "{{to}}"; // 占位符
@@ -122,13 +122,14 @@ export const DEFAULT_RULE = {
fromLang: "auto",
toLang: "zh-CN",
textStyle: OPT_STYLE_DASHLINE,
transOpen: false,
transOpen: true,
};
export const DEFAULT_SETTING = {
darkMode: false, // 深色模式
uiLang: "zh", // 界面语言
fetchLimit: DEFAULT_FETCH_LIMIT, // 请求并发数量
fetchLimit: DEFAULT_FETCH_LIMIT, // 最大任务数量
fetchInterval: DEFAULT_FETCH_INTERVAL, // 任务间隔时间
clearCache: false, // 是否在浏览器下次启动时清除缓存
googleUrl: "https://translate.googleapis.com/translate_a/single", // 谷歌翻译接口
openaiUrl: "https://api.openai.com/v1/chat/completions",

5
src/content.js
View File

@@ -11,6 +11,8 @@ import {
import Content from "./views/Content";
import { StoragesProvider } from "./hooks/Storage";
import { queryEls, getRules, matchRule } from "./libs";
import { getSetting } from "./libs";
import { transPool } from "./libs/pool";
/**
* 翻译类
@@ -127,6 +129,9 @@ class Translator {
* 入口函数
*/
(async () => {
const { fetchInterval, fetchLimit } = await getSetting();
transPool.update(fetchInterval, fetchLimit);
const rules = await getRules();
const rule = matchRule(rules, document.location.href);
const translator = new Translator(rule);

17
src/hooks/Translate.js
View File

@@ -1,14 +1,8 @@
import { useEffect } from "react";
import { useState } from "react";
import { apiTranslate } from "../apis";
import { transPool } from "../libs/pool";
import browser from "../libs/browser";
import {
MSG_TRANS_PUTRULE,
DEFAULT_FETCH_LIMIT,
MSG_FETCH_LIMIT,
} from "../config";
import { useSetting } from "./Setting";
import { sendMsg } from "../libs/msg";
import { MSG_TRANS_PUTRULE } from "../config";
import { detectLang } from "../libs";
/**
@@ -21,7 +15,6 @@ export function useTranslate(q, initRule) {
const [loading, setLoading] = useState(false);
const [sameLang, setSamelang] = useState(false);
const [rule, setRule] = useState(initRule);
const { fetchLimit = DEFAULT_FETCH_LIMIT } = useSetting() || {};
const { translator, fromLang, toLang, textStyle } = rule;
@@ -39,10 +32,6 @@ export function useTranslate(q, initRule) {
};
}, []);
useEffect(() => {
sendMsg(MSG_FETCH_LIMIT, { limit: fetchLimit });
}, [fetchLimit]);
useEffect(() => {
(async () => {
try {
@@ -51,7 +40,7 @@ export function useTranslate(q, initRule) {
if (toLang.includes(deLang)) {
setSamelang(true);
} else {
const [trText, isSame] = await apiTranslate({
const [trText, isSame] = await transPool.push({
translator,
q,
fromLang,

119
src/libs/fetch.js
View File

@@ -2,117 +2,34 @@ import browser from "./browser";
import { sendMsg } from "./msg";
import {
MSG_FETCH,
DEFAULT_FETCH_LIMIT,
DEFAULT_FETCH_INTERVAL,
CACHE_NAME,
OPT_TRANS_MICROSOFT,
OPT_TRANS_OPENAI,
} from "../config";
import { msAuth } from "./auth";
import { getSetting } from ".";
/**
* request 改造，因缓存必须是GET方法
* @param {*} request
* @returns
*/
const newCacheReq = async (request) => {
if (request.method === "GET") {
return request;
const newCacheReq = async (request, translator) => {
if (translator === OPT_TRANS_MICROSOFT) {
request.headers.delete("Authorization");
} else if (translator === OPT_TRANS_OPENAI) {
request.headers.delete("Authorization");
request.headers.delete("api-key");
}
const body = await request.clone().text();
if (request.method !== "GET") {
const body = await request.text();
const cacheUrl = new URL(request.url);
cacheUrl.pathname += body;
return new Request(cacheUrl.toString(), { method: "GET" });
};
/**
* request 改造，根据不同翻译服务
* @param {*} request
* @returns
*/
const newReq = async (request) => {
const translator = request.headers.get("X-Translator");
if (translator === OPT_TRANS_MICROSOFT) {
const [token] = await msAuth();
request.headers.set("Authorization", `Bearer ${token}`);
} else if (translator === OPT_TRANS_OPENAI) {
const { openaiKey } = await getSetting();
request.headers.set("Authorization", `Bearer ${openaiKey}`); // OpenAI
request.headers.set("api-key", openaiKey); // Azure OpenAI
request = new Request(cacheUrl.toString(), { method: "GET" });
}
request.headers.delete("X-Translator");
return request;
};
/**
* 请求池
* @param {*} l
* @param {*} t
* @returns
*/
const _fetchPool = (l = 1, t = 1000) => {
let limitCount = l; // 限制并发数量
const intervalTime = t; // 请求间隔时间
const pool = []; // 请求池
const maxRetry = 2; // 最大重试次数
let currentCount = 0; // 当前请求数量
setInterval(async () => {
const count = limitCount - currentCount;
if (pool.length === 0 || count <= 0) {
return;
}
for (let i = 0; i < count; i++) {
const item = pool.shift();
if (item) {
const { request, resolve, reject, retry } = item;
currentCount++;
try {
const req = await request();
const res = await fetch(req);
resolve(res);
} catch (err) {
if (retry < maxRetry) {
pool.push({ request, resolve, reject, retry: retry + 1 });
} else {
reject(err);
}
} finally {
currentCount--;
}
}
}
}, intervalTime);
return [
async (req, usePool) => {
const request = () => newReq(req.clone());
if (usePool) {
return new Promise((resolve, reject) => {
pool.push({ request, resolve, reject, retry: 0 });
});
} else {
return fetch(await request());
}
},
(limit = -1) => {
if (limit >= 1 && limit <= 10 && limitCount !== limit) {
limitCount = limit;
}
},
];
};
export const [_fetch, setFetchLimit] = _fetchPool(
DEFAULT_FETCH_LIMIT,
DEFAULT_FETCH_INTERVAL
);
/**
* 调用fetch接口
* @param {*} input
@@ -121,15 +38,17 @@ export const [_fetch, setFetchLimit] = _fetchPool(
*/
export const fetchData = async (
input,
{ useCache = false, usePool = false, ...init } = {}
init,
{ useCache = false, translator } = {}
) => {
const req = new Request(input, init);
const cacheReq = await newCacheReq(req);
const cacheReq = await newCacheReq(req.clone(), translator);
const cache = await caches.open(CACHE_NAME);
let res;
// 查询缓存
if (useCache) {
// console.log("usecache")
try {
res = await cache.match(cacheReq);
} catch (err) {
@@ -139,7 +58,8 @@ export const fetchData = async (
// 发送请求
if (!res) {
res = await _fetch(req, usePool);
// console.log("usefetch")
res = await fetch(req);
}
if (!res?.ok) {
@@ -166,12 +86,13 @@ export const fetchData = async (
* 兼容性封装
* @param {*} input
* @param {*} init
* @param {*} opts
* @returns
*/
export const fetchPolyfill = async (input, init) => {
export const fetchPolyfill = async (input, init, opts) => {
if (browser?.runtime) {
// 插件调用
const res = await sendMsg(MSG_FETCH, { input, init });
const res = await sendMsg(MSG_FETCH, { input, init, opts });
if (res.error) {
throw new Error(res.error);
}
@@ -179,5 +100,5 @@ export const fetchPolyfill = async (input, init) => {
}
// 网页直接调用
return await fetchData(input, init);
return await fetchData(input, init, opts);
};

86
src/libs/pool.js Normal file
View File

@@ -0,0 +1,86 @@
import {
DEFAULT_FETCH_INTERVAL,
DEFAULT_FETCH_LIMIT,
OPT_TRANS_MICROSOFT,
} from "../config";
import { apiTranslate } from "../apis";
import { msAuth } from "./auth";
const _taskPool = (fn, preFn, _interval = 500, _limit = 100) => {
const pool = [];
const maxRetry = 2; // 最大重试次数
let maxCount = _limit; // 最大数量
let curCount = 0; // 当前数量
let interval = _interval; // 间隔时间
let timer;
const handleTask = async (item, preArgs) => {
curCount++;
const { args, resolve, reject, retry } = item;
try {
const res = await fn(args, preArgs);
resolve(res);
} catch (err) {
if (retry < maxRetry) {
pool.push({ args, resolve, reject, retry: retry + 1 });
} else {
reject(err);
}
} finally {
curCount--;
}
};
(async function run() {
// console.log("timer", timer);
if (curCount < maxCount) {
const item = pool.shift();
if (item) {
try {
const preArgs = await preFn(item.args);
handleTask(item, preArgs);
} catch (err) {
console.log("[preFn]", err);
pool.push(item);
}
}
}
timer && clearTimeout(timer);
timer = setTimeout(run, interval);
})();
return {
push: async (args) => {
return new Promise((resolve, reject) => {
pool.push({ args, resolve, reject, retry: 0 });
});
},
update: (_interval = 500, _limit = 100) => {
if (_interval >= 0 && _interval <= 5000 && _interval !== interval) {
interval = _interval;
}
if (_limit >= 1 && _limit <= 100 && _limit !== maxCount) {
maxCount = _limit;
}
},
clear: () => {
pool.length = 0;
curCount = 0;
timer && clearTimeout(timer);
},
};
};
export const transPool = _taskPool(
apiTranslate,
async ({ translator }) => {
if (translator === OPT_TRANS_MICROSOFT) {
const [token] = await msAuth();
return { token };
}
return {};
},
DEFAULT_FETCH_INTERVAL,
DEFAULT_FETCH_LIMIT
);

15
src/views/Options/Setting.js
View File

@@ -23,6 +23,7 @@ export default function Settings() {
uiLang,
googleUrl,
fetchLimit,
fetchInterval,
openaiUrl,
openaiKey,
openaiModel,
@@ -59,7 +60,19 @@ export default function Settings() {
defaultValue={fetchLimit}
onChange={(e) => {
updateSetting({
fetchLimit: limitNumber(e.target.value, 1, 10),
fetchLimit: limitNumber(e.target.value, 1, 100),
});
}}
/>
<TextField
size="small"
label={i18n("fetch_interval")}
type="number"
defaultValue={fetchInterval}
onChange={(e) => {
updateSetting({
fetchInterval: limitNumber(e.target.value, 0, 5000),
});
}}
/>