Rubicon/blog
1
0
Fork 0
Code Activity Actions

pre-fetched wikipedia summary

Browse source
This commit is contained in:
thisLight 2021-12-31 20:12:43 +08:00
parent 781759d545
commit e5d64e7153
3 changed files with 96 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

78
scripts/wikipedia.js
View file

@ -1,3 +1,6 @@
// Requires axios and rate-limiter-flexible
var axios = require("axios");
var RateLimiterMemory = require("rate-limiter-flexible").RateLimiterMemory;
function buildArgsHash(args) {
let argsHash = {};
@ -9,6 +12,67 @@ function buildArgsHash(args) {
return argsHash;
}
let gRateLimiter = new RateLimiterMemory({
points: 200,
duration: 1,
}); // Per wikipedia's recommendation, 200 reqs per second
async function requestWikipediaRequest(baseUrl) {
while (true) {
try {
await gRateLimiter.consume(baseUrl, 1);
break;
} catch (limited) {
if (limited instanceof Error) {
throw limited;
} else {
await new Promise(resolve => {
setTimeout(resolve, limited.msBeforeNext);
});
}
}
}
}
async function requestWikipediaDone(baseUrl) {
await gRateLimiter.reward(baseUrl, 1);
}
async function generatePrefetchedWikipediaTagHtml(args, content) {
const argsHash = buildArgsHash(args);
const title = argsHash['title'];
const lang = argsHash['lang'] !== undefined ? argsHash['lang'] : 'en';
const baseUrl = `https://${lang}.wikipedia.org`;
const url = `${baseUrl}/api/rest_v1/page/summary/${title}`;
await requestWikipediaRequest(baseUrl);
let response = await axios.get(url, {
headers: {
'accept': 'application/json; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/Summary/1.4.2"',
'api-user-agent': `Hexo Wikipedia Tag (Prefetch, from ${hexo.config.url})`,
},
responseType: 'json',
timeout: 3000,
params: {
redirect: false,
},
transitional: {
silentJSONParsing: false,
},
}).catch(reason => {
hexo.log.warn(`fetch failed for "${url}": ${reason}`);
return Promise.reject(reason);
});
await requestWikipediaDone(baseUrl);
let extractedText = response.data.extract;
let contentText = extractedText;
if (argsHash['wikiButton'] === 'true') {
contentText += `<p><a href="${baseUrl}/wiki/${title}">Wikipedia:${title}</a></p>`;
}
return `<blockquote>${contentText}</blockquote>`
}
function generateWikipediaTagHtml(args, content) {
const argsHash = buildArgsHash(args);
const title = argsHash['title'];
@ -18,7 +82,7 @@ function generateWikipediaTagHtml(args, content) {
const url = `${baseUrl}/api/rest_v1/page/summary/${title}?redirect=false`;
const tagId = Math.round(Math.random() * 100000);
const tagId = "wikipedia-"+Math.round(Math.random() * 100000);
const embeddedScript = `
window.addEventListener('load', function() {
var element = document.getElementById('${tagId}');
@ -34,15 +98,25 @@ function generateWikipediaTagHtml(args, content) {
req.open('GET', '${url}');
req.responseType = 'json';
req.setRequestHeader('accept', 'application/json; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/Summary/1.4.2"');
req.setRequestHeader('api-user-agent', 'Hexo Wikipedia Tag (from ${hexo.config.url})');
req.send();
});
`;
let contentText = `<script>${embeddedScript}</script>`;
if (argsHash['wikiButton'] === 'true') {
contentText += `<p><a href="${baseUrl}/wiki/${title}">Wikipedia:${title}</a></p>`;
} else {
contentText += `<noscript><p><a href="${baseUrl}/wiki/${title}">Wikipedia:${title}</a></p></noscript>`;
}
return `<blockquote id='${tagId}'>${contentText}</blockquote>`;
}
hexo.extend.tag.register('wikipedia', generateWikipediaTagHtml);
function wikipediaTag(args, content) {
// We use the client-fetching method as fallback
return generatePrefetchedWikipediaTagHtml(args, content).catch(reason => {
return generateWikipediaTagHtml(args, content);
});
}
hexo.extend.tag.register('wikipedia', wikipediaTag, {async: true});