blog/scripts/wikipedia.js

// Requires axios and rate-limiter-flexible
var axios = require("axios");
var RateLimiterMemory = require("rate-limiter-flexible").RateLimiterMemory;

function buildArgsHash(args) {
    let argsHash = {};
    args.forEach(arg => {
        const params = arg.split(':');
        argsHash[params[0]] = params.slice(1).join(':');
    });

    return argsHash;
}

let gRateLimiter = new RateLimiterMemory({
    points: 200,
    duration: 1,
}); // Per wikipedia's recommendation, 200 reqs per second

async function requestWikipediaRequest(baseUrl) {
    while (true) {
        try {
            await gRateLimiter.consume(baseUrl, 1);
            break;
        } catch (limited) {
            if (limited instanceof Error) {
                throw limited;
            } else {
                await new Promise(resolve => {
                    setTimeout(resolve, limited.msBeforeNext);
                });
            }
        }
    }
}

async function requestWikipediaDone(baseUrl) {
    await gRateLimiter.reward(baseUrl, 1);
}

async function generatePrefetchedWikipediaTagHtml(args, content) {
    const argsHash = buildArgsHash(args);
    const title = argsHash['title'];
    const escapedTitle = encodeURIComponent(title);

    const lang = argsHash['lang'] !== undefined ? argsHash['lang'] : 'en';
    const baseUrl = `https://${lang}.wikipedia.org`;

    const url = `${baseUrl}/api/rest_v1/page/summary/${escapedTitle}`;
    await requestWikipediaRequest(baseUrl);
    let response = await axios.get(url, {
        headers: {
            'accept': 'application/json; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/Summary/1.4.2"',
            'api-user-agent': `Hexo Wikipedia Tag (Prefetch, from ${hexo.config.url})`,
        },
        responseType: 'json',
        timeout: 3000,
        params: {
            redirect: false,
        },
        transitional: {
            silentJSONParsing: false,
        },
    }).catch(reason => {
        hexo.log.warn(`fetch failed for "${url}": ${reason}`);
        return Promise.reject(reason);
    });
    await requestWikipediaDone(baseUrl);
    let extractedText = response.data.extract;
    let contentText = extractedText;
    if (argsHash['wikiButton'] === 'true') {
        contentText += `<p><a href="${baseUrl}/wiki/${title}">Wikipedia:${title}</a></p>`;
    }
    return `<blockquote>${contentText}</blockquote>`
}

function generateWikipediaTagHtml(args, content) {
    const argsHash = buildArgsHash(args);
    const title = argsHash['title'];
    const escapedTitle = encodeURIComponent(title);

    const lang = argsHash['lang'] !== undefined ? argsHash['lang'] : 'en';
    const baseUrl = `https://${lang}.wikipedia.org`;

    const url = `${baseUrl}/api/rest_v1/page/summary/${escapedTitle}?redirect=false`;

    const tagId = "wikipedia-"+Math.round(Math.random() * 100000);
    const embeddedScript = `
      window.addEventListener('load', function() {
        var element = document.getElementById('${tagId}');
        var req = new XMLHttpRequest();
        req.addEventListener("load", function() {
          var result = this.response;
          const extract = result.extract;
          element.prepend(extract);
        });
        req.addEventListener("error", function() {
          element.prepend('Failed to fetch wikipedia data for "${title}".');
        });
        req.open('GET', '${url}');
        req.responseType = 'json';
        req.setRequestHeader('accept', 'application/json; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/Summary/1.4.2"');
        req.setRequestHeader('api-user-agent', 'Hexo Wikipedia Tag (from ${hexo.config.url})');
        req.send();
      });
    `;
    let contentText = `<script>${embeddedScript}</script>`;
    if (argsHash['wikiButton'] === 'true') {
        contentText += `<p><a href="${baseUrl}/wiki/${title}">Wikipedia:${title}</a></p>`;
    } else {
        contentText += `<noscript><p><a href="${baseUrl}/wiki/${title}">Wikipedia:${title}</a></p></noscript>`;
    }

    return `<blockquote id='${tagId}'>${contentText}</blockquote>`;
}

function wikipediaTag(args, content) {
    // We use the client-fetching method as fallback
    return generatePrefetchedWikipediaTagHtml(args, content).catch(reason => {
        return generateWikipediaTagHtml(args, content);
    });
}

hexo.extend.tag.register('wikipedia', wikipediaTag, {async: true});