Merge pull request #110 from DIYgod/master

[pull] master from diygod:master

lib/router.js

@@ -376,7 +376,7 @@ router.get('/autotrader/:query', lazyloadRouteHandler('./routes/autotrader'));
 router.get('/geekpark/breakingnews', lazyloadRouteHandler('./routes/geekpark/breakingnews'));
 
 // 搜狗
-router.get('/sogou/doodles', lazyloadRouteHandler('./routes/sogou/doodles'));
+// router.get('/sogou/doodles', lazyloadRouteHandler('./routes/sogou/doodles'));
 
 // 香港天文台
 router.get('/hko/weather', lazyloadRouteHandler('./routes/hko/weather'));
@@ -690,7 +690,7 @@ router.get('/digitaling/articles/:category/:subcate?', lazyloadRouteHandler('./r
 router.get('/digitaling/projects/:category', lazyloadRouteHandler('./routes/digitaling/project'));
 
 // Bing壁纸
-router.get('/bing', lazyloadRouteHandler('./routes/bing/index'));
+// router.get('/bing', lazyloadRouteHandler('./routes/bing/index'));
 
 // AlgoCasts
 router.get('/algocasts', lazyloadRouteHandler('./routes/algocasts/all'));

lib/utils/got.js

@@ -14,6 +14,11 @@ const custom = got.extend({
                 options.retryCount = count;
             },
         ],
+        beforeRedirect: [
+            (options, response) => {
+                logger.http(`Redirecting to ${options.url} for ${response.requestUrl}`);
+            },
+        ],
         afterResponse: [
             (response) => {
                 try {

lib/utils/request-wrapper.js

@@ -76,9 +76,9 @@ const requestWrapper = (
         }
     }
     if (prxied) {
-        logger.debug(`Proxy for ${url}`);
+        logger.http(`Proxy for ${url}`);
     } else {
-        logger.debug(`Requesting ${url}`);
+        logger.http(`Requesting ${url}`);
     }
 
     // ua

lib/v2/apkpure/versions.js

@@ -13,7 +13,7 @@ module.exports = async (ctx) => {
     page.on('request', (request) => {
         request.resourceType() === 'document' ? request.continue() : request.abort();
     });
-    logger.debug(`Requesting ${link}`);
+    logger.http(`Requesting ${link}`);
     await page.goto(link, {
         waitUntil: 'domcontentloaded',
     });

lib/v2/baidu/maintainer.js

@@ -1,5 +1,6 @@
 module.exports = {
     '/gushitong/index': ['CaoMeiYouRen'],
+    '/search/:keyword': ['CaoMeiYouRen'],
     '/tieba/forum/good/:kw/:cid?/:sortBy?': ['u3u'],
     '/tieba/forum/:kw/:sortBy?': ['u3u'],
     '/tieba/post/:id': ['u3u'],

lib/v2/baidu/radar.js

@@ -1,6 +1,17 @@
 module.exports = {
     'baidu.com': {
         _name: '百度',
+        www: [
+            {
+                title: '搜索',
+                docs: 'https://docs.rsshub.app/routes/other#bai-du-sou-suo',
+                source: ['/'],
+                target: (params, url) => {
+                    const keyword = new URL(url).searchParams.get('wd');
+                    return `/baidu/search/${keyword}`;
+                },
+            },
+        ],
         gushitong: [
             {
                 title: '首页指数',
@@ -73,7 +84,7 @@ module.exports = {
         top: [
             {
                 title: '热搜榜单',
-                docs: 'https://docs.rsshub.app/routes/other#bai-du-re-sou',
+                docs: 'https://docs.rsshub.app/routes/other#bai-du-re-sou-bang-dan',
                 source: ['/board'],
                 target: (_, url) => `/baidu/top/${new URL(url).searchParams.get('tab')}`,
             },

lib/v2/baidu/router.js

@@ -1,5 +1,6 @@
 module.exports = (router) => {
     router.get('/gushitong/index', require('./gushitong'));
+    router.get('/search/:keyword', require('./search'));
     router.get('/tieba/forum/good/:kw/:cid?/:sortBy?', require('./tieba/forum'));
     router.get('/tieba/forum/:kw/:sortBy?', require('./tieba/forum'));
     router.get('/tieba/post/:id', require('./tieba/post'));

lib/v2/baidu/search.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { art } = require('@/utils/render');
+const path = require('path');
+const renderDescription = (description, images) => art(path.join(__dirname, './templates/description.art'), { description, images });
+const config = require('@/config').value;
+
+module.exports = async (ctx) => {
+    const { keyword } = ctx.params;
+    const url = `https://www.baidu.com/s?wd=${encodeURIComponent(keyword)}`;
+    const key = `baidu-search:${url}`;
+
+    const items = await ctx.cache.tryGet(
+        key,
+        async () => {
+            const response = (await got(url)).data;
+            const visitedLinks = new Set();
+            const $ = cheerio.load(response);
+            const contentLeft = $('#content_left');
+            const containers = contentLeft.find('.c-container');
+            return containers
+                .map((i, el) => {
+                    const element = $(el);
+                    const link = element.find('h3 a').first().attr('href');
+                    if (link && !visitedLinks.has(link)) {
+                        visitedLinks.add(link);
+                        const imgs = element
+                            .find('img')
+                            .map((_j, _el) => $(_el).attr('src'))
+                            .toArray();
+                        const description = element.find('.c-gap-top-small [class^="content-right_"]').first().text() || element.find('.c-row').first().text() || element.find('.cos-row').first().text();
+                        return {
+                            title: element.find('h3').first().text(),
+                            description: renderDescription(description, imgs),
+                            link: element.find('h3 a').first().attr('href'),
+                            author: element.find('.c-row .c-color-gray').first().text() || '',
+                        };
+                    }
+                    return null;
+                })
+                .toArray()
+                .filter((e) => e?.link);
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    ctx.state.data = {
+        title: `${keyword} - 百度搜索`,
+        description: `${keyword} - 百度搜索`,
+        link: url,
+        item: items,
+    };
+};

lib/v2/baidu/templates/description.art

@@ -0,0 +1,6 @@
+{{@ description }}
+{{if images}}
+    {{each images}}
+        <img src="{{ $value }}">
+    {{/each}}
+{{/if}}

lib/routes/bing/index.js → lib/v2/bing/daily-wallpaper.js

@@ -1,22 +1,21 @@
 const got = require('@/utils/got');
-const queryString = require('query-string');
 
 module.exports = async (ctx) => {
     const response = await got({
         method: 'get',
         prefixUrl: 'https://cn.bing.com',
         url: 'HPImageArchive.aspx',
-        searchParams: queryString.stringify({
+        searchParams: {
             format: 'js',
             idx: 0,
-            n: 7,
+            n: ctx.query.limit ? parseInt(ctx.query.limit, 10) : 7,
             mkt: 'zh-CN',
-        }),
+        },
     });
     const data = response.data;
     ctx.state.data = {
         title: 'Bing每日壁纸',
-        link: `https://cn.bing.com/`,
+        link: 'https://cn.bing.com/',
         item: data.images.map((item) => ({
             title: item.copyright,
             description: `<img src="https://cn.bing.com${item.url}">`,

lib/v2/bing/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/': ['FHYunCai'],
+    '/search/:keyword': ['CaoMeiYouRen'],
+};

lib/v2/bing/radar.js

@@ -0,0 +1,22 @@
+module.exports = {
+    'bing.com': {
+        _name: 'Bing',
+        cn: [
+            {
+                title: '每日壁纸',
+                docs: 'https://docs.rsshub.app/routes/picture#bing',
+                source: '/',
+                target: '/bing',
+            },
+            {
+                title: '搜索',
+                docs: 'https://docs.rsshub.app/routes/other#bing',
+                source: '/',
+                target: (params, url) => {
+                    const q = new URL(url).searchParams.get('q');
+                    return `/bing/search/${q}`;
+                },
+            },
+        ],
+    },
+};

lib/v2/bing/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/', require('./daily-wallpaper'));
+    router.get('/search/:keyword', require('./search'));
+};

lib/v2/bing/search.js

@@ -0,0 +1,28 @@
+const parser = require('@/utils/rss-parser');
+const { parseDate } = require('@/utils/parse-date');
+const dayjs = require('dayjs');
+const localizedFormat = require('dayjs/plugin/localizedFormat');
+require('dayjs/locale/zh-cn');
+dayjs.extend(localizedFormat);
+
+module.exports = async (ctx) => {
+    const q = ctx.params.keyword;
+    const searchParams = new URLSearchParams({
+        format: 'rss',
+        q,
+    });
+    const url = new URL('https://cn.bing.com/search');
+    url.search = searchParams.toString();
+    const data = await parser.parseURL(url.toString());
+    ctx.state.data = {
+        title: data.title,
+        link: data.link,
+        description: data.description + ' - ' + data.copyright,
+        image: data.image.url,
+        item: data.items.map((e) => ({
+            ...e,
+            description: e.content,
+            pubDate: parseDate(e.pubDate, 'dddd, DD MMM YYYY HH:mm:ss [GMT]', 'zh-cn'),
+        })),
+    };
+};

lib/v2/cw/utils.js

@@ -33,7 +33,7 @@ const getCookie = async (browser, tryGet) => {
             page.on('request', (request) => {
                 request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
             });
-            logger.debug(`Requesting ${baseUrl}/user/get/cookie-bar`);
+            logger.http(`Requesting ${baseUrl}/user/get/cookie-bar`);
             await page.goto(`${baseUrl}/user/get/cookie-bar`, {
                 waitUntil: 'domcontentloaded',
             });
@@ -55,7 +55,7 @@ const parsePage = async (path, browser, ctx) => {
         request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
     });
     await setCookies(page, cookie, 'cw.com.tw');
-    logger.debug(`Requesting ${pageUrl}`);
+    logger.http(`Requesting ${pageUrl}`);
     await page.goto(pageUrl, {
         waitUntil: 'domcontentloaded',
     });
@@ -93,7 +93,7 @@ const parseItems = (list, browser, tryGet) =>
                     request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
                 });
                 await setCookies(page, cookie, 'cw.com.tw');
-                logger.debug(`Requesting ${item.link}`);
+                logger.http(`Requesting ${item.link}`);
                 await page.goto(item.link, {
                     waitUntil: 'domcontentloaded',
                 });

lib/v2/douyin/live.js

@@ -27,7 +27,7 @@ module.exports = async (ctx) => {
                     roomInfo = await response.json();
                 }
             });
-            logger.debug(`Requesting ${pageUrl}`);
+            logger.http(`Requesting ${pageUrl}`);
             await page.goto(pageUrl, {
                 waitUntil: 'networkidle2',
             });

lib/v2/fortnite/news.js

@@ -29,7 +29,7 @@ module.exports = async (ctx) => {
     });
 
     // log manually (necessary for puppeteer)
-    logger.debug(`Requesting ${apiUrl}`);
+    logger.http(`Requesting ${apiUrl}`);
     await page.goto(apiUrl, {
         waitUntil: 'networkidle0', // if use 'domcontentloaded', `await page.content()` is necessary
     });

lib/v2/google/maintainer.js

@@ -6,6 +6,7 @@ module.exports = {
     '/fonts/:sort?': ['Fatpandac'],
     '/news/:category/:locale': ['zoenglinghou'],
     '/scholar/:query': ['HenryQW'],
+    '/search/:keyword/:language?': ['CaoMeiYouRen'],
     '/sites/:id': ['hoilc'],
     '/sites/recentChanges/:id': ['nczitzk'],
 };

lib/v2/google/radar.js

@@ -1,6 +1,18 @@
 module.exports = {
     'google.com': {
         _name: '谷歌',
+        www: [
+            {
+                title: '搜索',
+                docs: 'https://docs.rsshub.app/routes/other#google',
+                source: '/',
+                target: (params, url, document) => {
+                    const q = new URL(url).searchParams.get('q');
+                    const lang = document.documentElement.lang;
+                    return `/google/search/${q}/${lang}`;
+                },
+            },
+        ],
         chrome: [
             {
                 title: '插件更新',

lib/v2/google/router.js

@@ -6,4 +6,5 @@ module.exports = function (router) {
     router.get('/fonts/:sort?', require('./fonts'));
     router.get('/news/:category/:locale', require('./news'));
     router.get('/scholar/:query', require('./scholar'));
+    router.get('/search/:keyword/:language?', require('./search'));
 };

lib/v2/google/search.js

@@ -0,0 +1,62 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { art } = require('@/utils/render');
+const path = require('path');
+const config = require('@/config').value;
+
+const renderDescription = (description, images) => art(path.join(__dirname, './templates/description.art'), { description, images });
+
+module.exports = async (ctx) => {
+    const { keyword, language } = ctx.params;
+    const searchParams = new URLSearchParams({
+        q: keyword,
+    });
+    const tempUrl = new URL('https://www.google.com/search');
+    tempUrl.search = searchParams.toString();
+    const url = tempUrl.toString();
+    const key = `google-search:${language}:${url}`;
+    const items = await ctx.cache.tryGet(
+        key,
+        async () => {
+            const response = (
+                await got(url, {
+                    headers: {
+                        'Accept-Language': language,
+                    },
+                })
+            ).data;
+            const $ = cheerio.load(response);
+            const content = $('#rso');
+            return content
+                .find('> div')
+                .map((i, el) => {
+                    const element = $(el);
+                    const link = element.find('div > div > div > div > div > span > a').first().attr('href');
+                    const title = element.find('div > div > div> div > div > span > a > h3').first().text();
+                    const imgs = element
+                        .find('img')
+                        .map((_j, _el) => $(_el).attr('src'))
+                        .toArray();
+                    const description = element.find('div[style="-webkit-line-clamp:2"]').first().text() || element.find('div[role="heading"]').first().text();
+                    const author = element.find('div > div > div > div > div > span > a > div > div > span').first().text() || '';
+                    return {
+                        link,
+                        title,
+                        description: renderDescription(description, imgs),
+                        author,
+                    };
+                })
+                .toArray()
+                .filter((e) => e?.link);
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    ctx.state.data = {
+        title: `${keyword} - Google Search`,
+        description: `${keyword} - Google Search`,
+        link: url,
+        item: items,
+    };
+};

lib/v2/google/templates/description.art

@@ -0,0 +1,6 @@
+{{@ description }}
+{{if images}}
+    {{each images}}
+        <img src="{{ $value }}">
+    {{/each}}
+{{/if}}

lib/v2/iqiyi/video.js

@@ -18,7 +18,7 @@ module.exports = async (ctx) => {
             page.on('request', (request) => {
                 request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
             });
-            logger.debug(`Requesting ${link}`);
+            logger.http(`Requesting ${link}`);
             await page.goto(link, {
                 waitUntil: 'domcontentloaded',
             });