diff --git a/demo/kickstarter_popular.js b/demo/kickstarter_popular.js index fadacc9..4715897 100644 --- a/demo/kickstarter_popular.js +++ b/demo/kickstarter_popular.js @@ -1,4 +1,4 @@ -const request = require('request-promise-native') +const puppeteer = require('puppeteer'); const crawl = require('../index') const setting = { @@ -8,63 +8,67 @@ const setting = { check: ['equal', '404'] }], type: 'list', - container: '#projects_list .project-card', - // listOption: [ 'limit', 3 ], - listOption: [ 'range', 0, 10 ], - // listOption: [ 'ignore', 0, 2, -1 ], - // listOption: [ 'focus', 3, -3 ], + container: '#projects .js-track-project-card', + // listOption: ['limit', 3], + listOption: ['range', 0, 10], + // listOption: ['ignore', 0, 2, -1], + // listOption: ['focus', 3, -3], crawl: { projectID: { - get: 'data-pid', + get: 'data-project_pid', }, name: { - elem: '.project-title', - get: 'text', + elem: 'h3', + get: 'text', }, image: { - elem: '.project-thumbnail img', - get: 'src' + elem: '.self-start img', + get: 'src' }, link: { - elem: '.project-title a', - get: 'href', + elem: '.self-start a', + get: 'href', process: [ - [ 'split', '?', 0 ], - [ 'prepend', 'https://www.kickstarter.com' ] + ['split', '?', 0], ] }, description: { - elem: '.project-blurb', - get: 'text' + get: 'data-project_description' }, - funded: { - elem: '.project-stats-value:eq(0)', - get: 'text' + pledged: { + elem: '[data-test-id="amount-pledged"]', + get: 'text' }, percentPledged: { - elem: '.project-percent-pledged', - get: 'style', - process: [ - [ 'split', /:\s?/g, 1 ] - ] + elem: '[data-test-id="percent-raised"] > span:first-child', + get: 'text' + }, + type: { + elem: '[data-test-id="time-left"] + div a:first-child', + get: 'text' + }, + place: { + elem: '[data-test-id="time-left"] + div a:last-child', + get: 'text' }, - pledged: { - elem: '.money.usd', - get: 'num' - } } } -const url = 'https://www.kickstarter.com/discover/popular?ref=popular' +const url = 'https://www.kickstarter.com/discover/popular' console.info('Crawl the popular list at Kickstarter.') ;(async function () { try { console.info(`Request: ${url}\n`) - const content = await request(url) - const result = content ? await crawl(content, setting) : 'No content' + const browser = await puppeteer.launch() + const page = await browser.newPage() + await page.setUserAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36') + await page.goto(url) + const bodyHTML = await page.content() + const result = bodyHTML ? await crawl(bodyHTML, setting) : 'No content' console.log('Result:', result) console.log('Total:', result.length) + await browser.close() } catch (err) { console.error(err) } })() diff --git a/demo/steam_Dota2.js b/demo/steam_Dota2.js index 4f1fb80..16c0dc8 100644 --- a/demo/steam_Dota2.js +++ b/demo/steam_Dota2.js @@ -1,4 +1,4 @@ -const request = require('request-promise-native') +const axios = require('axios') const crawl = require('../index') const setting = { @@ -11,7 +11,7 @@ const setting = { }, appName: { outOfContainer: true, - elem: '.apphub_AppName', + elem: '#appHubAppName', get: 'text' }, image: { @@ -66,8 +66,8 @@ console.info('Crawl Dota 2 description at Steam site.') ;(async function () { try { console.info(`Request: ${url}\n`) - const content = await request(url) - const result = content ? await crawl(content, setting) : 'No content' + const res = await axios.get(url) + const result = res.data ? await crawl(res.data, setting) : 'No content' console.log('Result:', result) } catch (err) { console.error(err) }