update test code

demo/kickstarter_popular.js

@@ -1,4 +1,4 @@
-const request = require('request-promise-native')
+const puppeteer = require('puppeteer');
 const crawl = require('../index')
 
 const setting = {
@@ -8,63 +8,67 @@ const setting = {
     check: ['equal', '404']
   }],
   type: 'list',
-  container: '#projects_list .project-card',
-  // listOption: [ 'limit', 3 ],
-  listOption: [ 'range', 0, 10 ],
-  // listOption: [ 'ignore', 0, 2, -1 ],
-  // listOption: [ 'focus', 3, -3 ],
+  container: '#projects .js-track-project-card',
+  // listOption: ['limit', 3],
+  listOption: ['range', 0, 10],
+  // listOption: ['ignore', 0, 2, -1],
+  // listOption: ['focus', 3, -3],
   crawl: {
     projectID: {
-      get: 'data-pid',
+      get: 'data-project_pid',
     },
     name: {
-      elem: '.project-title',
-      get:  'text',
+      elem: 'h3',
+      get: 'text',
     },
     image: {
-      elem: '.project-thumbnail img',
-      get:  'src'
+      elem: '.self-start img',
+      get: 'src'
     },
     link: {
-      elem: '.project-title a',
-      get:  'href',
+      elem: '.self-start a',
+      get: 'href',
       process: [
-        [ 'split', '?', 0 ],
-        [ 'prepend', 'https://www.kickstarter.com' ]
+        ['split', '?', 0],
       ]
     },
     description: {
-      elem: '.project-blurb',
-      get:  'text'
+      get: 'data-project_description'
     },
-    funded: {
-      elem: '.project-stats-value:eq(0)',
-      get:  'text'
+    pledged: {
+      elem: '[data-test-id="amount-pledged"]',
+      get: 'text'
     },
     percentPledged: {
-      elem: '.project-percent-pledged',
-      get:  'style',
-      process: [
-        [ 'split', /:\s?/g, 1 ]
-      ]
+      elem: '[data-test-id="percent-raised"] > span:first-child',
+      get: 'text'
+    },
+    type: {
+      elem: '[data-test-id="time-left"] + div a:first-child',
+      get: 'text'
+    },
+    place: {
+      elem: '[data-test-id="time-left"] + div a:last-child',
+      get: 'text'
     },
-    pledged: {
-      elem: '.money.usd',
-      get:  'num'
-    }
   }
 }
 
-const url = 'https://www.kickstarter.com/discover/popular?ref=popular'
+const url = 'https://www.kickstarter.com/discover/popular'
 console.info('Crawl the popular list at Kickstarter.')
 
 ;(async function () {
   try {
     console.info(`Request: ${url}\n`)
-    const content = await request(url)
-    const result = content ? await crawl(content, setting) : 'No content'
+    const browser = await puppeteer.launch()
+    const page = await browser.newPage()
+    await page.setUserAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36')
+    await page.goto(url)
+    const bodyHTML = await page.content()
+    const result = bodyHTML ? await crawl(bodyHTML, setting) : 'No content'
 
     console.log('Result:', result)
     console.log('Total:', result.length)
+    await browser.close()
   } catch (err) { console.error(err) }
 })()

demo/steam_Dota2.js

@@ -1,4 +1,4 @@
-const request = require('request-promise-native')
+const axios = require('axios')
 const crawl = require('../index')
 
 const setting = {
@@ -11,7 +11,7 @@ const setting = {
     },
     appName: {
       outOfContainer: true,
-      elem: '.apphub_AppName',
+      elem: '#appHubAppName',
       get:  'text'
     },
     image: {
@@ -66,8 +66,8 @@ console.info('Crawl Dota 2 description at Steam site.')
 ;(async function () {
   try {
     console.info(`Request: ${url}\n`)
-    const content = await request(url)
-    const result = content ? await crawl(content, setting) : 'No content'
+    const res = await axios.get(url)
+    const result = res.data ? await crawl(res.data, setting) : 'No content'
 
     console.log('Result:', result)
   } catch (err) { console.error(err) }