diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..76a4b4d --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Herman Fassett + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index fc7df45..c77ff30 100644 --- a/README.md +++ b/README.md @@ -22,14 +22,32 @@ The url query options are as follows: YouTube search query - page - Number - The page of YouTube search results + pageToken + String + (Optional) The token for page of YouTube search results. Returned by initial call + + key + String + (Optional) Search key. Required if using pageToken. Returned by initial call + -Note that if no page is specified the default is 1 +## Run with docker + +To run this project with docker, go to project root directory and run following commands. + +``` +docker build -t /youtube-scrape . +docker run -p 8080:8080 -d /youtube-scrape +``` + +## Deploy to Heroku + +[![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy) + + Here is an example call: ``` http://youtube-scrape.herokuapp.com/api/search?q=herman%20fassett&page=1 @@ -43,9 +61,27 @@ Example output: "title": "Herman Fassett", "url": "https://www.youtube.com/channel/UCsOlslmdfxy6aG3O9Xkh5kA", "snippet": "I've always loved playing around with music, and here I just upload what I come up with when I find the time and inspiration.", - "thumbnail_src": "//yt3.ggpht.com/a/AATXAJyN0lSWv-uFWlcLKD46Oq5-I8i1mbnlqnLE2-F15A=s176-c-k-c0x00ffffff-no-rj-mo", - "video_count": "16 videos", - "subscriber_count": "34 subscribers" + "thumbnail_src": "//yt3.ggpht.com/ytc/AAUvwniWIAGffEoRyJT647qKbp9A0Vt5k1FPnGdRsXjG2w=s176-c-k-c0x00ffffff-no-rj-mo", + "video_count": "18 videos", + "subscriber_count": "39 subscribers", + "verified": false + } + }, + { + "video": { + "id": "ZzATESobfOw", + "title": "Falling Gold - Herman Fassett", + "url": "https://www.youtube.com/watch?v=ZzATESobfOw", + "duration": "12:01", + "snippet": "Some gentle piano music for autumn and November and you.", + "upload_date": "2 weeks ago", + "thumbnail_src": "https://i.ytimg.com/vi/ZzATESobfOw/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCD8ZtpwSwl868wIAUb2pnGmIeTZA", + "views": "163 views" + }, + "uploader": { + "username": "Herman Fassett", + "url": "https://www.youtube.com/channel/UCsOlslmdfxy6aG3O9Xkh5kA", + "verified": false } }, { @@ -55,9 +91,9 @@ Example output: "url": "https://www.youtube.com/watch?v=xprYPd5QxhA", "duration": "3:14", "snippet": "Here's a quick cover of One Summer Night which I think was originally sung by Chelsia Chan and Kenny Bee.", - "upload_date": "4 months ago", - "thumbnail_src": "https://i.ytimg.com/vi/xprYPd5QxhA/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLC_lPQPmdmVu3icbFkKw9GyOJO9Qg", - "views": "210 views" + "upload_date": "9 months ago", + "thumbnail_src": "https://i.ytimg.com/vi/xprYPd5QxhA/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBJqrvHt08cSVDE7iKr5S-dtx_coQ", + "views": "414 views" }, "uploader": { "username": "Herman Fassett", @@ -66,16 +102,20 @@ Example output: } }, { - "playlist": { - "id": "PLCgnza08eA91orhHvNPTp_gEjECGyV9HT", - "title": "Top Tracks - Herman Fassett", - "url": "https://www.youtube.com/watch?v=IyvhwAC9TjE&list=PLCgnza08eA91orhHvNPTp_gEjECGyV9HT", - "thumbnail_src": "https://i.ytimg.com/vi/IyvhwAC9TjE/hqdefault.jpg?sqp=-oaymwEXCNACELwBSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCyUXrIe-5Outvjgt1nmAO3QGeMJw", - "video_count": "8" + "video": { + "id": "8F8LUGcGceg", + "title": "Whisper", + "url": "https://www.youtube.com/watch?v=8F8LUGcGceg", + "duration": "4:44", + "snippet": "Somehow I chose the most sibilant and plosive words for the lyrics. For piano and voice, which melody I cannot sing. Here's my ...", + "upload_date": "1 month ago", + "thumbnail_src": "https://i.ytimg.com/vi/8F8LUGcGceg/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDNWjudRN-TWAQkPnE5k5SdCdAjXA", + "views": "194 views" }, "uploader": { - "username": "Herman Fassett - Topic", - "url": "https://www.youtube.com/channel/UCDKLO7jQj_lq3_mq0ynlrAg" + "username": "Herman Fassett", + "url": "https://www.youtube.com/channel/UCsOlslmdfxy6aG3O9Xkh5kA", + "verified": false } }, { @@ -86,8 +126,8 @@ Example output: "duration": "5:09", "snippet": "This is a bit more of an experimental piece I've been putting time into here and there for many months. The instrumentation is a ...", "upload_date": "1 year ago", - "thumbnail_src": "https://i.ytimg.com/vi/UYDxk7ZKvS0/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDzl0TOq-F_nv2JjfM_VW3etBdZ2A", - "views": "89 views" + "thumbnail_src": "https://i.ytimg.com/vi/UYDxk7ZKvS0/hqdefault.jpg?sqp=-oaymwEjCOADEI4CSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDEvkQ3GUu8eFJEmFe26CFBKIH0xA", + "views": "112 views" }, "uploader": { "username": "Herman Fassett", @@ -103,25 +143,8 @@ Example output: "duration": "3:31", "snippet": "I wanted to change a few things to this piece, but I've been really busy and haven't uploaded anything in awhile. So, here it is as it ...", "upload_date": "5 years ago", - "thumbnail_src": "https://i.ytimg.com/vi/nv7xf0MUFzw/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLCfOaMBqB2Josadm0NP1y9Eka2dBQ", - "views": "177 views" - }, - "uploader": { - "username": "Herman Fassett", - "url": "https://www.youtube.com/channel/UCsOlslmdfxy6aG3O9Xkh5kA", - "verified": false - } - }, - { - "video": { - "id": "Za7OZ6MsmjU", - "title": "A Winter Day - Herman Fassett", - "url": "https://www.youtube.com/watch?v=Za7OZ6MsmjU", - "duration": "3:24", - "snippet": "Here's my Christmas / Winter / New Year piece. I created it a few days back, but made a few minor adjustments today. Image used: ...", - "upload_date": "5 years ago", - "thumbnail_src": "https://i.ytimg.com/vi/Za7OZ6MsmjU/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLBpsF7VGRrPApjRrJVocE5dA1UGvQ", - "views": "100 views" + "thumbnail_src": "https://i.ytimg.com/vi/nv7xf0MUFzw/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLC4l2eOwD0KYnB7N6WlqimFpxYz8w", + "views": "186 views" }, "uploader": { "username": "Herman Fassett", @@ -137,25 +160,8 @@ Example output: "duration": "2:13", "snippet": "I actually wrote this piece nearly 3 months ago and it was the quickest composition I've ever made. However, I've been distracted ...", "upload_date": "4 years ago", - "thumbnail_src": "https://i.ytimg.com/vi/WGYqoa0iePs/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLBhC9LVfywvpOFwlIZ0kmOHiaRvrw", - "views": "58 views" - }, - "uploader": { - "username": "Herman Fassett", - "url": "https://www.youtube.com/channel/UCsOlslmdfxy6aG3O9Xkh5kA", - "verified": false - } - }, - { - "video": { - "id": "N2RhnMPbOPc", - "title": "Our Barren Lands (Cello) - Herman Fassett", - "url": "https://www.youtube.com/watch?v=N2RhnMPbOPc", - "duration": "3:05", - "snippet": "I actually had this cello melody sort of lying around and thought I'd pull it out really quickly for this contest: ...", - "upload_date": "4 years ago", - "thumbnail_src": "https://i.ytimg.com/vi/N2RhnMPbOPc/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLBQ6kgY7npbyS9vReamwqqZzsQj4Q", - "views": "59 views" + "thumbnail_src": "https://i.ytimg.com/vi/WGYqoa0iePs/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBiGMYXJ0rCWc6RADDdi7p8UTFvBA", + "views": "62 views" }, "uploader": { "username": "Herman Fassett", @@ -170,26 +176,9 @@ Example output: "url": "https://www.youtube.com/watch?v=xDeNtHCLN88", "duration": "3:51", "snippet": "This is my entry into the #RoomieMelody challenge. I decided there can never be too many Christmas songs... Challenge origin ...", - "upload_date": "6 months ago", - "thumbnail_src": "https://i.ytimg.com/vi/xDeNtHCLN88/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDWOgUQuOCMfesf06uJj5Ra8ob46Q", - "views": "174 views" - }, - "uploader": { - "username": "Herman Fassett", - "url": "https://www.youtube.com/channel/UCsOlslmdfxy6aG3O9Xkh5kA", - "verified": false - } - }, - { - "video": { - "id": "ykYTImq7p9A", - "title": "Growth - Herman Fassett", - "url": "https://www.youtube.com/watch?v=ykYTImq7p9A", - "duration": "3:28", - "snippet": "A piece I've had in the works for a long time and have only just decided on a name. This is to signify the change around us as we ...", - "upload_date": "4 years ago", - "thumbnail_src": "https://i.ytimg.com/vi/ykYTImq7p9A/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLBgEasKTHDyoiJuaVYYDwiR2zA_4g", - "views": "69 views" + "upload_date": "11 months ago", + "thumbnail_src": "https://i.ytimg.com/vi/xDeNtHCLN88/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDpPFRUkrJSJSTTnw_rx6bBTkHaXQ", + "views": "213 views" }, "uploader": { "username": "Herman Fassett", @@ -205,8 +194,8 @@ Example output: "duration": "2:36", "snippet": "This is a song I wrote during my Thanksgiving break. I hope to be writing more pieces now that Christmas break here (or close).", "upload_date": "5 years ago", - "thumbnail_src": "https://i.ytimg.com/vi/7EY2qJ9vQHI/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDZg9edae9-S9R_qRziTaYCcFm5eQ", - "views": "292 views" + "thumbnail_src": "https://i.ytimg.com/vi/7EY2qJ9vQHI/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLDBNg5vXeK3x27lvwS96S9MAz_QiA", + "views": "305 views" }, "uploader": { "username": "Herman Fassett", @@ -214,6 +203,19 @@ Example output: "verified": false } }, + { + "playlist": { + "id": "PLCgnza08eA91orhHvNPTp_gEjECGyV9HT", + "title": "Top Tracks - Herman Fassett", + "url": "https://www.youtube.com/watch?v=_7TFT9CCN4Q&list=PLCgnza08eA91orhHvNPTp_gEjECGyV9HT", + "thumbnail_src": "https://i.ytimg.com/vi/_7TFT9CCN4Q/hqdefault.jpg?sqp=-oaymwEXCNACELwBSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLC9UMsM9KtNlfdxo4Fc9ubv14KuhQ", + "video_count": "16" + }, + "uploader": { + "username": "Herman Fassett - Topic", + "url": "https://www.youtube.com/channel/UCDKLO7jQj_lq3_mq0ynlrAg" + } + }, { "video": { "id": "fnpvgyuooPs", @@ -221,9 +223,26 @@ Example output: "url": "https://www.youtube.com/watch?v=fnpvgyuooPs", "duration": "2:01", "snippet": "My friends have been asking me what I really do all day... No, this isn't everything, but it's what I spent the majority of my Veteran's ...", - "upload_date": "5 years ago", - "thumbnail_src": "https://i.ytimg.com/vi/fnpvgyuooPs/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDRiPeann4Xydi555vL03jvld-bxA", - "views": "140 views" + "upload_date": "6 years ago", + "thumbnail_src": "https://i.ytimg.com/vi/fnpvgyuooPs/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLA36HnlcDWqhFKVHGS3PsWJk37A2g", + "views": "143 views" + }, + "uploader": { + "username": "Herman Fassett", + "url": "https://www.youtube.com/channel/UCsOlslmdfxy6aG3O9Xkh5kA", + "verified": false + } + }, + { + "video": { + "id": "nRWJdaymQdg", + "title": "Blessed are Those Who Mourn - Herman Fassett", + "url": "https://www.youtube.com/watch?v=nRWJdaymQdg", + "duration": "3:23", + "snippet": "This is a re-mixed version of a piece I wrote for the Documentary \"Echoes from The Heart\" on Gerondissa Makrina by Fiery ...", + "upload_date": "2 years ago", + "thumbnail_src": "https://i.ytimg.com/vi/nRWJdaymQdg/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLAozvZ6ZMlUkXqodXtErMXjoRrx9Q", + "views": "127 views" }, "uploader": { "username": "Herman Fassett", @@ -239,8 +258,8 @@ Example output: "duration": "2:52", "snippet": "Merry Christmas! This is my winter Christmas music this year -- actually I composed this as a gift to one of my sisters. ;) The video ...", "upload_date": "4 years ago", - "thumbnail_src": "https://i.ytimg.com/vi/d5JAd7N3mBY/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDrcOyeYxp0YXoLs4AS6iZO1Vqk2w", - "views": "61 views" + "thumbnail_src": "https://i.ytimg.com/vi/d5JAd7N3mBY/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBBfHLHK5ExHxBhq_GkSmmF7SVgsQ", + "views": "66 views" }, "uploader": { "username": "Herman Fassett", @@ -250,14 +269,14 @@ Example output: }, { "video": { - "id": "nn7Ogr2x1ho", - "title": "God Rest Ye Merry Gentlemen - Herman Fassett", - "url": "https://www.youtube.com/watch?v=nn7Ogr2x1ho", - "duration": "4:21", - "snippet": "A few hours of recording and mixing got me this. Nothing fancy, and so I was hoping to put some video story to it but yeah, didn't ...", - "upload_date": "2 years ago", - "thumbnail_src": "https://i.ytimg.com/vi/nn7Ogr2x1ho/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLCLzCWHUpiwvIViVlAFWRZVS51cgA", - "views": "52 views" + "id": "ykYTImq7p9A", + "title": "Growth - Herman Fassett", + "url": "https://www.youtube.com/watch?v=ykYTImq7p9A", + "duration": "3:28", + "snippet": "A piece I've had in the works for a long time and have only just decided on a name. This is to signify the change around us as we ...", + "upload_date": "4 years ago", + "thumbnail_src": "https://i.ytimg.com/vi/ykYTImq7p9A/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLB1PDcZ-Hm3BHuZsN5mtKmuQcnxug", + "views": "69 views" }, "uploader": { "username": "Herman Fassett", @@ -267,14 +286,14 @@ Example output: }, { "video": { - "id": "nRWJdaymQdg", - "title": "Blessed are Those Who Mourn - Herman Fassett", - "url": "https://www.youtube.com/watch?v=nRWJdaymQdg", - "duration": "3:23", - "snippet": "This is a re-mixed version of a piece I wrote for the Documentary \"Echoes from The Heart\" on Gerondissa Makrina by Fiery ...", - "upload_date": "2 years ago", - "thumbnail_src": "https://i.ytimg.com/vi/nRWJdaymQdg/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDUDXzJfA9P7WF_5LwjenGSoVbg-g", - "views": "110 views" + "id": "Za7OZ6MsmjU", + "title": "A Winter Day - Herman Fassett", + "url": "https://www.youtube.com/watch?v=Za7OZ6MsmjU", + "duration": "3:24", + "snippet": "Here's my Christmas / Winter / New Year piece. I created it a few days back, but made a few minor adjustments today. Image used: ...", + "upload_date": "5 years ago", + "thumbnail_src": "https://i.ytimg.com/vi/Za7OZ6MsmjU/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLB93vKkHO25kwjat25pHr-Lmaj5Iw", + "views": "115 views" }, "uploader": { "username": "Herman Fassett", @@ -284,14 +303,14 @@ Example output: }, { "video": { - "id": "p3REFOpIx6c", - "title": "Darkness Comes (WOT Fan Music) - Herman Fassett", - "url": "https://www.youtube.com/watch?v=p3REFOpIx6c", - "duration": "3:22", - "snippet": "This piece is kind of playing around with a very different style than I have ever done before, and I have also been reading the ...", - "upload_date": "4 years ago", - "thumbnail_src": "https://i.ytimg.com/vi/p3REFOpIx6c/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLCWOV3Dc0fGmQIyim8ObATL3Ix-8A", - "views": "79 views" + "id": "nn7Ogr2x1ho", + "title": "God Rest Ye Merry Gentlemen - Herman Fassett", + "url": "https://www.youtube.com/watch?v=nn7Ogr2x1ho", + "duration": "4:21", + "snippet": "A few hours of recording and mixing got me this. Nothing fancy, and so I was hoping to put some video story to it but yeah, didn't ...", + "upload_date": "2 years ago", + "thumbnail_src": "https://i.ytimg.com/vi/nn7Ogr2x1ho/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLAnzqzy5g0pe2S9pOrjByqeCeoT5Q", + "views": "58 views" }, "uploader": { "username": "Herman Fassett", @@ -301,87 +320,72 @@ Example output: }, { "video": { - "id": "Q5BO7SrqGng", - "title": "Brightness Davar", - "url": "https://www.youtube.com/watch?v=Q5BO7SrqGng", - "duration": "6:58", - "snippet": "Provided to YouTube by Soundrop Brightness Davar · Herman Fassett Growth ℗ 2017 Herman Fassett Released on: 2017-08-05 ...", - "upload_date": "Live", - "thumbnail_src": "https://i.ytimg.com/vi/Q5BO7SrqGng/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLD_zDfJApTvX8sb5x3Xjwva625OpQ", - "views": "No views" + "id": "p3REFOpIx6c", + "title": "Darkness Comes (WOT Fan Music) - Herman Fassett", + "url": "https://www.youtube.com/watch?v=p3REFOpIx6c", + "duration": "3:22", + "snippet": "This piece is kind of playing around with a very different style than I have ever done before, and I have also been reading the ...", + "upload_date": "4 years ago", + "thumbnail_src": "https://i.ytimg.com/vi/p3REFOpIx6c/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBmT8xuh188F3IY8qGjxJRVWzRnAQ", + "views": "86 views" }, "uploader": { - "username": "Herman Fassett - Topic", - "url": "https://www.youtube.com/channel/UCDKLO7jQj_lq3_mq0ynlrAg", + "username": "Herman Fassett", + "url": "https://www.youtube.com/channel/UCsOlslmdfxy6aG3O9Xkh5kA", "verified": false } }, { "video": { - "id": "IyvhwAC9TjE", - "title": "Darkness Comes", - "url": "https://www.youtube.com/watch?v=IyvhwAC9TjE", - "duration": "3:19", - "snippet": "Provided to YouTube by Soundrop Darkness Comes · Herman Fassett Growth ℗ 2017 Herman Fassett Released on: 2017-08-05 ...", - "upload_date": "Live", - "thumbnail_src": "https://i.ytimg.com/vi/IyvhwAC9TjE/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLA_p6RiCj6PuT4PfDI527ixPALe1Q", - "views": "2 views" + "id": "N2RhnMPbOPc", + "title": "Our Barren Lands (Cello) - Herman Fassett", + "url": "https://www.youtube.com/watch?v=N2RhnMPbOPc", + "duration": "3:05", + "snippet": "I actually had this cello melody sort of lying around and thought I'd pull it out really quickly for this contest: ...", + "upload_date": "4 years ago", + "thumbnail_src": "https://i.ytimg.com/vi/N2RhnMPbOPc/hq720.jpg?sqp=-oaymwEXCNAFEJQDSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLCRed2S-gR0bN55n86v8wpH5Scfsw", + "views": "61 views" }, "uploader": { - "username": "Herman Fassett - Topic", - "url": "https://www.youtube.com/channel/UCDKLO7jQj_lq3_mq0ynlrAg", + "username": "Herman Fassett", + "url": "https://www.youtube.com/channel/UCsOlslmdfxy6aG3O9Xkh5kA", "verified": false } }, { - "video": { - "id": "_7TFT9CCN4Q", - "title": "The Path to Freedom", - "url": "https://www.youtube.com/watch?v=_7TFT9CCN4Q", - "duration": "3:30", - "snippet": "Provided to YouTube by Soundrop The Path to Freedom · Herman Fassett Growth ℗ 2017 Herman Fassett Released on: ...", - "upload_date": "Live", - "thumbnail_src": "https://i.ytimg.com/vi/_7TFT9CCN4Q/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLBIfZIt397p2uwS1ULiT9bqNstTMQ", - "views": "4 views" - }, - "uploader": { - "username": "Herman Fassett - Topic", + "channel": { + "id": "UCDKLO7jQj_lq3_mq0ynlrAg", + "title": "Herman Fassett - Topic", "url": "https://www.youtube.com/channel/UCDKLO7jQj_lq3_mq0ynlrAg", + "snippet": "", + "thumbnail_src": "//yt3.ggpht.com/4YbD3rhI-u663smC4uJ8_KI5S__KbzvYO_R8E_p6ZhO3Yim3LujhtBK797bea-ObO8DN0owI=s176-c-k-c0x00ffffff-no-rj-mo", + "video_count": "16 videos", + "subscriber_count": "0 subscribers", "verified": false } }, { "video": { - "id": "DFMIl2nhJ7U", - "title": "Sunrise", - "url": "https://www.youtube.com/watch?v=DFMIl2nhJ7U", - "duration": "2:34", - "snippet": "Provided to YouTube by Soundrop Sunrise · Herman Fassett Growth ℗ 2017 Herman Fassett Released on: 2017-08-05 ...", - "upload_date": "Live", - "thumbnail_src": "https://i.ytimg.com/vi/DFMIl2nhJ7U/hqdefault.jpg?sqp=-oaymwEjCNACELwBSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLDWIIiPmq03jgLQpBwb9soCXcoLEQ", - "views": "2 views" + "id": "HAF11xeBOF0", + "title": "Sussex Carol (On Christmas Night) - Herman Fassett", + "url": "https://www.youtube.com/watch?v=HAF11xeBOF0", + "duration": "1:39", + "snippet": "Just a quick cover of a Christmas carol since we're so close (some of us)...", + "upload_date": "1 year ago", + "thumbnail_src": "https://i.ytimg.com/vi/HAF11xeBOF0/hqdefault.jpg?sqp=-oaymwEjCOADEI4CSFryq4qpAxUIARUAAAAAGAElAADIQj0AgKJDeAE=&rs=AOn4CLA-iD7F5MPj1OnHhiU8-FPozXh_wA", + "views": "34 views" }, "uploader": { - "username": "Herman Fassett - Topic", - "url": "https://www.youtube.com/channel/UCDKLO7jQj_lq3_mq0ynlrAg", + "username": "Herman Fassett", + "url": "https://www.youtube.com/channel/UCsOlslmdfxy6aG3O9Xkh5kA", "verified": false } } ], - "version": "0.1.0", - "parser": "json_format" + "version": "0.1.3", + "parser": "json_format.scraper_data", + "key": "PXzaSyAO_HN5RlqU8Q4STEHLGCilw_Y9_11qcZ7", + "estimatedResults": "428", + "nextPageToken": "XvPREg5oZXJtYW4gPmFzc2V0dBrAA1NCU0NBUmhWSXCDUGJITnNiV1JtZUhrMllVY3pUemxZYTJnMWEwR0NBUXRhZWtGVVJWTnZZbVpQZDRJQkMzaHdjbGxRWkRWUmVHaEJnZ0VMT0VZNFRGVkhZMGRqWldlQ0FRdFZXVVI0YXpkYVMzWlRNSUlCQzI1Mk4zaG1NRTFWUm5wM2dnRUxWMGRaY1c5aE1HbGxVSE9DQVF0NFJHVk9kRWhEVEU0NE9JSUJDemRGV1RKeFNqbDJVVWhKZ2dFaVVFeERaMjU2WVRBNFpVRTVNVzl5YUVoMlRsQlVjRjluUldwRlEwZDVWamxJVklJQkMyWnVjSFpuZVhWdmIxQnpnZ0VMYmxKWFNtUmhlVzFSWkdlQ0FRdGtOVXBCWkRkT00yMUNXWUlCQzNscldWUkpiWEUzY0RsQmdnRUxXbUUzVDFvMlRYTnRhbFdDQVF0dWJqZFBaM0l5ZURGb2I0SUJDM0F6VWtWR1QzQkplRFpqZ2dFTFRqSlNhRzVOVUdKUFVHT0NBUmhWUTBSTFRFODNhbEZxWDJ4eE0xOXRjVEI1Ym14eVFXZUNBUXRJUVVZeE1YaGxRazlHTUElM0QlM0TKARsaF2h0dHBzOi8vd3d3LnlvdXR1YmUuY29tIgAYgeDoGCILc2VhcmNoLWZlZWQ%3D" } -``` -## Run with docker - -To run this project with docker, go to project root directory and run following commands. - -``` -docker build -t /youtube-scrape . -docker run -p 8080:8080 -d /youtube-scrape -``` - -## Deploy to Heroku - -[![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy) - +``` \ No newline at end of file diff --git a/package.json b/package.json index c1b56c7..03385ef 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "youtube-scrape", - "version": "0.1.3", + "version": "0.1.4", "description": "Scrape YouTube searches", "main": "server.js", "author": "Herman Fassett", diff --git a/scraper.js b/scraper.js index 9d10f7a..f14aa64 100644 --- a/scraper.js +++ b/scraper.js @@ -1,19 +1,42 @@ const request = require('request'); -async function youtube(query, page) { +async function youtube(query, key, pageToken) { return new Promise((resolve, reject) => { - // Specify YouTube search url - let url = `https://www.youtube.com/results?q=${encodeURIComponent(query)}${page ? `&page=${page}` : ''}`; - - // Access YouTube search - request(url, (error, response, html) => { - // Check for errors - if (!error && response.statusCode === 200) { - let json = { results: [], version: require('./package.json').version }; + let json = { results: [], version: require('./package.json').version }; - // If that fails, we have to parse new format from json data in html script tag - if (!json.results.length) { + // Specify YouTube search url + if (key) { + json["parser"] = "json_format.page_token"; + json["key"] = key; + + // Access YouTube search API + request.post(`https://www.youtube.com/youtubei/v1/search?key=${key}`, { + json: { + context: { + client: { + clientName: "WEB", + clientVersion: "2.20201022.01.01", + }, + }, + continuation: pageToken + }, + }, (error, response, body) => { + if (!error && response.statusCode === 200) { + parseJsonFormat(body.onResponseReceivedCommands[0].appendContinuationItemsAction.continuationItems, json); + return resolve(json); + } + resolve({ error: error }); + }); + } + else { + let url = `https://www.youtube.com/results?q=${encodeURIComponent(query)}`; + + // Access YouTube search + request(url, (error, response, html) => { + // Check for errors + if (!error && response.statusCode === 200) { json["parser"] = "json_format"; + json["key"] = html.match(/"innertubeApiKey":"([^"]*)/)[1]; // Get script json data from html to parse let data, sectionLists = []; @@ -36,40 +59,54 @@ async function youtube(query, page) { } // Loop through all objects and parse data according to type - sectionLists.filter(x => x.hasOwnProperty("itemSectionRenderer")).forEach(sectionList => { - try { - sectionList.itemSectionRenderer.contents.forEach(content => { - try { - if (content.hasOwnProperty("channelRenderer")) { - json.results.push(parseChannelRenderer(content.channelRenderer)); - } - if (content.hasOwnProperty("videoRenderer")) { - json.results.push(parseVideoRenderer(content.videoRenderer)); - } - if (content.hasOwnProperty("radioRenderer")) { - json.results.push(parseRadioRenderer(content.radioRenderer)); - } - if (content.hasOwnProperty("playlistRenderer")) { - json.results.push(parsePlaylistRenderer(content.playlistRenderer)); - } - } - catch(ex) { - console.error("Failed to parse renderer:", ex); - console.log(content); - } - }); + parseJsonFormat(sectionLists, json); + + return resolve(json); + } + resolve({ error: error }); + }); + } + }); +}; + +/** + * Parse youtube search results from json sectionList array and add to json result object + * @param {Array} contents - The array of sectionLists + * @param {Object} json - The object being returned to caller + */ +function parseJsonFormat(contents, json) { + contents.forEach(sectionList => { + try { + if (sectionList.hasOwnProperty("itemSectionRenderer")) { + sectionList.itemSectionRenderer.contents.forEach(content => { + try { + if (content.hasOwnProperty("channelRenderer")) { + json.results.push(parseChannelRenderer(content.channelRenderer)); } - catch (ex) { - console.error("Failed to read contents for section list:", ex); - console.log(sectionList); + if (content.hasOwnProperty("videoRenderer")) { + json.results.push(parseVideoRenderer(content.videoRenderer)); } - }); - } - - return resolve(json); + if (content.hasOwnProperty("radioRenderer")) { + json.results.push(parseRadioRenderer(content.radioRenderer)); + } + if (content.hasOwnProperty("playlistRenderer")) { + json.results.push(parsePlaylistRenderer(content.playlistRenderer)); + } + } + catch(ex) { + console.error("Failed to parse renderer:", ex); + console.log(content); + } + }); + } + else if (sectionList.hasOwnProperty("continuationItemRenderer")) { + json["nextPageToken"] = sectionList.continuationItemRenderer.continuationEndpoint.continuationCommand.token; } - resolve({ error: error }); - }); + } + catch (ex) { + console.error("Failed to read contents for section list:", ex); + console.log(sectionList); + } }); } diff --git a/server.js b/server.js index 20f7523..5682bff 100644 --- a/server.js +++ b/server.js @@ -9,7 +9,7 @@ app.get('/', (req, res) => { //API route app.get('/api/search', (req, res) => { - scraper.youtube(req.query.q, req.query.page) + scraper.youtube(req.query.q, req.query.key, req.query.pageToken) .then(x => res.json(x)) .catch(e => res.send(e)); });