Skip to content

Commit

Permalink
fix: suppress 403 error
Browse files Browse the repository at this point in the history
  • Loading branch information
everettsouthwick committed Aug 26, 2023
1 parent 9423447 commit 00b1120
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 32 deletions.
8 changes: 6 additions & 2 deletions src/utils/buildEmbeds.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,14 @@ const { fetchWebsiteDetails } = require('./fetchWebsite.js');

async function buildEmbeds(platform, originalUrl, newUrl) {
try {
const data = await fetchWebsiteDetails(originalUrl);

if (!data) {
return [];
}

const embeds = [];
const addedImageUrls = new Set();

const data = await fetchWebsiteDetails(originalUrl);
const base = new URL(data.ogUrl || data.twitterUrl || data.requestUrl || originalUrl);

if (data.favicon?.includes('.svg')) {
Expand Down
62 changes: 32 additions & 30 deletions src/utils/fetchWebsite.js
Original file line number Diff line number Diff line change
@@ -1,43 +1,45 @@
const ogs = require('open-graph-scraper');


async function fetchWebsiteDetails(url) {
try {
const options = { url, headers:
{
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'accept-language': 'en-US,en;q=0.9',
'cache-control': 'max-age=0',
'if-modified-since': 'Sat, 26 Aug 2023 17:58:14 GMT',
'sec-ch-ua': '"Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
},
};
const data = await ogs(options);
const options = {
url,
headers: {
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'accept-language': 'en-US,en;q=0.9',
'cache-control': 'max-age=0',
'if-modified-since': 'Sat, 26 Aug 2023 17:58:14 GMT',
'sec-ch-ua': '"Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
},
};

const { error, result } = data;
if (error) {
throw error;
}
if (!result.success) {
throw new Error('No Open Graph data found');
}
const { error, result } = await ogs(options).catch(err => ({ error: err }));

return result;
if (error && error.includes('403')) {
return null;
}
catch (error) {

if (error) {
console.error('Error fetching website details:', error);
throw error;
}

if (!result.success) {
const errorMsg = 'No Open Graph data found';
console.error('Error fetching website details:', errorMsg);
throw new Error(errorMsg);
}

return result;
}

module.exports = {
fetchWebsiteDetails,
};
};

0 comments on commit 00b1120

Please sign in to comment.