From d74de89ef1ad5589a4e9a125c894a6ac71169cb2 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Tue, 23 Oct 2018 00:35:24 +0200 Subject: [PATCH 001/132] Start rework --- app.js | 26 ----- bin/www | 49 -------- config.js | 30 ----- core/discord.js | 156 ------------------------- core/proxy.js | 20 ---- core/proxyActiveSessions.js | 66 ----------- core/proxyConfig.js | 43 ------- core/redirect.js | 22 ---- core/serverManager.js | 127 --------------------- core/stats.js | 55 --------- index.js | 3 + package.json | 26 +++-- routes/routes.js | 219 ------------------------------------ src/app.js | 41 +++++++ src/config.js | 28 +++++ src/core/servers.js | 71 ++++++++++++ src/core/sessions.js | 74 ++++++++++++ src/core/transcodes.js | 16 +++ src/routes.js | 39 +++++++ src/routes/api.js | 76 +++++++++++++ src/utils.js | 22 ++++ utils/getIp.js | 9 -- utils/reloadConf.js | 31 ----- 23 files changed, 384 insertions(+), 865 deletions(-) delete mode 100644 app.js delete mode 100644 bin/www delete mode 100644 config.js delete mode 100644 core/discord.js delete mode 100644 core/proxy.js delete mode 100644 core/proxyActiveSessions.js delete mode 100644 core/proxyConfig.js delete mode 100644 core/redirect.js delete mode 100644 core/serverManager.js delete mode 100644 core/stats.js create mode 100644 index.js delete mode 100644 routes/routes.js create mode 100644 src/app.js create mode 100644 src/config.js create mode 100644 src/core/servers.js create mode 100644 src/core/sessions.js create mode 100644 src/core/transcodes.js create mode 100644 src/routes.js create mode 100644 src/routes/api.js create mode 100644 src/utils.js delete mode 100644 utils/getIp.js delete mode 100644 utils/reloadConf.js diff --git a/app.js b/app.js deleted file mode 100644 index 64cb9c4..0000000 --- a/app.js +++ /dev/null @@ -1,26 +0,0 @@ -// Requires -const express = require('express'); -const cors = require('cors'); - -// Customs requires -const routes = require('./routes/routes'); -const proxy = require('./core/proxy'); -const config = require('./config'); -const stats = require('./core/stats'); - -// Init Express App -const app = express(); - -// Allow CORS -app.use(cors()); - -// Sessions files -app.use('/api/sessions', express.static(config.plex.sessions)); - -// Default routes -app.use('/', routes); - -// Export app -module.exports = app; - - diff --git a/bin/www b/bin/www deleted file mode 100644 index 606924f..0000000 --- a/bin/www +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env node - -let app = require('../app') -let config = require('../config') -let debug = require('debug')('UnicornLoadBalancer:server') -let http = require('http') -const proxy = require('../core/proxy'); - -app.set(config.port) -let server = http.createServer(app) - -server.listen(config.loadBalancer.port) -server.on('error', onError) -server.on('listening', onListening) -server.on('upgrade', function (req, res) { - proxy.ws(req, res); -}); - -function onError(error) { - if (error.syscall !== 'listen') { - throw error - } - - var bind = typeof config.loadBalancer.port === 'string' - ? 'Pipe ' + config.loadBalancer.port - : 'Port ' + config.loadBalancer.port - - // handle specific listen errors with friendly messages - switch (error.code) { - case 'EACCES': - console.error(bind + ' requires elevated privileges') - process.exit(1) - break - case 'EADDRINUSE': - console.error(bind + ' is already in use') - process.exit(1) - break - default: - throw error - } -} - -function onListening() { - var addr = server.address() - var bind = typeof addr === 'string' - ? 'pipe ' + addr - : 'port ' + addr.port - debug('Listening on ' + bind) -} diff --git a/config.js b/config.js deleted file mode 100644 index e786b64..0000000 --- a/config.js +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Created by Maxime Baconnais on 29/08/2017. - */ - -module.exports = { - cluster: [ - 'https://transcoder1.myplex.com', - 'https://transcoder2.myplex.com' - ], - preprod: { - enabled: false, - devIps: [ - '127.0.0.1/32' - ], - server: 'https://beta-transcoder.myplex.com' - }, - plex: { - host: 'localhost', - port: 32400, - sessions: '/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Cache/Transcode/Sessions', - database: '/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db' - }, - loadBalancer: { - port: 3001 - }, - alerts: { - discord: '' - }, - productName: 'UnicornLoadBalancer' -}; diff --git a/core/discord.js b/core/discord.js deleted file mode 100644 index b63f6b1..0000000 --- a/core/discord.js +++ /dev/null @@ -1,156 +0,0 @@ -const snekfetch = require("snekfetch"); -const endpoint = "https://discordapp.com/api/v6/webhooks"; - -class Webhook { - constructor(url) { - this.url = url; - this.id = ''; - this.token = ''; - this.meta = {}; - } - - getWebhookProps() { - return new Promise((resolve, reject) => { - snekfetch.get(this.url) - .then((res) => { - let parsed = JSON.parse(res.text); - Object.assign(this.meta, parsed); - this.id = parsed['id']; - this.token = parsed['token']; - resolve(); - }) - .catch((err) => { - throw err; - }) - }) - - } - - checkEndpoint(payload) { - if (!this.endpoint) this.getWebhookProps().then( () => - this.sendPayload(payload) - ) - else this.sendPayload(payload) - } - - sendPayload(payload) { - return new Promise((resolve, reject) => { - snekfetch.post(this.endpoint) - .send(payload) - .then(() => { - resolve() - }) - .catch((err) => { - reject(err.text) - }) - }); - } - - get endpoint() { - if (this.id == '' || this.token == '') return false; - return `${endpoint}/${this.id}/${this.token}/slack` - } - - err(name, message) { - let payload = { - "username": name, - "text": "[]()", - "attachments": [{ - "color": "#ff0000", - "fields": [{ - "title": "Error", - "value": message - }], - "ts": new Date() / 1000 - }] - }; - - return this.checkEndpoint(payload); - } - - info(name, message) { - let payload = { - "username": name, - "text": "[]()", - "attachments": [{ - "color": "#00fffa", - "fields": [{ - "title": "Information", - "value": message - }], - "ts": new Date() / 1000 - }] - }; - - return this.checkEndpoint(payload); - } - - success(name, message) { - let payload = { - "username": name, - "text": "[]()", - "attachments": [{ - "color": "#04ff00", - "fields": [{ - "title": "Success", - "value": message - }], - "ts": new Date() / 1000 - }] - }; - - return this.checkEndpoint(payload); - } - - warn(name, message) { - let payload = { - "username": name, - "text": "[]()", - "attachments": [{ - "color": "#ffe900", - "fields": [{ - "title": "Warning", - "value": message - }], - "ts": new Date() / 1000 - }] - }; - - return this.checkEndpoint(payload); - } - - custom(name, message, title, color) { - let payload; - if (color) { - payload = { - "username": name, - "text": "[]()", - "attachments": [{ - "color": color, - "fields": [{ - "title": title, - "value": message - }], - "ts": new Date() / 1000 - }] - } - } else { - payload = { - "username": name, - "text": "[]()", - "attachments": [{ - - "fields": [{ - "title": title, - "value": message - }], - "ts": new Date() / 1000 - }] - } - } - - return this.checkEndpoint(payload); - } -} - -module.exports = Webhook; \ No newline at end of file diff --git a/core/proxy.js b/core/proxy.js deleted file mode 100644 index b9c3f30..0000000 --- a/core/proxy.js +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Created by Maxime Baconnais on 29/08/2017. - */ - -const httpProxy = require('http-proxy'); -const config = require('../config'); - -let proxy = httpProxy.createProxyServer({ - target: { - host: config.plex.host, - port: config.plex.port - } -}); - -proxy.on('error', (err, req, res) => { - res.writeHead(404, {}); - res.end('Plex not respond in time, proxy request fails'); -}); - -module.exports = proxy; \ No newline at end of file diff --git a/core/proxyActiveSessions.js b/core/proxyActiveSessions.js deleted file mode 100644 index a296787..0000000 --- a/core/proxyActiveSessions.js +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Created by Maxime Baconnais on 23/06/2018. - */ - -const httpProxy = require('http-proxy'); -const config = require('../config'); -const serverManager = require('./serverManager'); -const stats = require('./stats'); - -let proxy = httpProxy.createProxyServer({ - target: { - host: config.plex.host, - port: config.plex.port - }, - selfHandleResponse: true -}); - -proxy.on('proxyRes', (proxyRes, req, res) => { - let body = new Buffer(''); - proxyRes.on('data', (data) => { - body = Buffer.concat([body, data]); - }); - proxyRes.on('end', () => { - body = body.toString(); - - let videos = body.split(' 0) - server = stats[server].config.serverName; - else - server = server.replace('https://www.', '').replace('http://www.', '').replace('https://', '').replace('http://', ''); - - // Get player - let player = videos[i].split(' { - console.log('error', err); - res.writeHead(404, {}); - res.end('Plex not respond in time, proxy request fails'); -}); - -module.exports = proxy; \ No newline at end of file diff --git a/core/proxyConfig.js b/core/proxyConfig.js deleted file mode 100644 index ac7c09e..0000000 --- a/core/proxyConfig.js +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Created by Maxime Baconnais on 23/06/2018. - */ - -const httpProxy = require('http-proxy'); -const config = require('../config'); - -let proxy = httpProxy.createProxyServer({ - target: { - host: config.plex.host, - port: config.plex.port - }, - selfHandleResponse: true -}); - -proxy.on('proxyRes', (proxyRes, req, res) => { - let body = new Buffer(''); - proxyRes.on('data', (data) => { - body = Buffer.concat([body, data]); - }); - proxyRes.on('end', () => { - body = body.toString(); - res.header("Content-Type", "text/xml;charset=utf-8"); - res.send(body - .replace("streamingBrainABRVersion=", "DISABLEDstreamingBrainABRVersion=") // Disable Streaming adaptative - .replace('allowSync="1"', 'allowSync="0"') // Disable Sync option - .replace('sync="1"', 'DISABLEDsync="1"') // Disable Sync option - .replace('updater="1"', 'updater="0"') // Disable updates - .replace('backgroundProcessing="1"', 'DISABLEDbackgroundProcessing="1"') // Disable Optimizing feature - .replace('livetv="', 'DISABLEDlivetv="') // Disable LiveTV - .replace('allowTuners="', 'DISABLEDallowTuners="') // Disable Tuners - .replace('ownerFeatures="', 'ownerFeatures="session_kick,') // Enable Session Kick Feature - ); - }); -}); - -proxy.on('error', (err, req, res) => { - console.log('error', err); - res.writeHead(404, {}); - res.end('Plex not respond in time, proxy request fails'); -}); - -module.exports = proxy; \ No newline at end of file diff --git a/core/redirect.js b/core/redirect.js deleted file mode 100644 index 3c213e5..0000000 --- a/core/redirect.js +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Created by Maxime Baconnais on 29/08/2017. - */ - -const debug = require('debug')('redirect'); - -const serverManager = require('./serverManager'); -const getIp = require('../utils/getIp'); - -let redirect = (req, res) => { - const sessionId = serverManager.getSession(req); - const serverUrl = serverManager.chooseServer(sessionId, getIp(req)); - - res.writeHead(302, { - 'Location': serverUrl + req.url - }); - res.end(); - - debug('Send 302 for ' + sessionId + ' to ' + serverUrl); -} - -module.exports = redirect; diff --git a/core/serverManager.js b/core/serverManager.js deleted file mode 100644 index 5c0b2f0..0000000 --- a/core/serverManager.js +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Created by Maxime Baconnais on 29/08/2017. - */ - -const geoip = require('geoip-lite'); -const Netmask = require('netmask').Netmask; -const config = require('../config'); -const stats = require('../core/stats'); - -// Hot reload geoip -geoip.startWatchingDataUpdate(); - -let serverManager = {} - -serverManager.cacheSession = {}; -serverManager.sessions = {}; -serverManager.stoppedSessions = {}; - -serverManager.saveSession = (req) => { - if (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined' && typeof (req.query.session) !== 'undefined') { - serverManager.cacheSession[req.query['X-Plex-Session-Identifier']] = req.query.session.toString(); - } -}; - -serverManager.calculateServerLoad = (stats) => { - // The configuration is unavailable, the server is probably unavailable - if (!stats) - return (1000); - - // Default load 0 - let load = 0; - - // Each transcode add 1 to the load - load += stats.transcoding; - - // Each HEVC sessions add 1.5 to the load - if (stats.codecs.hevc) - load += stats.codecs.hevc * 1.5; - - // Server already have too much sessions - if (stats.config && stats.sessions >= stats.config.preferredMaxSessions) - load += 2.5; - - // Server already have too much transcodes - if (stats.config && stats.transcoding >= stats.config.preferredMaxTranscodes) - load += 5; - - // Server already have too much downloads - if (stats.config && stats.downloads >= stats.config.preferredMaxDownloads) - load += 1; - - // Return load - return (load); -} - -serverManager.getSession = (req) => { - if (typeof (req.params.sessionId) !== 'undefined') - return (req.params.sessionId); - if (typeof (req.query.session) !== 'undefined') - return (req.query.session); - if (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined' && typeof (serverManager.cacheSession[req.query['X-Plex-Session-Identifier']]) !== 'undefined') - return (serverManager.cacheSession[req.query['X-Plex-Session-Identifier']]); - if (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined') - return (req.query['X-Plex-Session-Identifier']); - if (typeof (req.query['X-Plex-Client-Identifier']) !== 'undefined') - return (req.query['X-Plex-Client-Identifier']); - return (false); -}; - -serverManager.removeServer = (url) => { - for (let session in serverManager.sessions) { - if (serverManager.sessions[session] == url) - delete serverManager.sessions[session]; - } -}; - -serverManager.forceStopStream = (session, reason) => { - serverManager.stoppedSessions[session] = reason; -}; - -serverManager.geoIp = (url, ip) => { - - const geo = geoip.lookup(ip); - - if (typeof (stats[url].routing) !== 'undefined') { - if (typeof (stats[url].routing[geo.country + '-' + geo.region]) !== 'undefined') // 'COUNTRY-REGION' tag like 'US-TX' - return (stats[url].routing[geo.country + '-' + geo.region]) - if (typeof (stats[url].routing[geo.country]) !== 'undefined') // 'COUNTRY' tag like 'US' - return (stats[url].routing[geo.country]) - } - return (url); -} - -serverManager.chooseServer = (session, ip = false) => { - - //Pre-prod - if (config.preprod.enabled && ip) { - for (let i = 0; i < config.preprod.devIps.length; i++) { - let mask = new Netmask(config.preprod.devIps[i]); - - if (mask.contains(ip)) - return config.preprod.server; - } - } - - let count = config.cluster.length; - if (count == 0) - return (false); - if (typeof (serverManager.sessions[session]) !== 'undefined' && - config.cluster.indexOf(serverManager.sessions[session]) != -1 && - stats[serverManager.sessions[session]]) { - return (serverManager.geoIp(serverManager.sessions[session], ip)); - } - - let sortedServers = config.cluster.sort((url) => { return (serverManager.calculateServerLoad(stats[url])); }); - - serverManager.sessions[session] = sortedServers[0]; - - return (serverManager.geoIp(sortedServers[0], ip)); -}; - -serverManager.removeSession = (session) => { - delete serverManager.sessions[session]; - delete serverManager.stoppedSessions[session]; -}; - -module.exports = serverManager; \ No newline at end of file diff --git a/core/stats.js b/core/stats.js deleted file mode 100644 index 18e4038..0000000 --- a/core/stats.js +++ /dev/null @@ -1,55 +0,0 @@ -const request = require('request'); - -const Webhook = require("../core/discord"); -const config = require('../config'); - -let serversStats = {}; - -const sendAlertDead = (url) => { - if (config.alerts.discord) { - const Hook = new Webhook(config.alerts.discord); - Hook.err(config.productName, "The server `"+url+"` is now unavailable."); - } -}; - -const sendAlertAlive = (url) => { - if (config.alerts.discord) { - const Hook = new Webhook(config.alerts.discord); - Hook.success(config.productName, "The server `"+url+"` is now available."); - } -}; - -const getInformations = () => { - config.cluster.map((url) => { - request(url + '/api/stats', (error, response, body) => { - try { - if (!error) - { - const notif = (serversStats[url] === false); - serversStats[url] = JSON.parse(body); - if (notif) - sendAlertAlive(url); - } - else - { - if (serversStats[url] !== false) - sendAlertDead(url); - serversStats[url] = false; - } - } - catch (err) { - if (serversStats[url] !== false) - sendAlertDead(url); - serversStats[url] = false; - } - }) - }); -}; - -setInterval(() => { - getInformations(); -}, 2000); - -getInformations(); - -module.exports = serversStats; \ No newline at end of file diff --git a/index.js b/index.js new file mode 100644 index 0000000..a2d5592 --- /dev/null +++ b/index.js @@ -0,0 +1,3 @@ +// This file is the ES6 module loader, don't edit it, go to src/app.js +require = require("esm")(module/*, options*/) +module.exports = require("./src/app.js") diff --git a/package.json b/package.json index 4ac95b6..beb7332 100644 --- a/package.json +++ b/package.json @@ -1,21 +1,23 @@ { - "name": "unicorn-loadbalancer", - "version": "0.0.1", - "private": true, + "name": "unicorn-load-balancer", + "version": "2.0.0", + "description": "", + "main": "index.js", "scripts": { - "start": "node ./bin/www", - "geo-build" : "node ./node_modules/geoip-lite/scripts/updatedb.js" + "start": "node index.js", + "geo-build": "node ./node_modules/geoip-lite/scripts/updatedb.js", + "test": "echo \"Error: no test specified\" && exit 1" }, + "author": "Maxime Baconnais", + "license": "MIT", "dependencies": { "cors": "^2.8.4", - "debug": "^3.0.0", + "debug": "^4.0.1", + "esm": "^3.0.84", "express": "^4.16.3", - "geoip-lite": "^1.3.2", + "getenv": "^0.7.0", "http-proxy": "^1.17.0", - "netmask": "^1.0.6", - "request": "^2.83.0", - "require-reload": "^0.2.2", - "snekfetch": "^3.6.4", - "sqlite3": "^4.0.0" + "redis": "^2.8.0", + "sqlite3": "^4.0.2" } } diff --git a/routes/routes.js b/routes/routes.js deleted file mode 100644 index a629d3c..0000000 --- a/routes/routes.js +++ /dev/null @@ -1,219 +0,0 @@ -/** - * Created by Maxime Baconnais on 29/08/2017. - */ - -const express = require('express'); -const request = require('request'); -const httpProxy = require('http-proxy'); -const sqlite3 = require('sqlite3').verbose(); -const router = express.Router(); - -const config = require('../config'); -const proxy = require('../core/proxy'); -const proxyConfig = require('../core/proxyConfig'); -const proxyActiveSessions = require('../core/proxyActiveSessions'); -const redirect = require('../core/redirect'); -const serverManager = require('../core/serverManager'); -const stats = require('../core/stats'); -const getIp = require('../utils/getIp'); -const reloadConf = require('../utils/reloadConf'); - -//Reload Config -router.get('/api/reload', reloadConf.reloadConf); - -// Return the scores of transcoders -router.get('/api/scores', (req, res) => { - let output = {}; - for (let i = 0; i < config.cluster.length; i++) { - output[config.cluster[i]] = serverManager.calculateServerLoad(stats[config.cluster[i]]); - } - res.send(JSON.stringify(output)); -}); - -// Return the stats of transcoders -router.get('/api/stats', (req, res) => { - let output = {}; - for (let i = 0; i < config.cluster.length; i++) { - output[config.cluster[i]] = stats[config.cluster[i]]; - } - res.send(JSON.stringify(output)); -}); - -// Reverse plex download ID to path -router.get('/api/pathname/:downloadid', (req, res) => { - try { - let db = new sqlite3.Database(config.plex.database); - db.get("SELECT * FROM media_parts WHERE id=? LIMIT 0,1", req.params.downloadid, (err, row) => { - if (row && row.file) - res.send(JSON.stringify(row)); - else - res.status(404).send('File not found in Plex Database'); - db.close(); - }); - } - catch (err) { - res.status(404).send('File not found in Plex Database'); - } -}); - -// Direct plex call -router.all('/api/plex/*', (req, res) => { - req.url = req.url.slice('/api/plex'.length); - return (proxy.web(req, res)); -}); - -//Dash routes -router.get('/video/:/transcode/universal/start.mpd', (req, res) => { - - let sessionId = false; - if (typeof(req.query['X-Plex-Session-Identifier']) !== 'undefined' && typeof(serverManager.cacheSession[req.query['X-Plex-Session-Identifier']]) !== 'undefined') - sessionId = serverManager.cacheSession[req.query['X-Plex-Session-Identifier']]; - - if (sessionId !== false) { - const serverUrl = serverManager.chooseServer(sessionId, getIp(req)); - if (typeof(serverUrl) !== 'undefined') - request(serverUrl + '/video/:/transcode/universal/stop?session=' + sessionId, (err, httpResponse, body) => { - serverManager.saveSession(req); - redirect(req, res); - }); - } else { - serverManager.saveSession(req); - redirect(req, res); - } -}); -router.get('/video/:/transcode/universal/dash/:sessionId/:streamId/initial.mp4', redirect); -router.get('/video/:/transcode/universal/dash/:sessionId/:streamId/:partId.m4s', redirect); - -//Stream mode -router.get('/video/:/transcode/universal/start', (req, res) => { - serverManager.saveSession(req); - redirect(req, res); -}); -router.get('/video/:/transcode/universal/subtitles', redirect); - -//m3u8 mode -router.get('/video/:/transcode/universal/start.m3u8', (req, res) => { - serverManager.saveSession(req); - redirect(req, res); -}); -router.get('/video/:/transcode/universal/session/:sessionId/base/index.m3u8', redirect); -router.get('/video/:/transcode/universal/session/:sessionId/base-x-mc/index.m3u8', redirect); -router.get('/video/:/transcode/universal/session/:sessionId/:fileType/:partId.ts', redirect); -router.get('/video/:/transcode/universal/session/:sessionId/:fileType/:partId.vtt', redirect); - -//Universal endpoints -router.get('/video/:/transcode/universal/stop', (req, res) => { - proxy.web(req, res); - - const sessionId = serverManager.getSession(req); - const serverUrl = serverManager.chooseServer(sessionId, getIp(req)); - - if (typeof(serverUrl) !== 'undefined') - request(serverUrl + '/video/:/transcode/universal/stop?session=' + sessionId); - - setTimeout(() => { - serverManager.removeSession(sessionId); - if (typeof(serverManager.stoppedSessions[req.query['X-Plex-Session-Identifier']]) != 'undefined') - delete serverManager.stoppedSessions[req.query['X-Plex-Session-Identifier']]; - }, 1000); -}); - -router.get('/video/:/transcode/universal/ping', (req, res) => { - proxy.web(req, res); - - const sessionId = serverManager.getSession(req); - const serverUrl = serverManager.chooseServer(sessionId, getIp(req)); - - if (typeof(serverUrl) !== 'undefined') - request(serverUrl + '/video/:/transcode/universal/ping?session=' + sessionId); -}); - -router.get('/:/timeline', (req, res) => { - - const sessionId = serverManager.getSession(req); - const serverUrl = serverManager.chooseServer(sessionId, getIp(req)); - - let cproxy = false; - if (typeof(req.query['X-Plex-Session-Identifier']) !== 'undefined' && typeof(serverManager.stoppedSessions[req.query['X-Plex-Session-Identifier']]) != 'undefined') - { - cproxy = httpProxy.createProxyServer({ - target: { - host: config.plex.host, - port: config.plex.port - }, - selfHandleResponse: true - }); - cproxy.on('proxyRes', (proxyRes, req, res) => { - let body = new Buffer(''); - proxyRes.on('data', (data) => { - body = Buffer.concat([body, data]); - }); - proxyRes.on('end', () => { - body = body.toString(); - res.header("Content-Type", "text/xml;charset=utf-8"); - res.send(body.replace(" { - console.log('error', err); - res.writeHead(404, {}); - res.end('Plex not respond in time, proxy request fails'); - }); - } - else - cproxy = proxy; - - if (req.query.state == 'stopped' || (typeof(req.query['X-Plex-Session-Identifier']) !== 'undefined' && typeof(serverManager.stoppedSessions[req.query['X-Plex-Session-Identifier']]) != 'undefined')) - { - cproxy.web(req, res); - - if (typeof(serverUrl) !== 'undefined') - request(serverUrl + '/video/:/transcode/universal/stop?session=' + sessionId); - setTimeout(() => { - serverManager.removeSession(sessionId); - if (typeof(serverManager.stoppedSessions[req.query['X-Plex-Session-Identifier']]) != 'undefined') - delete serverManager.stoppedSessions[req.query['X-Plex-Session-Identifier']]; - }, 1000); - } - else - { - proxy.web(req, res); - if (typeof(serverUrl) !== 'undefined') - request(serverUrl + '/video/:/transcode/universal/ping?session=' + sessionId); - } -}); - -router.get('/status/sessions/terminate', (req, res) => { - res.header("Content-Type", "text/xml;charset=utf-8"); - res.send(` - -`); - if (typeof(req.query.sessionId) !== 'undefined' && typeof(req.query.reason) !== 'undefined') - { - const sessionId = req.query.sessionId; - serverManager.forceStopStream(sessionId, req.query.reason); - } -}); - -// Download files -router.get('/library/parts/:id1/:id2/file.*', redirect); - -// Plex activity page -/*router.get('/status/sessions', (req, res) => { - proxyActiveSessions.web(req, res); -});*/ - -// Plex configuration get -/*router.get('/', (req, res) => { - if (req.query['X-Plex-Device-Name']) - proxyConfig.web(req, res); - else - proxy.web(req, res); -});*/ - -// Reverse all others to plex -router.all('*', (req, res) => { - proxy.web(req, res); -}); - -module.exports = router; diff --git a/src/app.js b/src/app.js new file mode 100644 index 0000000..d608592 --- /dev/null +++ b/src/app.js @@ -0,0 +1,41 @@ +import express from 'express'; +import cors from 'cors'; +import bodyParser from 'body-parser'; + +import config from './config'; +import Router from './routes'; +import { internalUrl } from './utils'; + +import debug from 'debug'; + +// Debugger +const D = debug('UnicornLoadBalancer'); + +// Welcome +D('Version: ' + config.version) + +// Init Express +const app = express(); + +// CORS +app.use(cors()); + +// Body parsing +app.use(bodyParser.json()); +app.use(bodyParser.urlencoded({ + extended: true +})); +app.use((err, req, res, next) => { + if (err instanceof SyntaxError && err.status >= 400 && err.status < 500 && err.message.indexOf('JSON')) + res.status(400).send({ error: { code: 'INVALID_BODY', message: 'Syntax error in the JSON body' } }); +}); + +// Init routes +D('Initializing API routes...'); + +// Routes +Router(app); + +// Bind and start +app.listen(config.server.port) +D('Launched on ' + internalUrl()); diff --git a/src/config.js b/src/config.js new file mode 100644 index 0000000..6b821b4 --- /dev/null +++ b/src/config.js @@ -0,0 +1,28 @@ +import env from 'getenv'; + +export default { + version: '2.0.0', + server: { + port: env.int('SERVER_PORT', 3001), + host: env.string('SERVER_DOMAIN', '127.0.0.1'), + ssl: env.bool('SERVER_SSL', false) + }, + plex: { + host: env.string('PLEX_HOST', '127.0.0.1'), + port: env.int('PLEX_PORT', 32400), + path: { + usr: env.string('PLEX_PATH_USR', '/usr/lib/plexmediaserver/'), + sessions: env.string('PLEX_PATH_SESSIONS', '/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Cache/Transcode/Sessions'), + database: env.string('PLEX_PATH_DATABASE', '/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db') + } + }, + redis: { + host: env.string('REDIS_HOST', '127.0.0.1'), + port: env.int('REDIS_PORT', 6379), + password: env.string('REDIS_PASSWORD', ''), + db: env.int('REDIS_DB', 0) + }, + scan: { + timer: 3000 + } +}; \ No newline at end of file diff --git a/src/core/servers.js b/src/core/servers.js new file mode 100644 index 0000000..87bf964 --- /dev/null +++ b/src/core/servers.js @@ -0,0 +1,71 @@ +let servers = []; + +let ServersManager = {}; + +ServersManager.add = (url) => { + if (servers.indexOf(url) !== -1) + return (false); + servers.push(url); + return (true); +}; + +ServersManager.remove = (url) => { + servers = server.filter((e) => (e !== url)); + return (true); +}; + +ServersManager.list = () => { + return (servers); +} + +// Calculate server score +ServersManager.calculateScore = (stats) => { + // The configuration is unavailable, the server is probably unavailable + if (!stats) + return (1000); + + // Default load 0 + let load = 0; + + // Each transcode add 1 to the load + load += stats.transcoding; + + // Each HEVC sessions add 1.5 to the load + if (stats.codecs.hevc) + load += stats.codecs.hevc * 1.5; + + // Server already have too much sessions + if (stats.config && stats.sessions >= stats.config.preferredMaxSessions) + load += 2.5; + + // Server already have too much transcodes + if (stats.config && stats.transcoding >= stats.config.preferredMaxTranscodes) + load += 5; + + // Server already have too much downloads + if (stats.config && stats.downloads >= stats.config.preferredMaxDownloads) + load += 1; + + // Return load + return (load); +} + +// Calculate all the server scores +ServersManager.scores = () => { + let output = {}; + ServersManager.list().forEach((e) => { + output[e] = ServersManager.calculateScore(); // Todo: Bind stats in call + }); + return (output); +} + +// Returns all the server stats +ServersManager.stats = () => { + let output = {}; + ServersManager.list().forEach((e) => { + output[e] = false; // Todo: Bind stats + }); + return (output); +} + +export default ServersManager; diff --git a/src/core/sessions.js b/src/core/sessions.js new file mode 100644 index 0000000..8a60421 --- /dev/null +++ b/src/core/sessions.js @@ -0,0 +1,74 @@ +import debug from 'debug'; + +import config from '../config'; +import { publicUrl, plexUrl, redis } from '../utils'; + +// Debugger +const D = debug('UnicornLoadBalancer:SessionsManager'); + +let SessionsManager = {}; + +// Parse FFmpeg parameters with internal bindings +SessionsManager.parseFFmpegParameters = (args) => { + // Extract Session ID + const regex = /^http\:\/\/127.0.0.1:32400\/video\/:\/transcode\/session\/(.*)\/progress$/; + const sessions = args.filter(e => (regex.test(e))).map(e => (e.match(regex)[1])) + const sessionFull = (typeof (sessions[0]) !== 'undefined') ? sessions[0] : false; + const sessionId = (typeof (sessions[0]) !== 'undefined') ? sessions[0].split('/')[0] : false; + + // Check Session Id + if (!sessionId || !sessionFull) + return (false); + + // Debug + D('Session found: ' + sessionId + ' (' + sessionFull + ')'); + + // Parse arguments + const parsedArgs = args.map((e) => { + + // Progress + if (e.indexOf('/progress') !== -1) + return (e.replace(plexUrl(), publicUrl())); + + // Manifest and seglist + if (e.indexOf('/manifest') !== -1 || e.indexOf('/seglist') !== -1) + return (e.replace(plexUrl(), '{INTERNAL_TRANSCODER}')); + + // Other + return (e.replace(plexUrl(), publicUrl()).replace(config.plex.path.sessions, publicUrl() + 'api/sessions/').replace(config.plex.path.usr, '{INTERNAL_RESOURCES}')); + }); + + // Add seglist to arguments if needed + const segList = '{INTERNAL_TRANSCODER}video/:/transcode/session/' + sessionFull + '/seglist'; + let finalArgs = []; + let segListMode = false; + parsedArgs.forEach((e, i) => { + if (e === '-segment_list') { + segListMode = true; + finalArgs.push(e); + return (true); + } + if (segListMode) { + finalArgs.push(segList); + if (parsedArgs[i + 1] !== '-segment_list_type') + finalArgs.push('-segment_list_type', 'csv', '-segment_list_size', '2147483647'); + segListMode = false; + return (true); + } + finalArgs.push(e); + }); + return (finalArgs); +} + +// Store the FFMPEG parameters in RedisCache +SessionsManager.storeFFmpegParameters = (args, env) => { + const parsedArguments = SessionsManager.parseFFmpegParameters(args); + redis.set(propersessionid, JSON.stringify({ + args: parsedArguments, + env + })); + return (parsedArguments); +}; + +// Export our SessionsManager +export default SessionsManager; \ No newline at end of file diff --git a/src/core/transcodes.js b/src/core/transcodes.js new file mode 100644 index 0000000..4ab1906 --- /dev/null +++ b/src/core/transcodes.js @@ -0,0 +1,16 @@ + +// Debugger +const D = debug('UnicornLoadBalancer:TranscodesManager'); + +let TranscodesManager = {}; + +TranscodesManager.create = () => { + +} + +TranscodesManager.remove = () => { + +} + +// Export our TranscodesManager +export default TranscodesManager; \ No newline at end of file diff --git a/src/routes.js b/src/routes.js new file mode 100644 index 0000000..182165e --- /dev/null +++ b/src/routes.js @@ -0,0 +1,39 @@ +import express from 'express'; + +import config from './config'; +import RoutesAPI from './routes/api'; + +export default (app) => { + + // UnicornLoadBalancer API + app.use('/api/sessions', express.static(config.plex.path.sessions)); + app.get('/api/scores', RoutesAPI.scores); + app.get('/api/stats', RoutesAPI.stats); + app.post('/api/ffmpeg', RoutesAPI.ffmpeg); + app.get('/api/path/:id', RoutesAPI.path); + app.post('/api/update', RoutesAPI.update); + app.all('/api/plex/*', RoutesAPI.plex); + + // MPEG Dash support + app.get('/video/:/transcode/universal/start.mpd'); // 302 => /transcode/dash/start.mpd?uid={UNICORNID} // Plex called by Transcoder + + // Long polling support + app.get('/video/:/transcode/universal/start'); // 302 => /transcode/polling/start?uid={UNICORNID}&offset=X // Plex called by Transcoder + app.get('/video/:/transcode/universal/subtitles'); // 302 => /transcode/polling/subtitles?uid={UNICORNID} // Don't call Plex + + // M3U8 support + app.get('/video/:/transcode/universal/start.m3u8'); // 302 => /transcode/m3u8/start.m3u8?uid={UNICORNID} // Plex called by Transcoder + app.get('/video/:/transcode/universal/session/:sessionId/base/index.m3u8'); // 302 => /transcode/m3u8/index.m3u8?uid={UNICORNID} // Don't call Plex + app.get('/video/:/transcode/universal/session/:sessionId/base-x-mc/index.m3u8'); // 302 => /transcode/m3u8/index-mc.m3u8?uid={UNICORNID} // Don't call Plex + + // Control support + app.get('/video/:/transcode/universal/stop'); // Proxy => /control/stop?uid={UNICORNID} + Call Plex + app.get('/video/:/transcode/universal/ping'); // Proxy => /control/ping?uid={UNICORNID} + Call Plex + app.get('/:/timeline'); // Proxy => /control/ping?uid={UNICORNID} + Call Plex + + // Download + app.get('/library/parts/:id1/:id2/file.*'); // 302 => /download?uid={UNICORNID} + + // Forward other to Plex + app.all('*'); +}; \ No newline at end of file diff --git a/src/routes/api.js b/src/routes/api.js new file mode 100644 index 0000000..f1d6e64 --- /dev/null +++ b/src/routes/api.js @@ -0,0 +1,76 @@ +import httpProxy from 'http-proxy'; +import sqlite3 from 'sqlite3'; + +import config from '../config'; +import SessionManager from '../core/sessions'; +import ServersManager from '../core/servers'; + +let RoutesAPI = {}; + +// Calculate scores for all the servers +RoutesAPI.scores = (req, res) => { + res.send(ServersManager.scores()); +}; + +// Returns all the stats of all the transcoders +RoutesAPI.stats = (req, res) => { + res.send(ServersManager.stats()); +}; + +// Save the FFMPEG arguments +// Body: {args: [], env: []} +RoutesAPI.ffmpeg = (req, res) => { + if (!req.body || !req.body.args || !req.body.env) + res.status(400).send({ error: { code: 'INVALID_ARGUMENTS', message: 'Invalid UnicornFFMPEG parameters' } }); + res.send(SessionManager.storeFFmpegParameters = (req.body.args, req.body.env)); +}; + +// Resolve path from file id +RoutesAPI.path = (req, res) => { + try { + const db = new sqlite3.verbose().Database(config.plex.database); + db.get("SELECT * FROM media_parts WHERE id=? LIMIT 0, 1", req.params.id, (err, row) => { + if (row && row.file) + res.send(JSON.stringify(row)); + else + res.status(400).send({ error: { code: 'FILE_NOT_FOUND', message: 'File not found in Plex Database' } }); + db.close(); + }); + } + catch (err) { + res.status(400).send({ error: { code: 'FILE_NOT_FOUND', message: 'File not found in Plex Database' } }); + } +}; + +// Register a new server +// Body: {url: ""} +RoutesAPI.register = (req, res) => { + if (req.body.url) + ServersManager.add(req.body.url) + res.send(ServersManager.list()); +}; + +// Remove a server +// Body: {url: ""} +RoutesAPI.unregister = (req, res) => { + if (req.body.url) + ServersManager.remove(req.body.url) + res.send(ServersManager.list()); +}; + +// Proxy to Plex +RoutesAPI.plex = (req, res) => { + const proxy = httpProxy.createProxyServer({ + target: { + host: config.plex.host, + port: config.plex.port + } + }).on('error', (err, req, res) => { + res.status(400).send({ error: { code: 'PROXY_TIMEOUT', message: 'Plex not respond in time, proxy request fails' } }); + }); + req.url = req.url.slice('/api/plex'.length); + return (proxy.web(req, res)); +}; + +// Export all our API routes +export default RoutesAPI; \ No newline at end of file diff --git a/src/utils.js b/src/utils.js new file mode 100644 index 0000000..aee4143 --- /dev/null +++ b/src/utils.js @@ -0,0 +1,22 @@ +import redisClient from 'redis'; + +import config from './config'; + +export const publicUrl = () => { + return ('http' + ((config.server.ssl) ? 's' : '') + '://' + config.server.host + (([80, 443].indexOf(config.server.port) === -1) ? ':' + config.server.port : '') + '/') +} + +export const internalUrl = () => { + return ('http' + ((config.server.ssl) ? 's' : '') + '://' + config.server.host + ':' + config.server.port + '/') +} + +export const plexUrl = () => { + return ('http://' + config.plex.host + ':' + config.plex.port + '/') +} + +export const redis = redisClient.createClient(config.redis); +redis.on('error', (err) => { + if (err.errno === 'ECONNREFUSED') + return console.error('Failed to connect to REDIS, please check your configuration'); + return console.error(err.errno); +}); diff --git a/utils/getIp.js b/utils/getIp.js deleted file mode 100644 index 826f800..0000000 --- a/utils/getIp.js +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Created by Maxime Baconnais on 03/09/2017. - */ - -const getIp = (req) => { - return (req.headers['cf-connecting-ip'] || req.headers['x-forwarded-for'] || req.connection.remoteAddress); -} - -module.exports = getIp; \ No newline at end of file diff --git a/utils/reloadConf.js b/utils/reloadConf.js deleted file mode 100644 index d9e243f..0000000 --- a/utils/reloadConf.js +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Created by drouar_b on 01/02/2018. - */ - -const path = require('path'); -const reload = require('require-reload')(require); -const config = require('../config'); - -class ReloadConf { - static reloadConf(req, res) { - res.setHeader('Content-Type', 'application/json'); - - try { - reload(path.resolve('config.js')); - let newConf = require('../config'); - - Object.keys(newConf).map((key) => { - config[key] = newConf[key]; - }); - - res.send(JSON.stringify(config)); - } catch (e) { - console.log(e); - res.send(JSON.stringify({ - error: "Config reload Failed" - })) - } - } -} - -module.exports = ReloadConf; \ No newline at end of file From 760d6e4fadde4c2166d7ffe98cb9345e6be955ae Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Tue, 23 Oct 2018 22:20:30 +0200 Subject: [PATCH 002/132] Rework stats from transcoder, and implement new scores --- src/config.js | 4 +- src/core/servers.js | 106 +++++++++++++++++++++++++---------------- src/core/transcodes.js | 16 ------- src/routes.js | 1 - src/routes/api.js | 30 +++--------- src/utils.js | 2 + 6 files changed, 77 insertions(+), 82 deletions(-) delete mode 100644 src/core/transcodes.js diff --git a/src/config.js b/src/config.js index 6b821b4..a7d5cb1 100644 --- a/src/config.js +++ b/src/config.js @@ -22,7 +22,7 @@ export default { password: env.string('REDIS_PASSWORD', ''), db: env.int('REDIS_DB', 0) }, - scan: { - timer: 3000 + scores: { + timeout: env.int('SCORES_TIMEOUT', 10) } }; \ No newline at end of file diff --git a/src/core/servers.js b/src/core/servers.js index 87bf964..934ce34 100644 --- a/src/core/servers.js +++ b/src/core/servers.js @@ -1,71 +1,97 @@ -let servers = []; +import { time } from '../utils'; +import config from '../config'; + +let servers = {}; let ServersManager = {}; -ServersManager.add = (url) => { - if (servers.indexOf(url) !== -1) - return (false); - servers.push(url); - return (true); +// Add/update a server +ServersManager.update = (e) => { + const name = (e.name) ? e.name : (e.url) ? e.url : ''; + if (!name) + return (ServersManager.list()); + servers[name] = { + name, + sessions: ((!Array.isArray(e.sessions)) ? [] : e.sessions.map((s) => ({ + id: ((s.id) ? s.id : false), + status: ((s.status && ['DONE', 'DOWNLOAD', 'TRANSCODE'].indexOf(s.status.toUpperCase()) !== -1) ? s.status.toUpperCase() : false), + codec: ((s.codec) ? s.codec : false), + lastChunkDownload: ((s.lastChunkDownload) ? s.lastChunkDownload : 0) + }))).filter((s) => (s.id !== false && s.status !== false)), + settings: { + maxSessions: ((typeof (e.settings) !== 'undefined' && typeof (e.settings.maxSessions) !== 'undefined') ? parseInt(e.settings.maxSessions) : 0), + maxDownloads: ((typeof (e.settings) !== 'undefined' && typeof (e.settings.maxDownloads) !== 'undefined') ? parseInt(e.settings.maxDownloads) : 0), + maxTranscodes: ((typeof (e.settings) !== 'undefined' && typeof (e.settings.maxTranscodes) !== 'undefined') ? parseInt(e.settings.maxTranscodes) : 0), + }, + url: ((e.url) ? e.url : false), + time: time() + }; + return (ServersManager.list()); }; -ServersManager.remove = (url) => { - servers = server.filter((e) => (e !== url)); - return (true); +// Remove a server +ServersManager.remove = (e) => { + const name = (e.name) ? e.name : (e.url) ? e.url : ''; + delete servers[name]; + return (ServersManager.list()); }; +// List all the servers with scores ServersManager.list = () => { - return (servers); + let output = {}; + Object.keys(servers).forEach((i) => { + output[i] = { ...servers[i], score: ServersManager.score(servers[i]) }; + }); + return (output); } // Calculate server score -ServersManager.calculateScore = (stats) => { - // The configuration is unavailable, the server is probably unavailable - if (!stats) - return (1000); +ServersManager.score = (e) => { + + // The configuration wasn't updated since X seconds, the server is probably unavailable + if (time() - e.time > config.scores.timeout) + return (100); // Default load 0 let load = 0; - // Each transcode add 1 to the load - load += stats.transcoding; + // Add load value for each session + e.sessions.forEach((s) => { + + // Transcode streams + if (s.status === 'TRANSCODE') { + load += 1; + if (s.codec === 'hevc') { + load += 1.5; + } + } + + // Serving streams + if (s.status === 'DONE') { + load += 0.5; + } - // Each HEVC sessions add 1.5 to the load - if (stats.codecs.hevc) - load += stats.codecs.hevc * 1.5; + // Download streams + if (s.status === 'DOWNLOAD') { + load += 0.25; + } + }) // Server already have too much sessions - if (stats.config && stats.sessions >= stats.config.preferredMaxSessions) + if (e.sessions.filter((s) => (['TRANSCODE', 'DONE'].indexOf(s.status) !== -1)).length > e.settings.maxSessions) load += 2.5; // Server already have too much transcodes - if (stats.config && stats.transcoding >= stats.config.preferredMaxTranscodes) + if (e.sessions.filter((s) => (['TRANSCODE'].indexOf(s.status) !== -1)).length > e.settings.maxTranscodes) load += 5; // Server already have too much downloads - if (stats.config && stats.downloads >= stats.config.preferredMaxDownloads) + if (e.sessions.filter((s) => (['DOWNLOAD'].indexOf(s.status) !== -1)).length > e.settings.maxDownloads) load += 1; // Return load return (load); } -// Calculate all the server scores -ServersManager.scores = () => { - let output = {}; - ServersManager.list().forEach((e) => { - output[e] = ServersManager.calculateScore(); // Todo: Bind stats in call - }); - return (output); -} - -// Returns all the server stats -ServersManager.stats = () => { - let output = {}; - ServersManager.list().forEach((e) => { - output[e] = false; // Todo: Bind stats - }); - return (output); -} - +// Returns our ServersManager export default ServersManager; diff --git a/src/core/transcodes.js b/src/core/transcodes.js deleted file mode 100644 index 4ab1906..0000000 --- a/src/core/transcodes.js +++ /dev/null @@ -1,16 +0,0 @@ - -// Debugger -const D = debug('UnicornLoadBalancer:TranscodesManager'); - -let TranscodesManager = {}; - -TranscodesManager.create = () => { - -} - -TranscodesManager.remove = () => { - -} - -// Export our TranscodesManager -export default TranscodesManager; \ No newline at end of file diff --git a/src/routes.js b/src/routes.js index 182165e..60b7833 100644 --- a/src/routes.js +++ b/src/routes.js @@ -7,7 +7,6 @@ export default (app) => { // UnicornLoadBalancer API app.use('/api/sessions', express.static(config.plex.path.sessions)); - app.get('/api/scores', RoutesAPI.scores); app.get('/api/stats', RoutesAPI.stats); app.post('/api/ffmpeg', RoutesAPI.ffmpeg); app.get('/api/path/:id', RoutesAPI.path); diff --git a/src/routes/api.js b/src/routes/api.js index f1d6e64..ed1569b 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -2,27 +2,27 @@ import httpProxy from 'http-proxy'; import sqlite3 from 'sqlite3'; import config from '../config'; -import SessionManager from '../core/sessions'; +import SessionsManager from '../core/sessions'; import ServersManager from '../core/servers'; let RoutesAPI = {}; -// Calculate scores for all the servers -RoutesAPI.scores = (req, res) => { - res.send(ServersManager.scores()); -}; - // Returns all the stats of all the transcoders RoutesAPI.stats = (req, res) => { res.send(ServersManager.stats()); }; +// Save the stats of a server +RoutesAPI.update = (req, res) => { + res.send(ServersManager.update(req.body)); +}; + // Save the FFMPEG arguments // Body: {args: [], env: []} RoutesAPI.ffmpeg = (req, res) => { if (!req.body || !req.body.args || !req.body.env) res.status(400).send({ error: { code: 'INVALID_ARGUMENTS', message: 'Invalid UnicornFFMPEG parameters' } }); - res.send(SessionManager.storeFFmpegParameters = (req.body.args, req.body.env)); + res.send(SessionsManager.storeFFmpegParameters = (req.body.args, req.body.env)); }; // Resolve path from file id @@ -42,22 +42,6 @@ RoutesAPI.path = (req, res) => { } }; -// Register a new server -// Body: {url: ""} -RoutesAPI.register = (req, res) => { - if (req.body.url) - ServersManager.add(req.body.url) - res.send(ServersManager.list()); -}; - -// Remove a server -// Body: {url: ""} -RoutesAPI.unregister = (req, res) => { - if (req.body.url) - ServersManager.remove(req.body.url) - res.send(ServersManager.list()); -}; - // Proxy to Plex RoutesAPI.plex = (req, res) => { const proxy = httpProxy.createProxyServer({ diff --git a/src/utils.js b/src/utils.js index aee4143..d17facc 100644 --- a/src/utils.js +++ b/src/utils.js @@ -20,3 +20,5 @@ redis.on('error', (err) => { return console.error('Failed to connect to REDIS, please check your configuration'); return console.error(err.errno); }); + +export const time = () => (Math.floor((new Date().getTime()) / 1000)); \ No newline at end of file From 5148f495f7c848ca5ead92a74ec77cc423cb7bc6 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 26 Oct 2018 22:53:22 +0200 Subject: [PATCH 003/132] Improvements --- package.json | 1 + src/core/sessions.js | 88 +++++++++++++++++++++++++++++++++++++++++ src/routes/transcode.js | 35 ++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 src/routes/transcode.js diff --git a/package.json b/package.json index beb7332..a9dea30 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "express": "^4.16.3", "getenv": "^0.7.0", "http-proxy": "^1.17.0", + "uniqid": "^5.0.3", "redis": "^2.8.0", "sqlite3": "^4.0.2" } diff --git a/src/core/sessions.js b/src/core/sessions.js index 8a60421..fcccbf6 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -1,4 +1,5 @@ import debug from 'debug'; +import uniqid from 'uniqid'; import config from '../config'; import { publicUrl, plexUrl, redis } from '../utils'; @@ -8,6 +9,93 @@ const D = debug('UnicornLoadBalancer:SessionsManager'); let SessionsManager = {}; +let sessions = [ + { + unicorn: '_UNiC0RN', + session: '', + sessionId: '', + sessionFull: '', + sessionIdentifier: '', + clientIdentifier: '', + args: [], + env: [], + pingUrl: '' + } +]; + +// Parse request to extract session information +SessionsManager.parseSessionFromRequest = (req) => { + const unicorn = (typeof (req.query.unicorn) !== 'undefined') ? { unicorn: req.query.unicorn } : false; + const sessionId = (typeof (req.params.sessionId) !== 'undefined') ? { sessionId: req.params.sessionId } : false; + const session = (typeof (req.query.session) !== 'undefined') ? { session: req.query.session } : false; + const sessionIdentifier = (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined') ? { "X-Plex-Session-Identifier": req.query['X-Plex-Session-Identifier'] } : false; + const clientIdentifier = (typeof (req.query['X-Plex-Client-Identifier']) !== 'undefined') ? { "X-Plex-Client-Identifier": req.query['X-Plex-Client-Identifier'] } : false; + return { + ...unicorn, + ...sessionId, + ...session, + ...sessionIdentifier, + ...clientIdentifier + } +}; + +// Get a session from its values +SessionsManager.getSessionFromRequest = (search) => { + const filtered = sessions.filter(e => { + let keys = Object.keys(search); + for (let i = 0; i < keys.length; i++) { + if (e[keys[i]] === search[keys[i]]) + return (true); + } + return (false); + }); + if (filtered.length === 0) + return (false); + return (filtered[0]); +}; + +// Get a session position from its values +SessionsManager.getIdFromRequest = (search) => { + for (let idx = 0; idx < sessions.length; idx++) { + let keys = Object.keys(search); + for (let i = 0; i < keys.length; i++) { + if (e[keys[i]] === search[keys[i]]) + return (idx); + } + } + return (false); +}; + +// Update the session stored +SessionsManager.updateSessionFromRequest = (req) => { + const args = SessionsManager.parseSessionFromRequest(req); + const search = SessionsManager.getSessionFromRequest(search); + const idx = SessionsManager.getIdFromRequest(search); + + if (!search) { + sessions.push({ + ...{ + unicorn: uniqid(), + session: '', + sessionId: '', + sessionFull: '', + sessionIdentifier: '', + clientIdentifier: '', + args: [], + env: [], + pingUrl: '' + }, + ...args + }); + return (true); + } + sessions[idx] = { + ...sessions[idx], + ...args + }; + return (false); +}; + // Parse FFmpeg parameters with internal bindings SessionsManager.parseFFmpegParameters = (args) => { // Extract Session ID diff --git a/src/routes/transcode.js b/src/routes/transcode.js new file mode 100644 index 0000000..8b8245f --- /dev/null +++ b/src/routes/transcode.js @@ -0,0 +1,35 @@ +import config from '../config'; +import SessionsManager from '../core/sessions'; +import ServersManager from '../core/servers'; + +let RoutesTranscode = {}; + +RoutesTranscode.redirect = (req, res) => { + //SessionsManager.updateSessionFromRequest(req); + + // Choose server + + // 302 Bro' +}; + +RoutesTranscode.ping = (req, res) => { + /*proxy.web(req, res); + + const sessionId = serverManager.getSession(req); + const serverUrl = serverManager.chooseServer(sessionId, getIp(req)); + + if (typeof (serverUrl) !== 'undefined') + request(serverUrl + '/video/:/transcode/universal/ping?session=' + sessionId);*/ +}; + +RoutesTranscode.timeline = (req, res) => { + + /*if (req.query.state == 'stopped' || (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined' && typeof (serverManager.stoppedSessions[req.query['X-Plex-Session-Identifier']]) != 'undefined')) { + request(serverUrl + '/video/:/transcode/universal/stop?session=' + sessionId); + } + else { + proxy.web(req, res); + }*/ +}; + +export default RoutesTranscode; \ No newline at end of file From cbb50ca8f9e21e4d650aa5ce0c50bd058812965f Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 26 Oct 2018 22:56:38 +0200 Subject: [PATCH 004/132] Syntax fixes --- src/core/servers.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/core/servers.js b/src/core/servers.js index 934ce34..644576d 100644 --- a/src/core/servers.js +++ b/src/core/servers.js @@ -45,6 +45,25 @@ ServersManager.list = () => { return (output); } +// Chose best server +ServersManager.chooseServer = (ip = false) => { + return (new Promise((resolve, reject) => { + let tab = []; + const list = ServersManager.list(); + Object.keys(list).forEach((i) => { + tab.push(list[i]); + }); + tab.sort((e) => (e.score)); + if (typeof (tab[0]) === 'undefined') + return (false); + + /*fetch(tab[0].url + '/api/resolve?ip=' + ip).then(res => res.json()) + .then(json => console.log(json)); + });*/ + // TODO : Resolve URL from transcoder here + })); +}; + // Calculate server score ServersManager.score = (e) => { From 1645ef04f990ced4adeb60e51068dd145e4f593a Mon Sep 17 00:00:00 2001 From: Benjamin DROUARD Date: Sat, 27 Oct 2018 12:48:52 +0200 Subject: [PATCH 005/132] Improvements --- package.json | 1 + src/core/servers.js | 19 +++++++++ src/core/sessions.js | 88 +++++++++++++++++++++++++++++++++++++++++ src/routes/transcode.js | 35 ++++++++++++++++ 4 files changed, 143 insertions(+) create mode 100644 src/routes/transcode.js diff --git a/package.json b/package.json index beb7332..a9dea30 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "express": "^4.16.3", "getenv": "^0.7.0", "http-proxy": "^1.17.0", + "uniqid": "^5.0.3", "redis": "^2.8.0", "sqlite3": "^4.0.2" } diff --git a/src/core/servers.js b/src/core/servers.js index 934ce34..644576d 100644 --- a/src/core/servers.js +++ b/src/core/servers.js @@ -45,6 +45,25 @@ ServersManager.list = () => { return (output); } +// Chose best server +ServersManager.chooseServer = (ip = false) => { + return (new Promise((resolve, reject) => { + let tab = []; + const list = ServersManager.list(); + Object.keys(list).forEach((i) => { + tab.push(list[i]); + }); + tab.sort((e) => (e.score)); + if (typeof (tab[0]) === 'undefined') + return (false); + + /*fetch(tab[0].url + '/api/resolve?ip=' + ip).then(res => res.json()) + .then(json => console.log(json)); + });*/ + // TODO : Resolve URL from transcoder here + })); +}; + // Calculate server score ServersManager.score = (e) => { diff --git a/src/core/sessions.js b/src/core/sessions.js index 8a60421..19266fd 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -1,4 +1,5 @@ import debug from 'debug'; +import uniqid from 'uniqid'; import config from '../config'; import { publicUrl, plexUrl, redis } from '../utils'; @@ -8,6 +9,93 @@ const D = debug('UnicornLoadBalancer:SessionsManager'); let SessionsManager = {}; +let sessions = [ + { + unicorn: '_UNICORN', + session: '', + sessionId: '', + sessionFull: '', + sessionIdentifier: '', + clientIdentifier: '', + args: [], + env: [], + pingUrl: '' + } +]; + +// Parse request to extract session information +SessionsManager.parseSessionFromRequest = (req) => { + const unicorn = (typeof (req.query.unicorn) !== 'undefined') ? { unicorn: req.query.unicorn } : false; + const sessionId = (typeof (req.params.sessionId) !== 'undefined') ? { sessionId: req.params.sessionId } : false; + const session = (typeof (req.query.session) !== 'undefined') ? { session: req.query.session } : false; + const sessionIdentifier = (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined') ? { "X-Plex-Session-Identifier": req.query['X-Plex-Session-Identifier'] } : false; + const clientIdentifier = (typeof (req.query['X-Plex-Client-Identifier']) !== 'undefined') ? { "X-Plex-Client-Identifier": req.query['X-Plex-Client-Identifier'] } : false; + return { + ...unicorn, + ...sessionId, + ...session, + ...sessionIdentifier, + ...clientIdentifier + } +}; + +// Get a session from its values +SessionsManager.getSessionFromRequest = (search) => { + const filtered = sessions.filter(e => { + let keys = Object.keys(search); + for (let i = 0; i < keys.length; i++) { + if (e[keys[i]] === search[keys[i]]) + return (true); + } + return (false); + }); + if (filtered.length === 0) + return (false); + return (filtered[0]); +}; + +// Get a session position from its values +SessionsManager.getIdFromRequest = (search) => { + for (let idx = 0; idx < sessions.length; idx++) { + let keys = Object.keys(search); + for (let i = 0; i < keys.length; i++) { + if (e[keys[i]] === search[keys[i]]) + return (idx); + } + } + return (false); +}; + +// Update the session stored +SessionsManager.updateSessionFromRequest = (req) => { + const args = SessionsManager.parseSessionFromRequest(req); + const search = SessionsManager.getSessionFromRequest(search); + const idx = SessionsManager.getIdFromRequest(search); + + if (!search) { + sessions.push({ + ...{ + unicorn: uniqid(), + session: '', + sessionId: '', + sessionFull: '', + sessionIdentifier: '', + clientIdentifier: '', + args: [], + env: [], + pingUrl: '' + }, + ...args + }); + return (true); + } + sessions[idx] = { + ...sessions[idx], + ...args + }; + return (false); +}; + // Parse FFmpeg parameters with internal bindings SessionsManager.parseFFmpegParameters = (args) => { // Extract Session ID diff --git a/src/routes/transcode.js b/src/routes/transcode.js new file mode 100644 index 0000000..8b8245f --- /dev/null +++ b/src/routes/transcode.js @@ -0,0 +1,35 @@ +import config from '../config'; +import SessionsManager from '../core/sessions'; +import ServersManager from '../core/servers'; + +let RoutesTranscode = {}; + +RoutesTranscode.redirect = (req, res) => { + //SessionsManager.updateSessionFromRequest(req); + + // Choose server + + // 302 Bro' +}; + +RoutesTranscode.ping = (req, res) => { + /*proxy.web(req, res); + + const sessionId = serverManager.getSession(req); + const serverUrl = serverManager.chooseServer(sessionId, getIp(req)); + + if (typeof (serverUrl) !== 'undefined') + request(serverUrl + '/video/:/transcode/universal/ping?session=' + sessionId);*/ +}; + +RoutesTranscode.timeline = (req, res) => { + + /*if (req.query.state == 'stopped' || (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined' && typeof (serverManager.stoppedSessions[req.query['X-Plex-Session-Identifier']]) != 'undefined')) { + request(serverUrl + '/video/:/transcode/universal/stop?session=' + sessionId); + } + else { + proxy.web(req, res); + }*/ +}; + +export default RoutesTranscode; \ No newline at end of file From 382a4684b71cf340c9ea0215b1c544fdc17d30ca Mon Sep 17 00:00:00 2001 From: Benjamin DROUARD Date: Sat, 27 Oct 2018 14:39:27 +0200 Subject: [PATCH 006/132] Redis and Local session stores --- src/core/sessions.js | 6 +--- src/store/local.js | 52 +++++++++++++++++++++++++++++++ src/store/redis.js | 74 ++++++++++++++++++++++++++++++++++++++++++++ src/utils.js | 30 ++++++++++++------ 4 files changed, 147 insertions(+), 15 deletions(-) create mode 100644 src/store/local.js create mode 100644 src/store/redis.js diff --git a/src/core/sessions.js b/src/core/sessions.js index 19266fd..b279513 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -2,7 +2,7 @@ import debug from 'debug'; import uniqid from 'uniqid'; import config from '../config'; -import { publicUrl, plexUrl, redis } from '../utils'; +import { publicUrl, plexUrl } from '../utils'; // Debugger const D = debug('UnicornLoadBalancer:SessionsManager'); @@ -151,10 +151,6 @@ SessionsManager.parseFFmpegParameters = (args) => { // Store the FFMPEG parameters in RedisCache SessionsManager.storeFFmpegParameters = (args, env) => { const parsedArguments = SessionsManager.parseFFmpegParameters(args); - redis.set(propersessionid, JSON.stringify({ - args: parsedArguments, - env - })); return (parsedArguments); }; diff --git a/src/store/local.js b/src/store/local.js new file mode 100644 index 0000000..235135b --- /dev/null +++ b/src/store/local.js @@ -0,0 +1,52 @@ +import EventEmitter from 'events'; + +class LocalSessionStore { + constructor() { + this.sessionEvents = new EventEmitter(); + this.sessionStore = {}; + } + + /** + * Get a session, or wait for it for 10s + * @param sessionId + * @returns {Promise} + */ + get(sessionId) { + return new Promise((resolve, reject) => { + if (sessionId in this.sessionStore) + return resolve(this.sessionStore[sessionId]); + + let timeout = null; + + let eventCb = (...args) => { + clearTimeout(timeout); + this.sessionEvents.removeListener(sessionId, eventCb); + resolve(...args); + }; + + let timeoutCb = () => { + this.sessionEvents.removeListener(sessionId, eventCb); + reject('timeout'); + }; + + timeout = setTimeout(timeoutCb, 10000); + this.sessionEvents.on(sessionId, eventCb); + }) + } + + /** + * Store a value in the store and trigger the pending gets + * @param sessionId + * @param value + * @returns {Promise} + */ + set(sessionId, value) { + return new Promise((resolve) => { + this.sessionStore[sessionId] = value; + this.sessionEvents.emit(sessionId, value); + resolve('OK'); + }) + } +} + +export default LocalSessionStore; \ No newline at end of file diff --git a/src/store/redis.js b/src/store/redis.js new file mode 100644 index 0000000..6d567b9 --- /dev/null +++ b/src/store/redis.js @@ -0,0 +1,74 @@ +import {getRedisClient} from '../utils'; +import config from "../config"; + +class RedisSessionStore { + constructor() { + this.redis = getRedisClient(); + this.redisSubscriber = this.redis.duplicate(); + } + + _parseSession(session) { + return new Promise((resolve, reject) => { + try { + resolve(JSON.parse(session)) + } catch(err) { + reject(err) + } + }) + } + + /** + * Get a session, or wait for it for 10s + * @param sessionId + * @returns {Promise} + */ + get(sessionId) { + return new Promise((resolve, reject) => { + this.redis.get(sessionId, (err, session) => { + if (err) + return reject(err); + if (session != null) + return resolve(this._parseSession(session)); + + let redisSubKey = "__keyspace@" + config.redis.db + "__:" + sessionId; + + let timeout = setTimeout(() => { + this.redisSubscriber.unsubscribe(redisSubKey); + reject('timeout'); + }, 10000); + + this.redisSubscriber.on("message", (eventKey, action) => { + if (action !== 'set' || eventKey !== redisSubKey) + return; + + clearTimeout(timeout); + this.redisSubscriber.unsubscribe(redisSubKey); + this.redis.get(sessionId, (err, session) => { + if (err) + return reject(err); + return resolve(this._parseSession(session)); + }) + }); + this.redisSubscriber.subscribe(redisSubKey) + }) + }) + } + + /** + * Store a value in the store and trigger the pending gets + * @param sessionId + * @param value + * @returns {Promise} + */ + set(sessionId, value) { + return new Promise((resolve, reject) => { + this.redis.set(sessionId, JSON.stringify(value), (err) => { + if (err) + return reject(err); + resolve('OK'); + }) + }) + } +} + +export default RedisSessionStore; \ No newline at end of file diff --git a/src/utils.js b/src/utils.js index d17facc..e25bec6 100644 --- a/src/utils.js +++ b/src/utils.js @@ -4,21 +4,31 @@ import config from './config'; export const publicUrl = () => { return ('http' + ((config.server.ssl) ? 's' : '') + '://' + config.server.host + (([80, 443].indexOf(config.server.port) === -1) ? ':' + config.server.port : '') + '/') -} +}; export const internalUrl = () => { return ('http' + ((config.server.ssl) ? 's' : '') + '://' + config.server.host + ':' + config.server.port + '/') -} +}; export const plexUrl = () => { return ('http://' + config.plex.host + ':' + config.plex.port + '/') -} - -export const redis = redisClient.createClient(config.redis); -redis.on('error', (err) => { - if (err.errno === 'ECONNREFUSED') - return console.error('Failed to connect to REDIS, please check your configuration'); - return console.error(err.errno); -}); +}; + +export const getRedisClient = () => { + if (config.redis.password === '') + delete config.redis.password; + + let redis = redisClient.createClient(config.redis); + redis.on('error', (err) => { + if (err.errno === 'ECONNREFUSED') + return console.error('Failed to connect to REDIS, please check your configuration'); + return console.error(err.errno); + }); + + redis.on('connect', () => { + redis.send_command('config', ['set', 'notify-keyspace-events', 'KEA']) + }); + return redis; +}; export const time = () => (Math.floor((new Date().getTime()) / 1000)); \ No newline at end of file From 5648218a40f26509fbcff9b9feb3be805cce04cd Mon Sep 17 00:00:00 2001 From: Benjamin DROUARD Date: Sat, 27 Oct 2018 15:14:13 +0200 Subject: [PATCH 007/132] SessionStore choose the backend depending on redis config --- src/config.js | 4 +++- src/core/sessions.js | 4 +++- src/store/index.js | 20 ++++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 src/store/index.js diff --git a/src/config.js b/src/config.js index a7d5cb1..43aa569 100644 --- a/src/config.js +++ b/src/config.js @@ -1,5 +1,7 @@ import env from 'getenv'; +env.disableErrors(); + export default { version: '2.0.0', server: { @@ -17,7 +19,7 @@ export default { } }, redis: { - host: env.string('REDIS_HOST', '127.0.0.1'), + host: env('REDIS_HOST', undefined), port: env.int('REDIS_PORT', 6379), password: env.string('REDIS_PASSWORD', ''), db: env.int('REDIS_DB', 0) diff --git a/src/core/sessions.js b/src/core/sessions.js index b279513..1b12b92 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -3,6 +3,7 @@ import uniqid from 'uniqid'; import config from '../config'; import { publicUrl, plexUrl } from '../utils'; +import SessionStore from '../store'; // Debugger const D = debug('UnicornLoadBalancer:SessionsManager'); @@ -146,11 +147,12 @@ SessionsManager.parseFFmpegParameters = (args) => { finalArgs.push(e); }); return (finalArgs); -} +}; // Store the FFMPEG parameters in RedisCache SessionsManager.storeFFmpegParameters = (args, env) => { const parsedArguments = SessionsManager.parseFFmpegParameters(args); + //TODO Store with SessionStore.set(sessionId, { args: parsedArguments, env: env }).then(...) return (parsedArguments); }; diff --git a/src/store/index.js b/src/store/index.js new file mode 100644 index 0000000..4df48b2 --- /dev/null +++ b/src/store/index.js @@ -0,0 +1,20 @@ +import config from '../config'; +import RedisSessionStore from './redis'; +import LocalSessionStore from './local'; +import debug from 'debug'; +// Debugger +const D = debug('UnicornLoadBalancer:SessionStore'); + + +let SessionStore; + +if (config.redis.host !== 'undefined') { + D('Using redis as session store'); + SessionStore = new RedisSessionStore(); +} else { + D('Redis not found, fallback on LocalSessionStore'); + D('WARNING: On restart all sessions will be lost'); + SessionStore = new LocalSessionStore(); +} + +export default SessionStore; From bcc598f404823a852351877d6fc8155054112faa Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Sun, 11 Nov 2018 21:45:13 +0100 Subject: [PATCH 008/132] Routes ready but not tested --- src/core/sessions.js | 6 ++- src/routes/api.js | 2 +- src/routes/index.js | 2 +- src/routes/transcode.js | 117 +++++++++++++++++++++++++++++++--------- 4 files changed, 99 insertions(+), 28 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 68dd33e..2cf55e8 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -19,7 +19,8 @@ let sessions = [ clientIdentifier: '', args: [], env: [], - pingUrl: '' + serverUrl: '', + pingUrl: '', } ]; @@ -84,6 +85,7 @@ SessionsManager.updateSession = (args) => { clientIdentifier: '', args: [], env: [], + serverUrl: '', pingUrl: '', ...args }); @@ -161,7 +163,7 @@ SessionsManager.storeFFmpegParameters = (args, env) => { session: parsed.session, sessionFull: parsed.sessionFull }); - SessionStore.set(session.unicorn, session).then(() => { + SessionStore.set(session.session, session).then(() => { }).catch((err) => { diff --git a/src/routes/api.js b/src/routes/api.js index bf95db6..0b7ee8e 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -58,7 +58,7 @@ RoutesAPI.plex = (req, res) => { // Returns sessions from UnicornID RoutesAPI.session = (req, res) => { - SessionStore.get(req.params.unicorn).then((data) => { + SessionStore.get(req.params.session).then((data) => { res.send(data); }).catch((err) => { res.status(400).send({ error: { code: 'SESSION_TIMEOUT', message: 'The session wasn\'t launched in time, request fails' } }); diff --git a/src/routes/index.js b/src/routes/index.js index b2cc21d..db7d58c 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -16,7 +16,7 @@ export default (app) => { app.post('/api/ffmpeg', RoutesAPI.ffmpeg); app.get('/api/path/:id', RoutesAPI.path); app.post('/api/update', RoutesAPI.update); - app.get('/api/session/:unicorn', RoutesAPI.session); + app.get('/api/session/:session', RoutesAPI.session); app.all('/api/plex/*', RoutesAPI.plex); // MPEG Dash support diff --git a/src/routes/transcode.js b/src/routes/transcode.js index 524cca5..beb576b 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -1,52 +1,121 @@ -import config from '../config'; +import debug from 'debug'; +import fetch from 'node-fetch'; + import SessionsManager from '../core/sessions'; import ServersManager from '../core/servers'; +import Proxy from './proxy'; -let RoutesTranscode = {}; +// Debugger +const D = debug('UnicornLoadBalancer:transcode'); -// TODO: Some stuff to do here :P +let RoutesTranscode = {}; RoutesTranscode.redirect = (req, res) => { SessionsManager.updateSessionFromRequest(req); - // Choose server if we don't have (check in server manager) - // Save server in sessionManager + const search = SessionsManager.parseSessionFromRequest(req); + const session = SessionsManager.getSessionFromRequest(search); - ServersManager.chooseServer(req.connection.remoteAddress); + const redirectRequest = (server) => { + if (server) { + res.writeHead(302, { + 'Location': (server + req.url + '&unicorn=' + session.unicorn) + }); + res.end(); + D('Send 302 for ' + session.session + ' to ' + server); + return; + } + D('Fail to 302 for ' + session.session + ' to ' + server); + }; - /*res.writeHead(302, { - 'Location': serverUrl + req.url + '&unicorn=' + UNICORNID - }); - res.end(); - - debug('Send 302 for ' + sessionId + ' to ' + serverUrl);*/ + if (session.serverUrl) { + return (redirectRequest(session.serverUrl)); + } + + ServersManager.chooseServer(req.connection.remoteAddress).then((server) => { + SessionsManager.updateSession({ ...session, serverUrl: server }); + return (redirectRequest(server)); + }); }; RoutesTranscode.ping = (req, res) => { - /*proxy.web(req, res); // + '&unicorn=' + UNICORNID + SessionsManager.updateSessionFromRequest(req); - const sessionId = serverManager.getSession(req); - const serverUrl = serverManager.chooseServer(sessionId, getIp(req)); + const search = SessionsManager.parseSessionFromRequest(req); + const session = SessionsManager.getSessionFromRequest(search); - if (typeof (serverUrl) !== 'undefined') - request(serverUrl + '/video/:/transcode/universal/ping?session=' + sessionId);*/// + '&unicorn=' + UNICORNID + req.url += '&unicorn=' + session.unicorn; + Proxy.web(req, res); + + const pingRequest = (server) => { + fetch(server + '/' + req.params.formatType + '/:/transcode/universal/ping?session=' + session.session + '&unicorn=' + session.unicorn); + }; + + if (session.serverUrl) { + return (pingRequest(session.serverUrl)); + } + + ServersManager.chooseServer(req.connection.remoteAddress).then((server) => { + SessionsManager.updateSession({ ...session, serverUrl: server }); + return (pingRequest(server)); + }); }; RoutesTranscode.timeline = (req, res) => { + SessionsManager.updateSessionFromRequest(req); + + const search = SessionsManager.parseSessionFromRequest(req); + const session = SessionsManager.getSessionFromRequest(search); - // /!\ Need to catch Dash end here + req.url += '&unicorn=' + session.unicorn; + Proxy.web(req, res); - /*if (req.query.state == 'stopped' || (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined' && typeof (serverManager.stoppedSessions[req.query['X-Plex-Session-Identifier']]) != 'undefined')) { - request(serverUrl + '/video/:/transcode/universal/stop?session=' + sessionId);// + '&unicorn=' + UNICORNID + const pingRequest = (server) => { + fetch(server + '/' + req.params.formatType + '/:/transcode/universal/ping?session=' + session.session + '&unicorn=' + session.unicorn); + }; + + const stopRequest = (server) => { + fetch(server + '/' + req.params.formatType + '/:/transcode/universal/stop?session=' + session.session + '&unicorn=' + session.unicorn); + }; + + const autoRequest = (server) => { + if (req.query.state == 'stopped') + stopRequest(server); + else + pingRequest(server); + } + + if (session.serverUrl) { + return (autoRequest(session.serverUrl)); } - else { - proxy.web(req, res); - }*/ + + ServersManager.chooseServer(req.connection.remoteAddress).then((server) => { + SessionsManager.updateSession({ ...session, serverUrl: server }); + return (autoRequest(server)); + }); }; RoutesTranscode.stop = (req, res) => { + SessionsManager.updateSessionFromRequest(req); + + const search = SessionsManager.parseSessionFromRequest(req); + const session = SessionsManager.getSessionFromRequest(search); + + req.url += '&unicorn=' + session.unicorn; + Proxy.web(req, res); + + const stopRequest = (server) => { + fetch(server + '/' + req.params.formatType + '/:/transcode/universal/stop?session=' + session.session + '&unicorn=' + session.unicorn); + }; + + if (session.serverUrl) { + return (stopRequest(session.serverUrl)); + } - // ??? + ServersManager.chooseServer(req.connection.remoteAddress).then((server) => { + SessionsManager.updateSession({ ...session, serverUrl: server }); + return (stopRequest(server)); + }); }; export default RoutesTranscode; \ No newline at end of file From 366895bfd4eba80eb75e50c5040e68a9b196e461 Mon Sep 17 00:00:00 2001 From: Maxime BACONNAIS Date: Mon, 12 Nov 2018 16:34:41 +0100 Subject: [PATCH 009/132] Fix import --- src/routes/proxy.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/routes/proxy.js b/src/routes/proxy.js index 6ba5971..bd097d3 100644 --- a/src/routes/proxy.js +++ b/src/routes/proxy.js @@ -1,3 +1,4 @@ +import httpProxy from 'http-proxy'; import config from '../config'; let RoutesProxy = {}; From 16cf1a3a307a4893ce75222d9e13b3e9fb55d37b Mon Sep 17 00:00:00 2001 From: Maxime BACONNAIS Date: Mon, 12 Nov 2018 16:41:36 +0100 Subject: [PATCH 010/132] Fix scope --- src/core/sessions.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 2cf55e8..d3f2a2c 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -58,7 +58,7 @@ SessionsManager.getIdFromRequest = (search) => { let keys = Object.keys(search).filter(e => (['args', 'env', 'pingUrl'].indexOf(e) === -1)); for (let idx = 0; idx < sessions.length; idx++) { for (let i = 0; i < keys.length; i++) { - if (e[keys[i]] === search[keys[i]] && e[keys[i]]) + if (sessions[idx][keys[i]] === search[keys[i]] && sessions[idx][keys[i]]) return (idx); } } @@ -172,4 +172,4 @@ SessionsManager.storeFFmpegParameters = (args, env) => { }; // Export our SessionsManager -export default SessionsManager; \ No newline at end of file +export default SessionsManager; From 9490eb07b15ece1fa6efda7f48f6e15f431064d2 Mon Sep 17 00:00:00 2001 From: Maxime BACONNAIS Date: Mon, 12 Nov 2018 17:07:12 +0100 Subject: [PATCH 011/132] Fix /api/stats --- src/routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/api.js b/src/routes/api.js index 0b7ee8e..a853fff 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -9,7 +9,7 @@ let RoutesAPI = {}; // Returns all the stats of all the transcoders RoutesAPI.stats = (req, res) => { - res.send(ServersManager.stats()); + res.send(ServersManager.list()); }; // Save the stats of a server From 0aacf2310b63f5f450938a79d5b7124499c7c3f8 Mon Sep 17 00:00:00 2001 From: Benjamin DROUARD Date: Mon, 12 Nov 2018 17:42:27 +0100 Subject: [PATCH 012/132] Resolve URL from transcoder --- src/core/servers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/servers.js b/src/core/servers.js index 69e4005..94d8658 100644 --- a/src/core/servers.js +++ b/src/core/servers.js @@ -63,7 +63,7 @@ ServersManager.chooseServer = (ip = false) => { .then(body => { // TODO: Adapt the server handshake console.log(body); - return resolve('URL') + return resolve(body.client) }); })); }; From ba80ff10923e3f2b6c3645123d08735a5a26f08f Mon Sep 17 00:00:00 2001 From: Maxime BACONNAIS Date: Mon, 12 Nov 2018 17:50:20 +0100 Subject: [PATCH 013/132] Fix SessionStore import --- src/routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/routes/api.js b/src/routes/api.js index a853fff..5fbd0db 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -2,6 +2,7 @@ import httpProxy from 'http-proxy'; import sqlite3 from 'sqlite3'; import config from '../config'; +import SessionStore from '../store'; import SessionsManager from '../core/sessions'; import ServersManager from '../core/servers'; From 1e5213200226d206d47db3be3a8b9e362cffa778 Mon Sep 17 00:00:00 2001 From: Maxime BACONNAIS Date: Mon, 12 Nov 2018 17:53:55 +0100 Subject: [PATCH 014/132] Remove debug --- src/core/servers.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/core/servers.js b/src/core/servers.js index 94d8658..9908b51 100644 --- a/src/core/servers.js +++ b/src/core/servers.js @@ -61,8 +61,6 @@ ServersManager.chooseServer = (ip = false) => { fetch(tab[0].url + '/api/resolve?ip=' + ip) .then(res => res.json()) .then(body => { - // TODO: Adapt the server handshake - console.log(body); return resolve(body.client) }); })); From 15f8cf351cbc4410e327f63b3457d4bb4b07469d Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 12 Nov 2018 19:02:26 +0100 Subject: [PATCH 015/132] Fix variable name --- src/routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/api.js b/src/routes/api.js index 5fbd0db..7832296 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -23,7 +23,7 @@ RoutesAPI.update = (req, res) => { RoutesAPI.ffmpeg = (req, res) => { if (!req.body || !req.body.args || !req.body.env) res.status(400).send({ error: { code: 'INVALID_ARGUMENTS', message: 'Invalid UnicornFFMPEG parameters' } }); - res.send(SessionsManager.storeFFmpegParameters(req.body.args, req.body.env)); + res.send(SessionsManager.storeFFmpegParameters(req.body.arg, req.body.env)); }; // Resolve path from file id From 648dc5054c77dcb9efcb4eb1c6f8340db8003331 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 12 Nov 2018 19:03:21 +0100 Subject: [PATCH 016/132] Session default --- src/core/sessions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index d3f2a2c..a1151e7 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -99,7 +99,7 @@ SessionsManager.updateSession = (args) => { }; // Parse FFmpeg parameters with internal bindings -SessionsManager.parseFFmpegParameters = (args, env) => { +SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { // Extract Session ID const regex = /^http\:\/\/127.0.0.1:32400\/video\/:\/transcode\/session\/(.*)\/progress$/; const sessions = args.filter(e => (regex.test(e))).map(e => (e.match(regex)[1])) From 953a0348da65d281957f5b9e0097cfb04c3d6113 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 12 Nov 2018 19:06:46 +0100 Subject: [PATCH 017/132] Fix ffmpeg endpoint --- src/routes/api.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routes/api.js b/src/routes/api.js index 7832296..daf118e 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -21,9 +21,9 @@ RoutesAPI.update = (req, res) => { // Save the FFMPEG arguments // Body: {args: [], env: []} RoutesAPI.ffmpeg = (req, res) => { - if (!req.body || !req.body.args || !req.body.env) - res.status(400).send({ error: { code: 'INVALID_ARGUMENTS', message: 'Invalid UnicornFFMPEG parameters' } }); - res.send(SessionsManager.storeFFmpegParameters(req.body.arg, req.body.env)); + if (!req.body || !req.body.arg || !req.body.env) + return (res.status(400).send({ error: { code: 'INVALID_ARGUMENTS', message: 'Invalid UnicornFFMPEG parameters' } })); + return (res.send(SessionsManager.storeFFmpegParameters(req.body.arg, req.body.env))); }; // Resolve path from file id From 616b51e83fa180404814700147bb39b2520d917c Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 12 Nov 2018 19:09:33 +0100 Subject: [PATCH 018/132] Fix Proxy.web --- src/routes/transcode.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index beb576b..c8ef109 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -45,7 +45,7 @@ RoutesTranscode.ping = (req, res) => { const session = SessionsManager.getSessionFromRequest(search); req.url += '&unicorn=' + session.unicorn; - Proxy.web(req, res); + Proxy.plex(req, res); const pingRequest = (server) => { fetch(server + '/' + req.params.formatType + '/:/transcode/universal/ping?session=' + session.session + '&unicorn=' + session.unicorn); @@ -68,7 +68,7 @@ RoutesTranscode.timeline = (req, res) => { const session = SessionsManager.getSessionFromRequest(search); req.url += '&unicorn=' + session.unicorn; - Proxy.web(req, res); + Proxy.plex(req, res); const pingRequest = (server) => { fetch(server + '/' + req.params.formatType + '/:/transcode/universal/ping?session=' + session.session + '&unicorn=' + session.unicorn); @@ -102,7 +102,7 @@ RoutesTranscode.stop = (req, res) => { const session = SessionsManager.getSessionFromRequest(search); req.url += '&unicorn=' + session.unicorn; - Proxy.web(req, res); + Proxy.plex(req, res); const stopRequest = (server) => { fetch(server + '/' + req.params.formatType + '/:/transcode/universal/stop?session=' + session.session + '&unicorn=' + session.unicorn); From 495d7cf60756854dcf17be652f5137ebc8bfe49c Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 12 Nov 2018 19:16:55 +0100 Subject: [PATCH 019/132] Change ping/stop endpoints --- src/routes/transcode.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index c8ef109..814873f 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -48,7 +48,7 @@ RoutesTranscode.ping = (req, res) => { Proxy.plex(req, res); const pingRequest = (server) => { - fetch(server + '/' + req.params.formatType + '/:/transcode/universal/ping?session=' + session.session + '&unicorn=' + session.unicorn); + fetch(server + '/api/ping?session=' + session.session + '&unicorn=' + session.unicorn); }; if (session.serverUrl) { @@ -71,11 +71,11 @@ RoutesTranscode.timeline = (req, res) => { Proxy.plex(req, res); const pingRequest = (server) => { - fetch(server + '/' + req.params.formatType + '/:/transcode/universal/ping?session=' + session.session + '&unicorn=' + session.unicorn); + fetch(server + '/api/ping?session=' + session.session + '&unicorn=' + session.unicorn); }; const stopRequest = (server) => { - fetch(server + '/' + req.params.formatType + '/:/transcode/universal/stop?session=' + session.session + '&unicorn=' + session.unicorn); + fetch(server + '/api/stop?session=' + session.session + '&unicorn=' + session.unicorn); }; const autoRequest = (server) => { @@ -105,7 +105,7 @@ RoutesTranscode.stop = (req, res) => { Proxy.plex(req, res); const stopRequest = (server) => { - fetch(server + '/' + req.params.formatType + '/:/transcode/universal/stop?session=' + session.session + '&unicorn=' + session.unicorn); + fetch(server + '/api/stop?session=' + session.session + '&unicorn=' + session.unicorn); }; if (session.serverUrl) { From 2d8d83a3c8bd9ef898cbff62f3ef00e42ca36ad3 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 12 Nov 2018 19:22:01 +0100 Subject: [PATCH 020/132] Fix IP detection with reverse proxy --- src/routes/transcode.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index 814873f..73554cc 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -32,7 +32,7 @@ RoutesTranscode.redirect = (req, res) => { return (redirectRequest(session.serverUrl)); } - ServersManager.chooseServer(req.connection.remoteAddress).then((server) => { + ServersManager.chooseServer(req.headers['x-forwarded-for'] || req.connection.remoteAddress).then((server) => { SessionsManager.updateSession({ ...session, serverUrl: server }); return (redirectRequest(server)); }); @@ -55,7 +55,7 @@ RoutesTranscode.ping = (req, res) => { return (pingRequest(session.serverUrl)); } - ServersManager.chooseServer(req.connection.remoteAddress).then((server) => { + ServersManager.chooseServer(req.headers['x-forwarded-for'] || req.connection.remoteAddress).then((server) => { SessionsManager.updateSession({ ...session, serverUrl: server }); return (pingRequest(server)); }); @@ -89,7 +89,7 @@ RoutesTranscode.timeline = (req, res) => { return (autoRequest(session.serverUrl)); } - ServersManager.chooseServer(req.connection.remoteAddress).then((server) => { + ServersManager.chooseServer(req.headers['x-forwarded-for'] || req.connection.remoteAddress).then((server) => { SessionsManager.updateSession({ ...session, serverUrl: server }); return (autoRequest(server)); }); @@ -112,7 +112,7 @@ RoutesTranscode.stop = (req, res) => { return (stopRequest(session.serverUrl)); } - ServersManager.chooseServer(req.connection.remoteAddress).then((server) => { + ServersManager.chooseServer(req.headers['x-forwarded-for'] || req.connection.remoteAddress).then((server) => { SessionsManager.updateSession({ ...session, serverUrl: server }); return (stopRequest(server)); }); From 099dd31e4eaa0712e2b057d580f611fa7ca3e3da Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 12 Nov 2018 21:27:20 +0100 Subject: [PATCH 021/132] Fix Websocket --- src/app.js | 9 ++++++--- src/routes/proxy.js | 12 ++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/app.js b/src/app.js index d608592..9b95b1b 100644 --- a/src/app.js +++ b/src/app.js @@ -4,6 +4,7 @@ import bodyParser from 'body-parser'; import config from './config'; import Router from './routes'; +import Proxy from './routes/proxy'; import { internalUrl } from './utils'; import debug from 'debug'; @@ -25,7 +26,7 @@ app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: true })); -app.use((err, req, res, next) => { +app.use((err, _, res) => { if (err instanceof SyntaxError && err.status >= 400 && err.status < 500 && err.message.indexOf('JSON')) res.status(400).send({ error: { code: 'INVALID_BODY', message: 'Syntax error in the JSON body' } }); }); @@ -36,6 +37,8 @@ D('Initializing API routes...'); // Routes Router(app); -// Bind and start -app.listen(config.server.port) +// Create HTTP server, forward websocket and start +app.listen(config.server.port).on('upgrade', (req, res) => { + Proxy.ws(req, res); +}); D('Launched on ' + internalUrl()); diff --git a/src/routes/proxy.js b/src/routes/proxy.js index bd097d3..7503759 100644 --- a/src/routes/proxy.js +++ b/src/routes/proxy.js @@ -15,4 +15,16 @@ RoutesProxy.plex = (req, res) => { return (proxy.web(req, res)); }; +RoutesProxy.ws = (req, res) => { + const proxy = httpProxy.createProxyServer({ + target: { + host: config.plex.host, + port: config.plex.port + } + }).on('error', () => { + res.status(400).send({ error: { code: 'PROXY_TIMEOUT', message: 'Plex not respond in time, proxy request fails' } }); + }); + return (proxy.ws(req, res)); +}; + export default RoutesProxy; From 05003d291f93bccad107135d045306ec8b81e291 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 12 Nov 2018 21:31:23 +0100 Subject: [PATCH 022/132] Fix Websocket --- src/app.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/app.js b/src/app.js index 9b95b1b..35d3170 100644 --- a/src/app.js +++ b/src/app.js @@ -37,8 +37,13 @@ D('Initializing API routes...'); // Routes Router(app); -// Create HTTP server, forward websocket and start -app.listen(config.server.port).on('upgrade', (req, res) => { +// Create HTTP server +const httpServer = app.listen(config.server.port); + +// Forward websockets +httpServer.on('upgrade', (req, res) => { Proxy.ws(req, res); }); + +// Debug D('Launched on ' + internalUrl()); From 67c0786a870ea2186aa6a5f077f1e6316aa82fec Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 12 Nov 2018 21:32:53 +0100 Subject: [PATCH 023/132] Disable Websocket --- src/app.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app.js b/src/app.js index 35d3170..4e904fb 100644 --- a/src/app.js +++ b/src/app.js @@ -41,9 +41,9 @@ Router(app); const httpServer = app.listen(config.server.port); // Forward websockets -httpServer.on('upgrade', (req, res) => { +/*httpServer.on('upgrade', (req, res) => { Proxy.ws(req, res); -}); +});*/ // Debug D('Launched on ' + internalUrl()); From a9ea810c3a18c4156c425315cfe3c5fc1bf7bc3e Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 12 Nov 2018 21:37:50 +0100 Subject: [PATCH 024/132] Trying to fix LB --- src/app.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/app.js b/src/app.js index 4e904fb..39ebf36 100644 --- a/src/app.js +++ b/src/app.js @@ -26,9 +26,10 @@ app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: true })); -app.use((err, _, res) => { +app.use((err, _, res, next) => { if (err instanceof SyntaxError && err.status >= 400 && err.status < 500 && err.message.indexOf('JSON')) - res.status(400).send({ error: { code: 'INVALID_BODY', message: 'Syntax error in the JSON body' } }); + return (res.status(400).send({ error: { code: 'INVALID_BODY', message: 'Syntax error in the JSON body' } })); + next(); }); // Init routes @@ -41,9 +42,9 @@ Router(app); const httpServer = app.listen(config.server.port); // Forward websockets -/*httpServer.on('upgrade', (req, res) => { +httpServer.on('upgrade', (req, res) => { Proxy.ws(req, res); -});*/ +}); // Debug D('Launched on ' + internalUrl()); From 6a16f18eb6591c9a54c33e1560f59405d11e7d32 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 12 Nov 2018 22:09:26 +0100 Subject: [PATCH 025/132] Update README --- README.md | 44 ++++++++++++++++++++++++-------------------- src/config.js | 2 +- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 49dedbd..5611198 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer * Plex Media Server * NodeJS +* RedisCache (Optionnal) ## Setup @@ -19,28 +20,32 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer * Clone the repository * Install with `npm install` -* Edit the configuration - -| Variable | Description | -| ----------------- | ------------------------------------------------------------ | -| cluster | Array of UnicornTranscoder Servers in the cluster | -| preprod | If enabled, will filter on IP and send to the configured UnicornTranscoder, it allows to have a UnicornTranscoder server for developement without impacting users on the Plex Media Server | -| plex | Configuration of the Plex Media server | -| >host | Address to join the Plex Media Server | -| >port | Port of the Plex Media server | -| >sessions | Where Plex store sessions (to grab external subtitles) | -| >database | Plex Media Server Database | -| loadBalancer.port | Port UnicornLoadBalancer will listen | -| alerts.discord | Discord Webhook to notify unavailable UnicornTranscoder | - -* Configure Plex Media Server access Address - * In Settings -> Server -> Network - * Set `Custom server access URLs` to the address to access the UnicornLoadBalancer +* Setup some environnement variables to configure the *UnicornLoadBlancer* + +| Name | Description | Type | Default | +| ----------------- | ------------------------------------------------------------ | ------| ------- | +| **SERVER_HOST** | Host to access to the *UnicornLoadBalancer* | `string` | `127.0.0.1` | +| **SERVER_PORT** | Port used by the *UnicornLoadBalancer* | `int` | `3001` | +| **SERVER_SSL** | If HTTPS is enabled or not on the *UnicornLoadBalancer* | `bool` | `false` | +| **PLEX_HOST** | Host to access to Plex | `string` | `127.0.0.1` | +| **PLEX_PORT** | Port used by Plex | `int` | `32400` | +| **PLEX_PATH_USR** | The Plex's path | `string` | `/usr/lib/plexmediaserver/` | +| **PLEX_PATH_SESSIONS** | The path where Plex store sessions (to grab external subtitles) | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Cache/Transcode/Sessions` | +| **PLEX_PATH_DATABASE** | The path of the Plex database | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db` | +| **REDIS_HOST** | The host of the redis database | `string|undefined` | `undefined` | +| **REDIS_PORT** | Port used by Redis | `int` | `6379` | +| **REDIS_PASSWORD** | The password of the redis database | `string` | ` ` | +| **REDIS_DB** | The index of the redis database | `int` | `0` | +| **SCORES_TIMEOUT** | Seconds to consider a not-pinged server as unavailable | `int` | `10` | + +* Configure Plex Media Server access address + * In Settings -> Server -> Network + * Set `Custom server access URLs` to the address to access the UnicornLoadBalancer * Run with npm start -## 2. Notes +### 2. Notes -All the requests to this Plex Media Server should pass by the UnicornLoadBalancer, if someone reach the server directly without passing through UnicornLoadBalancer he will not be able to start a stream, since FFMPEG binary has been replaced. It is recomended to setup a nginx reverse proxy in front to setup a SSL certificate and to have an iptable to direct access to the users on port 32400. +All the requests to this Plex Media Server should pass through the *UnicornLoadBalancer*, if someone reach the server directly he will not be able to start a stream, since FFMPEG binary has been replaced. It is recomended to setup a nginx as reverse proxy in front to setup a SSL certificate and to have an iptable to direct access to the users on port 32400. ``` #Example iptable @@ -49,4 +54,3 @@ iptables -A INPUT -p tcp --dport 32400 -i eth0 -s -j ACCEPT #Deny all other incoming connections iptables -A INPUT -p tcp --dport 32400 -i eth0 -j DROP ``` - diff --git a/src/config.js b/src/config.js index 43aa569..cc445c8 100644 --- a/src/config.js +++ b/src/config.js @@ -6,7 +6,7 @@ export default { version: '2.0.0', server: { port: env.int('SERVER_PORT', 3001), - host: env.string('SERVER_DOMAIN', '127.0.0.1'), + host: env.string('SERVER_HOST', '127.0.0.1'), ssl: env.bool('SERVER_SSL', false) }, plex: { From a88a4b0955d94ee731cd13ad67a349fe373e76ce Mon Sep 17 00:00:00 2001 From: Maxime BACONNAIS Date: Mon, 12 Nov 2018 22:10:45 +0100 Subject: [PATCH 026/132] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5611198..24cfb3f 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer | **PLEX_PATH_USR** | The Plex's path | `string` | `/usr/lib/plexmediaserver/` | | **PLEX_PATH_SESSIONS** | The path where Plex store sessions (to grab external subtitles) | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Cache/Transcode/Sessions` | | **PLEX_PATH_DATABASE** | The path of the Plex database | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db` | -| **REDIS_HOST** | The host of the redis database | `string|undefined` | `undefined` | +| **REDIS_HOST** | The host of the redis database | `string` `undefined` | `undefined` | | **REDIS_PORT** | Port used by Redis | `int` | `6379` | | **REDIS_PASSWORD** | The password of the redis database | `string` | ` ` | | **REDIS_DB** | The index of the redis database | `int` | `0` | From 6d9af8fb3fe3d3b239a87cf117165c218f874462 Mon Sep 17 00:00:00 2001 From: Maxime BACONNAIS Date: Mon, 12 Nov 2018 22:12:02 +0100 Subject: [PATCH 027/132] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 24cfb3f..7743bc5 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer ### 2. Notes -All the requests to this Plex Media Server should pass through the *UnicornLoadBalancer*, if someone reach the server directly he will not be able to start a stream, since FFMPEG binary has been replaced. It is recomended to setup a nginx as reverse proxy in front to setup a SSL certificate and to have an iptable to direct access to the users on port 32400. +All the requests to this Plex Media Server should pass through the *UnicornLoadBalancer*, if someone reach the server directly he will not be able to start a stream, since FFMPEG binary has been replaced. It is recomended to setup a nginx as reverse proxy in front to setup a SSL certificate and to have an iptable to direct access to the users on port **32400**. ``` #Example iptable From 93bc6fd9cfcb214848f1a51e0a1a84fa5a18ddac Mon Sep 17 00:00:00 2001 From: DROUARD Benjamin Date: Tue, 13 Nov 2018 18:20:21 +0100 Subject: [PATCH 028/132] Fix Subtitles redirect route --- src/routes/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/index.js b/src/routes/index.js index db7d58c..8b2b2f0 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -26,7 +26,7 @@ export default (app) => { // Long polling support app.get('/:formatType/:/transcode/universal/start', RoutesTranscode.redirect); - app.get(':formatType/:/transcode/universal/subtitles', RoutesTranscode.redirect); + app.get('/:formatType/:/transcode/universal/subtitles', RoutesTranscode.redirect); // M3U8 support app.get('/:formatType/:/transcode/universal/start.m3u8', RoutesTranscode.redirect); @@ -45,4 +45,4 @@ export default (app) => { // Forward other to Plex app.all('*', RoutesProxy.plex); -}; \ No newline at end of file +}; From 1d77b0551191b271d5647aa71a3b2e8a37f844c5 Mon Sep 17 00:00:00 2001 From: DROUARD Benjamin Date: Tue, 13 Nov 2018 18:46:24 +0100 Subject: [PATCH 029/132] Fix unicorn id url patching --- src/routes/transcode.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index 73554cc..d157a67 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -19,7 +19,7 @@ RoutesTranscode.redirect = (req, res) => { const redirectRequest = (server) => { if (server) { res.writeHead(302, { - 'Location': (server + req.url + '&unicorn=' + session.unicorn) + 'Location': (server + req.url + (req.url.indexOf('?') === -1 ? '?' : '&') + 'unicorn=' + session.unicorn) }); res.end(); D('Send 302 for ' + session.session + ' to ' + server); @@ -118,4 +118,4 @@ RoutesTranscode.stop = (req, res) => { }); }; -export default RoutesTranscode; \ No newline at end of file +export default RoutesTranscode; From 77784490bd96977519087fe40b2a5b44ac0879e4 Mon Sep 17 00:00:00 2001 From: DROUARD Benjamin Date: Tue, 13 Nov 2018 18:47:42 +0100 Subject: [PATCH 030/132] Fix M3U8 routes --- src/routes/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/routes/index.js b/src/routes/index.js index 8b2b2f0..5a3b81a 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -29,7 +29,6 @@ export default (app) => { app.get('/:formatType/:/transcode/universal/subtitles', RoutesTranscode.redirect); // M3U8 support - app.get('/:formatType/:/transcode/universal/start.m3u8', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/base/index.m3u8', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/base-x-mc/index.m3u8', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/:fileType/:partId.ts', RoutesTranscode.redirect); From 470acf9fd017545552f4d0cddb80421fddf626dc Mon Sep 17 00:00:00 2001 From: Benjamin DROUARD Date: Wed, 14 Nov 2018 15:59:42 +0100 Subject: [PATCH 031/132] Delete in session store --- src/store/local.js | 14 +++++++++++++- src/store/redis.js | 17 ++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/store/local.js b/src/store/local.js index 235135b..448b7f5 100644 --- a/src/store/local.js +++ b/src/store/local.js @@ -38,7 +38,7 @@ class LocalSessionStore { * Store a value in the store and trigger the pending gets * @param sessionId * @param value - * @returns {Promise} + * @returns {Promise} */ set(sessionId, value) { return new Promise((resolve) => { @@ -47,6 +47,18 @@ class LocalSessionStore { resolve('OK'); }) } + + /** + * Delete a session from the store + * @param sessionId + * @returns {Promise} + */ + delete(sessionId) { + return new Promise((resolve) => { + delete this.sessionStore[sessionId]; + resolve('OK'); + }) + } } export default LocalSessionStore; \ No newline at end of file diff --git a/src/store/redis.js b/src/store/redis.js index 6d567b9..6b616f0 100644 --- a/src/store/redis.js +++ b/src/store/redis.js @@ -58,7 +58,7 @@ class RedisSessionStore { * Store a value in the store and trigger the pending gets * @param sessionId * @param value - * @returns {Promise} + * @returns {Promise} */ set(sessionId, value) { return new Promise((resolve, reject) => { @@ -69,6 +69,21 @@ class RedisSessionStore { }) }) } + + /** + * Delete a session from the store + * @param sessionId + * @returns {Promise} + */ + delete(sessionId) { + return new Promise((resolve, reject) => { + this.redis.del(sessionId, (err) => { + if (err) + return reject(err); + resolve('OK') + }) + }) + } } export default RedisSessionStore; \ No newline at end of file From 2d776d3a84182ddced2377c4a7471f03e3f65e58 Mon Sep 17 00:00:00 2001 From: Benjamin DROUARD Date: Wed, 14 Nov 2018 16:13:47 +0100 Subject: [PATCH 032/132] Delete session on start.m3u8 for android --- src/core/sessions.js | 4 ++++ src/routes/index.js | 1 + src/routes/transcode.js | 12 ++++++++++++ 3 files changed, 17 insertions(+) diff --git a/src/core/sessions.js b/src/core/sessions.js index a1151e7..c555b05 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -171,5 +171,9 @@ SessionsManager.storeFFmpegParameters = (args, env) => { return (parsed); }; +SessionsManager.cleanSession = (sessionId) => { + return SessionStore.delete(sessionId) +}; + // Export our SessionsManager export default SessionsManager; diff --git a/src/routes/index.js b/src/routes/index.js index 5a3b81a..b9826c9 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -29,6 +29,7 @@ export default (app) => { app.get('/:formatType/:/transcode/universal/subtitles', RoutesTranscode.redirect); // M3U8 support + app.get('/:formatType/:/transcode/universal/start.m3u8', RoutesTranscode.cleanSession, RoutesProxy.plex); app.get('/:formatType/:/transcode/universal/session/:sessionId/base/index.m3u8', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/base-x-mc/index.m3u8', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/:fileType/:partId.ts', RoutesTranscode.redirect); diff --git a/src/routes/transcode.js b/src/routes/transcode.js index d157a67..1825a57 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -118,4 +118,16 @@ RoutesTranscode.stop = (req, res) => { }); }; +RoutesTranscode.cleanSession = (req, res, next) => { + if (typeof req.query.session !== 'undefined') { + SessionsManager.cleanSession(req.query.session) + .then(() => { + next() + }) + .catch(() => { + res.status(500).send('Internal Server Error') + }) + } +}; + export default RoutesTranscode; From 416e107539a3df97999bb8d9444697970a0ff727 Mon Sep 17 00:00:00 2001 From: Benjamin DROUARD Date: Wed, 14 Nov 2018 16:19:44 +0100 Subject: [PATCH 033/132] More debug --- src/routes/transcode.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index 1825a57..ce46b05 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -120,6 +120,7 @@ RoutesTranscode.stop = (req, res) => { RoutesTranscode.cleanSession = (req, res, next) => { if (typeof req.query.session !== 'undefined') { + D('Clean ' + req.query.session + 'from session store'); SessionsManager.cleanSession(req.query.session) .then(() => { next() From 19b16b331ec8066c7e528685f356605e42d59f14 Mon Sep 17 00:00:00 2001 From: Benjamin DROUARD Date: Thu, 15 Nov 2018 16:41:53 +0100 Subject: [PATCH 034/132] Fix path --- src/routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/api.js b/src/routes/api.js index daf118e..4635078 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -29,7 +29,7 @@ RoutesAPI.ffmpeg = (req, res) => { // Resolve path from file id RoutesAPI.path = (req, res) => { try { - const db = new sqlite3.verbose().Database(config.plex.database); + const db = new (sqlite3.verbose().Database)(config.plex.path.database); db.get("SELECT * FROM media_parts WHERE id=? LIMIT 0, 1", req.params.id, (err, row) => { if (row && row.file) res.send(JSON.stringify(row)); From 24ef1827485deea6bee27cafdd38179dc9fda0e4 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 15 Nov 2018 19:55:41 +0100 Subject: [PATCH 035/132] Debug --- src/core/sessions.js | 12 ++++++++---- src/routes/api.js | 4 ++++ src/routes/index.js | 1 + 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index c555b05..9ae793c 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -11,7 +11,7 @@ const D = debug('UnicornLoadBalancer:SessionsManager'); let SessionsManager = {}; let sessions = [ - { + /*{ unicorn: '_UNiC0RN', session: '', sessionFull: '', @@ -21,9 +21,13 @@ let sessions = [ env: [], serverUrl: '', pingUrl: '', - } + }*/ ]; +SessionsManager.list = () => { + return (sessions); +}; + // Parse request to extract session information SessionsManager.parseSessionFromRequest = (req) => { const unicorn = (typeof (req.query.unicorn) !== 'undefined') ? { unicorn: req.query.unicorn } : false; @@ -40,7 +44,7 @@ SessionsManager.parseSessionFromRequest = (req) => { // Get a session from its values SessionsManager.getSessionFromRequest = (search) => { - let keys = Object.keys(search).filter(e => (['args', 'env', 'pingUrl'].indexOf(e) === -1)); + let keys = Object.keys(search).filter(e => (['args', 'env', 'pingUrl', 'serverUrl'].indexOf(e) === -1)); const filtered = sessions.filter(e => { for (let i = 0; i < keys.length; i++) { if (e[keys[i]] === search[keys[i]] && e[keys[i]]) @@ -55,7 +59,7 @@ SessionsManager.getSessionFromRequest = (search) => { // Get a session position from its values SessionsManager.getIdFromRequest = (search) => { - let keys = Object.keys(search).filter(e => (['args', 'env', 'pingUrl'].indexOf(e) === -1)); + let keys = Object.keys(search).filter(e => (['args', 'env', 'pingUrl', 'serverUrl'].indexOf(e) === -1)); for (let idx = 0; idx < sessions.length; idx++) { for (let i = 0; i < keys.length; i++) { if (sessions[idx][keys[i]] === search[keys[i]] && sessions[idx][keys[i]]) diff --git a/src/routes/api.js b/src/routes/api.js index 4635078..05040a1 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -13,6 +13,10 @@ RoutesAPI.stats = (req, res) => { res.send(ServersManager.list()); }; +RoutesAPI.sessions = (req, res) => { + res.send(SessionsManager.list()); +}; + // Save the stats of a server RoutesAPI.update = (req, res) => { res.send(ServersManager.update(req.body)); diff --git a/src/routes/index.js b/src/routes/index.js index b9826c9..df3128b 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -12,6 +12,7 @@ export default (app) => { // UnicornLoadBalancer API app.use('/api/sessions', express.static(config.plex.path.sessions)); + app.use('/api/debug', RoutesAPI.sessions); app.get('/api/stats', RoutesAPI.stats); app.post('/api/ffmpeg', RoutesAPI.ffmpeg); app.get('/api/path/:id', RoutesAPI.path); From 5028cd53759198b306507a9688f2597d521d37f3 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 15 Nov 2018 20:00:44 +0100 Subject: [PATCH 036/132] Fix --- src/core/sessions.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 9ae793c..88a7e65 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -32,8 +32,8 @@ SessionsManager.list = () => { SessionsManager.parseSessionFromRequest = (req) => { const unicorn = (typeof (req.query.unicorn) !== 'undefined') ? { unicorn: req.query.unicorn } : false; const session = (typeof (req.params.sessionId) !== 'undefined') ? { sessionId: req.params.sessionId } : ((typeof (req.query.session) !== 'undefined') ? { session: req.query.session } : false); - const sessionIdentifier = (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined') ? { "X-Plex-Session-Identifier": req.query['X-Plex-Session-Identifier'] } : false; - const clientIdentifier = (typeof (req.query['X-Plex-Client-Identifier']) !== 'undefined') ? { "X-Plex-Client-Identifier": req.query['X-Plex-Client-Identifier'] } : false; + const sessionIdentifier = (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined') ? { sessionIdentifier: req.query['X-Plex-Session-Identifier'] } : false; + const clientIdentifier = (typeof (req.query['X-Plex-Client-Identifier']) !== 'undefined') ? { clientIdentifier: req.query['X-Plex-Client-Identifier'] } : false; return { ...unicorn, ...session, @@ -59,7 +59,7 @@ SessionsManager.getSessionFromRequest = (search) => { // Get a session position from its values SessionsManager.getIdFromRequest = (search) => { - let keys = Object.keys(search).filter(e => (['args', 'env', 'pingUrl', 'serverUrl'].indexOf(e) === -1)); + let keys = Object.keys(search).filter(e => (['args', 'env', 'pingUrl', 'serverUrl', 'clientIdentifier'].indexOf(e) === -1)); for (let idx = 0; idx < sessions.length; idx++) { for (let i = 0; i < keys.length; i++) { if (sessions[idx][keys[i]] === search[keys[i]] && sessions[idx][keys[i]]) From 2622d949a76e2e80824ab8a2591d85edde9ece08 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 15 Nov 2018 20:11:54 +0100 Subject: [PATCH 037/132] Sessions fix --- src/core/sessions.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 88a7e65..8eef64b 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -20,7 +20,6 @@ let sessions = [ args: [], env: [], serverUrl: '', - pingUrl: '', }*/ ]; @@ -44,7 +43,7 @@ SessionsManager.parseSessionFromRequest = (req) => { // Get a session from its values SessionsManager.getSessionFromRequest = (search) => { - let keys = Object.keys(search).filter(e => (['args', 'env', 'pingUrl', 'serverUrl'].indexOf(e) === -1)); + let keys = Object.keys(search).filter(e => (['args', 'env', 'serverUrl', 'clientIdentifier'].indexOf(e) === -1)); const filtered = sessions.filter(e => { for (let i = 0; i < keys.length; i++) { if (e[keys[i]] === search[keys[i]] && e[keys[i]]) @@ -59,7 +58,7 @@ SessionsManager.getSessionFromRequest = (search) => { // Get a session position from its values SessionsManager.getIdFromRequest = (search) => { - let keys = Object.keys(search).filter(e => (['args', 'env', 'pingUrl', 'serverUrl', 'clientIdentifier'].indexOf(e) === -1)); + let keys = Object.keys(search).filter(e => (['args', 'env', 'serverUrl', 'clientIdentifier'].indexOf(e) === -1)); for (let idx = 0; idx < sessions.length; idx++) { for (let i = 0; i < keys.length; i++) { if (sessions[idx][keys[i]] === search[keys[i]] && sessions[idx][keys[i]]) @@ -90,7 +89,6 @@ SessionsManager.updateSession = (args) => { args: [], env: [], serverUrl: '', - pingUrl: '', ...args }); return (true); From 5f4f5ef1292020dbe02e3827088617d725e853ae Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 15 Nov 2018 20:23:17 +0100 Subject: [PATCH 038/132] Bugfix --- src/core/sessions.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/sessions.js b/src/core/sessions.js index 8eef64b..65d4f03 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -79,6 +79,9 @@ SessionsManager.updateSession = (args) => { const search = SessionsManager.getSessionFromRequest(args); const idx = SessionsManager.getIdFromRequest(args); + if (Object.keys(args).length === 1) + return 'Fail'; + if (!search) { sessions.push({ unicorn: uniqid(), From 736a49342165c835347c4a9065152f168f6d3270 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 15 Nov 2018 20:36:18 +0100 Subject: [PATCH 039/132] Fixes sessions --- src/core/sessions.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 65d4f03..1c52fd4 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -79,8 +79,8 @@ SessionsManager.updateSession = (args) => { const search = SessionsManager.getSessionFromRequest(args); const idx = SessionsManager.getIdFromRequest(args); - if (Object.keys(args).length === 1) - return 'Fail'; + if (Object.keys(args).length === 0 || (Object.keys(args).length === 1 && Object.keys(args).indexOf('clientIdentifier') !== -1)) + return (false); if (!search) { sessions.push({ From 025f6bc17b115db55e6c6088e5384dc192205ca0 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 16 Nov 2018 13:23:19 +0100 Subject: [PATCH 040/132] Fix Android and comment tricky code --- src/core/sessions.js | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 1c52fd4..984a20e 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -43,28 +43,56 @@ SessionsManager.parseSessionFromRequest = (req) => { // Get a session from its values SessionsManager.getSessionFromRequest = (search) => { + + // List of keys could be used to identify a session let keys = Object.keys(search).filter(e => (['args', 'env', 'serverUrl', 'clientIdentifier'].indexOf(e) === -1)); - const filtered = sessions.filter(e => { + + // Reverse sessions to start by the end + const rsessions = sessions.slice().reverse(); + + // Filter sessions + const filtered = rsessions.filter(e => { for (let i = 0; i < keys.length; i++) { if (e[keys[i]] === search[keys[i]] && e[keys[i]]) return (true); } return (false); }); - if (filtered.length === 0) - return (false); - return (filtered[0]); + + // Found, return the session + if (filtered.length > 0) + return (filtered[0]); + + // Android case, no session, only a sessionIdentifier + if (!search.session && search.sessionIdentifier) + return (SessionsManager.getSessionFromRequest({ ...search, session: search.sessionIdentifier })); + + // Not found + return (false); }; // Get a session position from its values SessionsManager.getIdFromRequest = (search) => { + + // List of keys could be used to identify a session let keys = Object.keys(search).filter(e => (['args', 'env', 'serverUrl', 'clientIdentifier'].indexOf(e) === -1)); - for (let idx = 0; idx < sessions.length; idx++) { + + // Reverse session to start by the end + const rsessions = sessions.slice().reverse(); + + // Filter sessions + for (let idx = 0; idx < rsessions.length; idx++) { for (let i = 0; i < keys.length; i++) { - if (sessions[idx][keys[i]] === search[keys[i]] && sessions[idx][keys[i]]) + if (rsessions[idx][keys[i]] === search[keys[i]] && rsessions[idx][keys[i]]) return (idx); } } + + // Android case, no session, only a sessionIdentifier + if (!search.session && search.sessionIdentifier) + return (SessionsManager.getIdFromRequest({ ...search, session: search.sessionIdentifier })); + + // Not be found return (false); }; From 7d02b45d39985465e3aec5ebd1770c7f15219a68 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 16 Nov 2018 13:45:52 +0100 Subject: [PATCH 041/132] Fix empty session detection --- src/core/sessions.js | 3 ++- src/routes/transcode.js | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 984a20e..f597ca2 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -107,7 +107,8 @@ SessionsManager.updateSession = (args) => { const search = SessionsManager.getSessionFromRequest(args); const idx = SessionsManager.getIdFromRequest(args); - if (Object.keys(args).length === 0 || (Object.keys(args).length === 1 && Object.keys(args).indexOf('clientIdentifier') !== -1)) + // Avoid to create empty session objects (Download case by example) + if (Object.keys(args).length === 0 || (!args.session && !args.sessionFull && !args.sessionIdentifier && !args.clientIdentifier)) return (false); if (!search) { diff --git a/src/routes/transcode.js b/src/routes/transcode.js index ce46b05..1aa8d0d 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -44,7 +44,6 @@ RoutesTranscode.ping = (req, res) => { const search = SessionsManager.parseSessionFromRequest(req); const session = SessionsManager.getSessionFromRequest(search); - req.url += '&unicorn=' + session.unicorn; Proxy.plex(req, res); const pingRequest = (server) => { @@ -67,7 +66,6 @@ RoutesTranscode.timeline = (req, res) => { const search = SessionsManager.parseSessionFromRequest(req); const session = SessionsManager.getSessionFromRequest(search); - req.url += '&unicorn=' + session.unicorn; Proxy.plex(req, res); const pingRequest = (server) => { @@ -101,7 +99,6 @@ RoutesTranscode.stop = (req, res) => { const search = SessionsManager.parseSessionFromRequest(req); const session = SessionsManager.getSessionFromRequest(search); - req.url += '&unicorn=' + session.unicorn; Proxy.plex(req, res); const stopRequest = (server) => { From 5ebab073b98e3bf303799878cd455047e1a1b556 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 16 Nov 2018 13:52:20 +0100 Subject: [PATCH 042/132] Android download fix --- src/core/sessions.js | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index f597ca2..5057d61 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -11,16 +11,6 @@ const D = debug('UnicornLoadBalancer:SessionsManager'); let SessionsManager = {}; let sessions = [ - /*{ - unicorn: '_UNiC0RN', - session: '', - sessionFull: '', - sessionIdentifier: '', - clientIdentifier: '', - args: [], - env: [], - serverUrl: '', - }*/ ]; SessionsManager.list = () => { @@ -67,6 +57,10 @@ SessionsManager.getSessionFromRequest = (search) => { if (!search.session && search.sessionIdentifier) return (SessionsManager.getSessionFromRequest({ ...search, session: search.sessionIdentifier })); + // Ok, Android really sucks, other case, no session, only a clientIdentifier + if (!search.session && search.clientIdentifier) + return (SessionsManager.getSessionFromRequest({ ...search, session: search.clientIdentifier })); + // Not found return (false); }; @@ -92,6 +86,10 @@ SessionsManager.getIdFromRequest = (search) => { if (!search.session && search.sessionIdentifier) return (SessionsManager.getIdFromRequest({ ...search, session: search.sessionIdentifier })); + // Ok, Android really sucks, other case, no session, only a clientIdentifier + if (!search.session && search.sessionIdentifier) + return (SessionsManager.getIdFromRequest({ ...search, session: search.clientIdentifier })); + // Not be found return (false); }; From e27fbacdda1e54220d00a85386586914b495a491 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 16 Nov 2018 13:52:51 +0100 Subject: [PATCH 043/132] Fixes --- src/core/sessions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 5057d61..e7aa4f7 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -87,7 +87,7 @@ SessionsManager.getIdFromRequest = (search) => { return (SessionsManager.getIdFromRequest({ ...search, session: search.sessionIdentifier })); // Ok, Android really sucks, other case, no session, only a clientIdentifier - if (!search.session && search.sessionIdentifier) + if (!search.session && search.clientIdentifier) return (SessionsManager.getIdFromRequest({ ...search, session: search.clientIdentifier })); // Not be found From 9111c0f7817310994d2012d5d1df80ea7ce66105 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 16 Nov 2018 14:03:55 +0100 Subject: [PATCH 044/132] Fixes --- src/core/sessions.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index e7aa4f7..11323bc 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -34,8 +34,8 @@ SessionsManager.parseSessionFromRequest = (req) => { // Get a session from its values SessionsManager.getSessionFromRequest = (search) => { - // List of keys could be used to identify a session - let keys = Object.keys(search).filter(e => (['args', 'env', 'serverUrl', 'clientIdentifier'].indexOf(e) === -1)); + // List of keys could be used to identify a session, we add clientIdentifier at the end + const keys = Object.keys(search).filter(e => (['args', 'env', 'serverUrl', 'clientIdentifier'].indexOf(e) === -1)).push('clientIdentifier'); // Reverse sessions to start by the end const rsessions = sessions.slice().reverse(); @@ -68,8 +68,8 @@ SessionsManager.getSessionFromRequest = (search) => { // Get a session position from its values SessionsManager.getIdFromRequest = (search) => { - // List of keys could be used to identify a session - let keys = Object.keys(search).filter(e => (['args', 'env', 'serverUrl', 'clientIdentifier'].indexOf(e) === -1)); + // List of keys could be used to identify a session, we add clientIdentifier at the end + const keys = Object.keys(search).filter(e => (['args', 'env', 'serverUrl', 'clientIdentifier'].indexOf(e) === -1)).push('clientIdentifier'); // Reverse session to start by the end const rsessions = sessions.slice().reverse(); From a10aa7289173b4183bbebff07aae271224f888b2 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 16 Nov 2018 14:29:35 +0100 Subject: [PATCH 045/132] Fixes --- src/core/sessions.js | 41 ++++++++--------------------------------- 1 file changed, 8 insertions(+), 33 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 11323bc..96e9d57 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -33,43 +33,17 @@ SessionsManager.parseSessionFromRequest = (req) => { // Get a session from its values SessionsManager.getSessionFromRequest = (search) => { - - // List of keys could be used to identify a session, we add clientIdentifier at the end - const keys = Object.keys(search).filter(e => (['args', 'env', 'serverUrl', 'clientIdentifier'].indexOf(e) === -1)).push('clientIdentifier'); - - // Reverse sessions to start by the end - const rsessions = sessions.slice().reverse(); - - // Filter sessions - const filtered = rsessions.filter(e => { - for (let i = 0; i < keys.length; i++) { - if (e[keys[i]] === search[keys[i]] && e[keys[i]]) - return (true); - } + const sessionIndex = SessionsManager.getIdFromRequest(search); + if (sessionindex === false) return (false); - }); - - // Found, return the session - if (filtered.length > 0) - return (filtered[0]); - - // Android case, no session, only a sessionIdentifier - if (!search.session && search.sessionIdentifier) - return (SessionsManager.getSessionFromRequest({ ...search, session: search.sessionIdentifier })); - - // Ok, Android really sucks, other case, no session, only a clientIdentifier - if (!search.session && search.clientIdentifier) - return (SessionsManager.getSessionFromRequest({ ...search, session: search.clientIdentifier })); - - // Not found - return (false); + return (sessions[sessionIndex]); }; // Get a session position from its values SessionsManager.getIdFromRequest = (search) => { - // List of keys could be used to identify a session, we add clientIdentifier at the end - const keys = Object.keys(search).filter(e => (['args', 'env', 'serverUrl', 'clientIdentifier'].indexOf(e) === -1)).push('clientIdentifier'); + // List of keys could be used to identify a session + const keys = ['unicorn', 'session', 'sessionFull', 'sessionIdentifier', 'clientIdentifier']; // Reverse session to start by the end const rsessions = sessions.slice().reverse(); @@ -77,7 +51,7 @@ SessionsManager.getIdFromRequest = (search) => { // Filter sessions for (let idx = 0; idx < rsessions.length; idx++) { for (let i = 0; i < keys.length; i++) { - if (rsessions[idx][keys[i]] === search[keys[i]] && rsessions[idx][keys[i]]) + if (rsessions[idx][keys[i]] && search[keys[i]] && rsessions[idx][keys[i]] === search[keys[i]]) return (idx); } } @@ -106,7 +80,7 @@ SessionsManager.updateSession = (args) => { const idx = SessionsManager.getIdFromRequest(args); // Avoid to create empty session objects (Download case by example) - if (Object.keys(args).length === 0 || (!args.session && !args.sessionFull && !args.sessionIdentifier && !args.clientIdentifier)) + if (!args.session && !args.sessionFull && !args.sessionIdentifier && !args.clientIdentifier) return (false); if (!search) { @@ -195,6 +169,7 @@ SessionsManager.storeFFmpegParameters = (args, env) => { session: parsed.session, sessionFull: parsed.sessionFull }); + console.log('lol', session.session, session); SessionStore.set(session.session, session).then(() => { }).catch((err) => { From 4ba2b5b86936eaa5540beb10ccbb1df0032ac660 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 16 Nov 2018 14:31:50 +0100 Subject: [PATCH 046/132] Fixes --- src/core/sessions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 96e9d57..f206f11 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -34,7 +34,7 @@ SessionsManager.parseSessionFromRequest = (req) => { // Get a session from its values SessionsManager.getSessionFromRequest = (search) => { const sessionIndex = SessionsManager.getIdFromRequest(search); - if (sessionindex === false) + if (sessionIndex === false) return (false); return (sessions[sessionIndex]); }; From 72e287d03caf97e46657578058999c297dd13e10 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 16 Nov 2018 14:37:36 +0100 Subject: [PATCH 047/132] Fixes Plex proxy --- src/routes/proxy.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/routes/proxy.js b/src/routes/proxy.js index 7503759..cf24c7a 100644 --- a/src/routes/proxy.js +++ b/src/routes/proxy.js @@ -21,7 +21,12 @@ RoutesProxy.ws = (req, res) => { host: config.plex.host, port: config.plex.port } - }).on('error', () => { + }).on('error', (err) => { + // On some Plex request from FFmpeg, Plex don't create a valid request + if (err.code === 'HPE_UNEXPECTED_CONTENT_LENGTH') + res.status(200).send(); + + // Other error res.status(400).send({ error: { code: 'PROXY_TIMEOUT', message: 'Plex not respond in time, proxy request fails' } }); }); return (proxy.ws(req, res)); From 831c932135f30ebcf4db86ac7816daf791193c9e Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 16 Nov 2018 14:44:35 +0100 Subject: [PATCH 048/132] Fixes Plex proxy --- src/routes/proxy.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/routes/proxy.js b/src/routes/proxy.js index cf24c7a..a2d4090 100644 --- a/src/routes/proxy.js +++ b/src/routes/proxy.js @@ -9,7 +9,12 @@ RoutesProxy.plex = (req, res) => { host: config.plex.host, port: config.plex.port } - }).on('error', () => { + }).on('error', (err) => { + // On some Plex request from FFmpeg, Plex don't create a valid request + if (err.code === 'HPE_UNEXPECTED_CONTENT_LENGTH') + res.status(200).send(); + + // Other error res.status(400).send({ error: { code: 'PROXY_TIMEOUT', message: 'Plex not respond in time, proxy request fails' } }); }); return (proxy.web(req, res)); @@ -21,12 +26,7 @@ RoutesProxy.ws = (req, res) => { host: config.plex.host, port: config.plex.port } - }).on('error', (err) => { - // On some Plex request from FFmpeg, Plex don't create a valid request - if (err.code === 'HPE_UNEXPECTED_CONTENT_LENGTH') - res.status(200).send(); - - // Other error + }).on('error', () => { res.status(400).send({ error: { code: 'PROXY_TIMEOUT', message: 'Plex not respond in time, proxy request fails' } }); }); return (proxy.ws(req, res)); From b52c2d6122e2ed2413fe4b47eda598b0f32c9e4a Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 16 Nov 2018 14:50:21 +0100 Subject: [PATCH 049/132] Fixes Plex proxy --- src/routes/proxy.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/proxy.js b/src/routes/proxy.js index a2d4090..be3f590 100644 --- a/src/routes/proxy.js +++ b/src/routes/proxy.js @@ -12,10 +12,10 @@ RoutesProxy.plex = (req, res) => { }).on('error', (err) => { // On some Plex request from FFmpeg, Plex don't create a valid request if (err.code === 'HPE_UNEXPECTED_CONTENT_LENGTH') - res.status(200).send(); + return (res.status(200).send()); // Other error - res.status(400).send({ error: { code: 'PROXY_TIMEOUT', message: 'Plex not respond in time, proxy request fails' } }); + return (res.status(400).send({ error: { code: 'PROXY_TIMEOUT', message: 'Plex not respond in time, proxy request fails' } })); }); return (proxy.web(req, res)); }; From a6eb83e280c1e6e575a42ce28f42bb5a39d65313 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 16 Nov 2018 15:22:30 +0100 Subject: [PATCH 050/132] Fixes --- src/core/sessions.js | 3 +-- src/routes/api.js | 2 +- src/routes/transcode.js | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index f206f11..db92891 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -124,7 +124,7 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { // Progress if (e.indexOf('/progress') !== -1) - return (e.replace(plexUrl(), publicUrl())); + return (e.replace(plexUrl(), '{INTERNAL_TRANSCODER}')); // Manifest and seglist if (e.indexOf('/manifest') !== -1 || e.indexOf('/seglist') !== -1) @@ -169,7 +169,6 @@ SessionsManager.storeFFmpegParameters = (args, env) => { session: parsed.session, sessionFull: parsed.sessionFull }); - console.log('lol', session.session, session); SessionStore.set(session.session, session).then(() => { }).catch((err) => { diff --git a/src/routes/api.js b/src/routes/api.js index 05040a1..9582782 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -65,7 +65,7 @@ RoutesAPI.plex = (req, res) => { RoutesAPI.session = (req, res) => { SessionStore.get(req.params.session).then((data) => { res.send(data); - }).catch((err) => { + }).catch(() => { res.status(400).send({ error: { code: 'SESSION_TIMEOUT', message: 'The session wasn\'t launched in time, request fails' } }); }) }; diff --git a/src/routes/transcode.js b/src/routes/transcode.js index 1aa8d0d..9c5952e 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -120,10 +120,10 @@ RoutesTranscode.cleanSession = (req, res, next) => { D('Clean ' + req.query.session + 'from session store'); SessionsManager.cleanSession(req.query.session) .then(() => { - next() + next(); }) .catch(() => { - res.status(500).send('Internal Server Error') + res.status(500).send({ error: { code: 'INTERNAL_ERROR', message: 'Internal Server Error' } }); }) } }; From 42b1e352020a3629820e2c8d5c3aa749dbe46183 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 16 Nov 2018 15:56:12 +0100 Subject: [PATCH 051/132] Fixes --- src/routes/api.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/routes/api.js b/src/routes/api.js index 9582782..6eb9a99 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -1,11 +1,15 @@ import httpProxy from 'http-proxy'; import sqlite3 from 'sqlite3'; +import debug from 'debug'; import config from '../config'; import SessionStore from '../store'; import SessionsManager from '../core/sessions'; import ServersManager from '../core/servers'; +// Debugger +const D = debug('UnicornLoadBalancer:api'); + let RoutesAPI = {}; // Returns all the stats of all the transcoders @@ -54,7 +58,11 @@ RoutesAPI.plex = (req, res) => { host: config.plex.host, port: config.plex.port } - }).on('error', () => { + }).on('error', (err) => { + if (err.code === 'HPE_UNEXPECTED_CONTENT_LENGTH') { + D('Ignore /progress error'); + return (res.status(200).send()); + } res.status(400).send({ error: { code: 'PROXY_TIMEOUT', message: 'Plex not respond in time, proxy request fails' } }); }); req.url = req.url.slice('/api/plex'.length); From d55abf5dbf195f0977b2becbfc8d4b0aad60c626 Mon Sep 17 00:00:00 2001 From: Benjamin DROUARD Date: Thu, 22 Nov 2018 18:37:45 +0100 Subject: [PATCH 052/132] Public port if using reverse proxy --- src/config.js | 1 + src/utils.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/config.js b/src/config.js index cc445c8..0e6f687 100644 --- a/src/config.js +++ b/src/config.js @@ -6,6 +6,7 @@ export default { version: '2.0.0', server: { port: env.int('SERVER_PORT', 3001), + public_port: env.int('SERVER_PUBLIC_PORT', 443), host: env.string('SERVER_HOST', '127.0.0.1'), ssl: env.bool('SERVER_SSL', false) }, diff --git a/src/utils.js b/src/utils.js index e25bec6..ddd3317 100644 --- a/src/utils.js +++ b/src/utils.js @@ -3,7 +3,7 @@ import redisClient from 'redis'; import config from './config'; export const publicUrl = () => { - return ('http' + ((config.server.ssl) ? 's' : '') + '://' + config.server.host + (([80, 443].indexOf(config.server.port) === -1) ? ':' + config.server.port : '') + '/') + return ('http' + ((config.server.ssl) ? 's' : '') + '://' + config.server.host + (([80, 443].indexOf(config.server.public_port) === -1) ? ':' + config.server.public_port : '') + '/') }; export const internalUrl = () => { From c4a202f9559133d7a9cddac5ab3685bc4353b752 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 19:13:01 +0100 Subject: [PATCH 053/132] Fixes --- src/config.js | 2 +- src/core/sessions.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config.js b/src/config.js index 0e6f687..5517b7f 100644 --- a/src/config.js +++ b/src/config.js @@ -15,7 +15,7 @@ export default { port: env.int('PLEX_PORT', 32400), path: { usr: env.string('PLEX_PATH_USR', '/usr/lib/plexmediaserver/'), - sessions: env.string('PLEX_PATH_SESSIONS', '/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Cache/Transcode/Sessions'), + sessions: env.string('PLEX_PATH_SESSIONS', '/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Cache/Transcode/Sessions/'), database: env.string('PLEX_PATH_DATABASE', '/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db') } }, diff --git a/src/core/sessions.js b/src/core/sessions.js index db92891..974f9cc 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -20,7 +20,7 @@ SessionsManager.list = () => { // Parse request to extract session information SessionsManager.parseSessionFromRequest = (req) => { const unicorn = (typeof (req.query.unicorn) !== 'undefined') ? { unicorn: req.query.unicorn } : false; - const session = (typeof (req.params.sessionId) !== 'undefined') ? { sessionId: req.params.sessionId } : ((typeof (req.query.session) !== 'undefined') ? { session: req.query.session } : false); + const session = (typeof (req.params.sessionId) !== 'undefined') ? { session: req.params.sessionId } : ((typeof (req.query.session) !== 'undefined') ? { session: req.query.session } : false); const sessionIdentifier = (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined') ? { sessionIdentifier: req.query['X-Plex-Session-Identifier'] } : false; const clientIdentifier = (typeof (req.query['X-Plex-Client-Identifier']) !== 'undefined') ? { clientIdentifier: req.query['X-Plex-Client-Identifier'] } : false; return { From ea96aa11f2b87df2208a319d77b2f823862b1f14 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 19:24:07 +0100 Subject: [PATCH 054/132] Try to fix session matching --- src/core/sessions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 974f9cc..688147b 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -43,7 +43,7 @@ SessionsManager.getSessionFromRequest = (search) => { SessionsManager.getIdFromRequest = (search) => { // List of keys could be used to identify a session - const keys = ['unicorn', 'session', 'sessionFull', 'sessionIdentifier', 'clientIdentifier']; + const keys = ['unicorn', 'session', 'sessionFull']; // Reverse session to start by the end const rsessions = sessions.slice().reverse(); From 7290017ce46591a6c2b495e63e5c8e50d75ad9f7 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 19:27:56 +0100 Subject: [PATCH 055/132] Fixes --- src/core/sessions.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 688147b..5acc6de 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -43,7 +43,7 @@ SessionsManager.getSessionFromRequest = (search) => { SessionsManager.getIdFromRequest = (search) => { // List of keys could be used to identify a session - const keys = ['unicorn', 'session', 'sessionFull']; + let keys = ['unicorn', 'session', 'sessionFull']; // Reverse session to start by the end const rsessions = sessions.slice().reverse(); @@ -56,6 +56,15 @@ SessionsManager.getIdFromRequest = (search) => { } } + keys = ['sessionIdentifier', 'clientIdentifier']; + + for (let idx = 0; idx < rsessions.length; idx++) { + for (let i = 0; i < keys.length; i++) { + if (rsessions[idx][keys[i]] && search[keys[i]] && rsessions[idx][keys[i]] === search[keys[i]]) + return (idx); + } + } + // Android case, no session, only a sessionIdentifier if (!search.session && search.sessionIdentifier) return (SessionsManager.getIdFromRequest({ ...search, session: search.sessionIdentifier })); From 2ebf96e79f0403c3d0ec921233a1f2aed7d4275e Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 19:34:47 +0100 Subject: [PATCH 056/132] Fixes --- src/core/sessions.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 5acc6de..bbd2d4a 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -43,7 +43,7 @@ SessionsManager.getSessionFromRequest = (search) => { SessionsManager.getIdFromRequest = (search) => { // List of keys could be used to identify a session - let keys = ['unicorn', 'session', 'sessionFull']; + let keys = ['unicorn', 'session', 'sessionFull', 'clientIdentifier']; // Reverse session to start by the end const rsessions = sessions.slice().reverse(); @@ -56,18 +56,18 @@ SessionsManager.getIdFromRequest = (search) => { } } - keys = ['sessionIdentifier', 'clientIdentifier']; + //keys = ['sessionIdentifier', 'clientIdentifier']; - for (let idx = 0; idx < rsessions.length; idx++) { + /*for (let idx = 0; idx < rsessions.length; idx++) { for (let i = 0; i < keys.length; i++) { if (rsessions[idx][keys[i]] && search[keys[i]] && rsessions[idx][keys[i]] === search[keys[i]]) return (idx); } - } + }*/ // Android case, no session, only a sessionIdentifier - if (!search.session && search.sessionIdentifier) - return (SessionsManager.getIdFromRequest({ ...search, session: search.sessionIdentifier })); + //if (!search.session && search.sessionIdentifier) + // return (SessionsManager.getIdFromRequest({ ...search, session: search.sessionIdentifier })); // Ok, Android really sucks, other case, no session, only a clientIdentifier if (!search.session && search.clientIdentifier) From 856beb0a8722b3e0cb2ee09e16c39e268805ce39 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 19:51:14 +0100 Subject: [PATCH 057/132] Fixes --- src/core/sessions.js | 2 +- src/routes/index.js | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index bbd2d4a..3ec4791 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -43,7 +43,7 @@ SessionsManager.getSessionFromRequest = (search) => { SessionsManager.getIdFromRequest = (search) => { // List of keys could be used to identify a session - let keys = ['unicorn', 'session', 'sessionFull', 'clientIdentifier']; + let keys = ['session', 'sessionIdentifier']; // Reverse session to start by the end const rsessions = sessions.slice().reverse(); diff --git a/src/routes/index.js b/src/routes/index.js index df3128b..99f46a0 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -21,16 +21,22 @@ export default (app) => { app.all('/api/plex/*', RoutesAPI.plex); // MPEG Dash support - app.get('/:formatType/:/transcode/universal/start.mpd', RoutesTranscode.redirect); + app.get('/:formatType/:/transcode/universal/start.mpd', (res, req) => { + return RoutesTranscode.redirect(req, res); + }); app.get('/:formatType/:/transcode/universal/dash/:sessionId/:streamId/initial.mp4', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/dash/:sessionId/:streamId/:partId.m4s', RoutesTranscode.redirect); // Long polling support - app.get('/:formatType/:/transcode/universal/start', RoutesTranscode.redirect); + app.get('/:formatType/:/transcode/universal/start', (res, req) => { + return RoutesTranscode.redirect(req, res); + }); app.get('/:formatType/:/transcode/universal/subtitles', RoutesTranscode.redirect); // M3U8 support - app.get('/:formatType/:/transcode/universal/start.m3u8', RoutesTranscode.cleanSession, RoutesProxy.plex); + app.get('/:formatType/:/transcode/universal/start.m3u8', (res, req) => { + return RoutesTranscode.redirect(req, res); + }); app.get('/:formatType/:/transcode/universal/session/:sessionId/base/index.m3u8', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/base-x-mc/index.m3u8', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/:fileType/:partId.ts', RoutesTranscode.redirect); From 22f39ff3b62341039f2ec95ebf682351cea0bab6 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 19:55:23 +0100 Subject: [PATCH 058/132] Fixes --- src/routes/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routes/index.js b/src/routes/index.js index 99f46a0..68c5c0c 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -21,20 +21,20 @@ export default (app) => { app.all('/api/plex/*', RoutesAPI.plex); // MPEG Dash support - app.get('/:formatType/:/transcode/universal/start.mpd', (res, req) => { + app.get('/:formatType/:/transcode/universal/start.mpd', (req, res) => { return RoutesTranscode.redirect(req, res); }); app.get('/:formatType/:/transcode/universal/dash/:sessionId/:streamId/initial.mp4', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/dash/:sessionId/:streamId/:partId.m4s', RoutesTranscode.redirect); // Long polling support - app.get('/:formatType/:/transcode/universal/start', (res, req) => { + app.get('/:formatType/:/transcode/universal/start', (req, res) => { return RoutesTranscode.redirect(req, res); }); app.get('/:formatType/:/transcode/universal/subtitles', RoutesTranscode.redirect); // M3U8 support - app.get('/:formatType/:/transcode/universal/start.m3u8', (res, req) => { + app.get('/:formatType/:/transcode/universal/start.m3u8', (req, res) => { return RoutesTranscode.redirect(req, res); }); app.get('/:formatType/:/transcode/universal/session/:sessionId/base/index.m3u8', RoutesTranscode.redirect); From 8b426828f5c8c1a8506d182576c8992838c727d6 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 20:00:09 +0100 Subject: [PATCH 059/132] Fixes --- src/routes/index.js | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/routes/index.js b/src/routes/index.js index 68c5c0c..df3128b 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -21,22 +21,16 @@ export default (app) => { app.all('/api/plex/*', RoutesAPI.plex); // MPEG Dash support - app.get('/:formatType/:/transcode/universal/start.mpd', (req, res) => { - return RoutesTranscode.redirect(req, res); - }); + app.get('/:formatType/:/transcode/universal/start.mpd', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/dash/:sessionId/:streamId/initial.mp4', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/dash/:sessionId/:streamId/:partId.m4s', RoutesTranscode.redirect); // Long polling support - app.get('/:formatType/:/transcode/universal/start', (req, res) => { - return RoutesTranscode.redirect(req, res); - }); + app.get('/:formatType/:/transcode/universal/start', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/subtitles', RoutesTranscode.redirect); // M3U8 support - app.get('/:formatType/:/transcode/universal/start.m3u8', (req, res) => { - return RoutesTranscode.redirect(req, res); - }); + app.get('/:formatType/:/transcode/universal/start.m3u8', RoutesTranscode.cleanSession, RoutesProxy.plex); app.get('/:formatType/:/transcode/universal/session/:sessionId/base/index.m3u8', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/base-x-mc/index.m3u8', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/:fileType/:partId.ts', RoutesTranscode.redirect); From d83250051a30fa9bd4d0d74c22a629a233dbfb2b Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 20:02:11 +0100 Subject: [PATCH 060/132] Fixes --- src/core/sessions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 3ec4791..bfd2460 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -19,7 +19,7 @@ SessionsManager.list = () => { // Parse request to extract session information SessionsManager.parseSessionFromRequest = (req) => { - const unicorn = (typeof (req.query.unicorn) !== 'undefined') ? { unicorn: req.query.unicorn } : false; + const unicorn = (typeof (req.query) !== 'undefined' && typeof (req.query.unicorn) !== 'undefined') ? { unicorn: req.query.unicorn } : false; const session = (typeof (req.params.sessionId) !== 'undefined') ? { session: req.params.sessionId } : ((typeof (req.query.session) !== 'undefined') ? { session: req.query.session } : false); const sessionIdentifier = (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined') ? { sessionIdentifier: req.query['X-Plex-Session-Identifier'] } : false; const clientIdentifier = (typeof (req.query['X-Plex-Client-Identifier']) !== 'undefined') ? { clientIdentifier: req.query['X-Plex-Client-Identifier'] } : false; From ae25bcc7e1f705d0f85c6c93498936ec447adb25 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 20:17:42 +0100 Subject: [PATCH 061/132] test --- src/core/sessions.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index bfd2460..32ca895 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -85,14 +85,13 @@ SessionsManager.updateSessionFromRequest = (req) => { // Update a session SessionsManager.updateSession = (args) => { - const search = SessionsManager.getSessionFromRequest(args); const idx = SessionsManager.getIdFromRequest(args); // Avoid to create empty session objects (Download case by example) if (!args.session && !args.sessionFull && !args.sessionIdentifier && !args.clientIdentifier) return (false); - if (!search) { + if (idx === false) { sessions.push({ unicorn: uniqid(), session: '', @@ -173,11 +172,15 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { // Store the FFMPEG parameters in RedisCache SessionsManager.storeFFmpegParameters = (args, env) => { const parsed = SessionsManager.parseFFmpegParameters(args, env); + console.log('FFMEG returns', parsed); SessionsManager.updateSession(parsed); + + const session = SessionsManager.getSessionFromRequest({ session: parsed.session, sessionFull: parsed.sessionFull }); +console.log('Gette', session) SessionStore.set(session.session, session).then(() => { }).catch((err) => { From 50dcf46db047ec85ca5d487f1b699e617764c07a Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 20:24:58 +0100 Subject: [PATCH 062/132] fix --- src/core/sessions.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 32ca895..9a3fa28 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -70,8 +70,8 @@ SessionsManager.getIdFromRequest = (search) => { // return (SessionsManager.getIdFromRequest({ ...search, session: search.sessionIdentifier })); // Ok, Android really sucks, other case, no session, only a clientIdentifier - if (!search.session && search.clientIdentifier) - return (SessionsManager.getIdFromRequest({ ...search, session: search.clientIdentifier })); + /*if (!search.session && search.clientIdentifier) + return (SessionsManager.getIdFromRequest({ ...search, session: search.clientIdentifier }));*/ // Not be found return (false); @@ -172,15 +172,14 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { // Store the FFMPEG parameters in RedisCache SessionsManager.storeFFmpegParameters = (args, env) => { const parsed = SessionsManager.parseFFmpegParameters(args, env); - console.log('FFMEG returns', parsed); - SessionsManager.updateSession(parsed); + SessionsManager.updateSession(parsed); const session = SessionsManager.getSessionFromRequest({ session: parsed.session, sessionFull: parsed.sessionFull }); -console.log('Gette', session) + SessionStore.set(session.session, session).then(() => { }).catch((err) => { From 252f5c6183770dccfadc84dbfd125c94c91bbcd9 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 20:28:28 +0100 Subject: [PATCH 063/132] fixes --- src/core/sessions.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 9a3fa28..85f7bde 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -85,11 +85,12 @@ SessionsManager.updateSessionFromRequest = (req) => { // Update a session SessionsManager.updateSession = (args) => { - const idx = SessionsManager.getIdFromRequest(args); - + // Avoid to create empty session objects (Download case by example) - if (!args.session && !args.sessionFull && !args.sessionIdentifier && !args.clientIdentifier) + if (!args.session) return (false); + + const idx = SessionsManager.getIdFromRequest(args); if (idx === false) { sessions.push({ From b610ab69076dccb633a84d6ac3a09ccc72a65484 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 23:10:54 +0100 Subject: [PATCH 064/132] Refactoring and tests --- src/core/sessions.js | 139 +++++++++++----------------------------- src/routes/index.js | 100 +++++++++++++++++++++++++++-- src/routes/transcode.js | 38 +++++------ 3 files changed, 146 insertions(+), 131 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 85f7bde..9c9dd07 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -1,9 +1,9 @@ import debug from 'debug'; -import uniqid from 'uniqid'; import config from '../config'; import { publicUrl, plexUrl } from '../utils'; import SessionStore from '../store'; +import ServersManager from './servers'; // Debugger const D = debug('UnicornLoadBalancer:SessionsManager'); @@ -13,105 +13,52 @@ let SessionsManager = {}; let sessions = [ ]; +// Plex table to match "session" and "X-Plex-Session-Identifier" +let cache = {}; + +// Table to link session to transcoder url +let urls = {} + +// list all the sessions SessionsManager.list = () => { return (sessions); }; -// Parse request to extract session information -SessionsManager.parseSessionFromRequest = (req) => { - const unicorn = (typeof (req.query) !== 'undefined' && typeof (req.query.unicorn) !== 'undefined') ? { unicorn: req.query.unicorn } : false; - const session = (typeof (req.params.sessionId) !== 'undefined') ? { session: req.params.sessionId } : ((typeof (req.query.session) !== 'undefined') ? { session: req.query.session } : false); - const sessionIdentifier = (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined') ? { sessionIdentifier: req.query['X-Plex-Session-Identifier'] } : false; - const clientIdentifier = (typeof (req.query['X-Plex-Client-Identifier']) !== 'undefined') ? { clientIdentifier: req.query['X-Plex-Client-Identifier'] } : false; - return { - ...unicorn, - ...session, - ...sessionIdentifier, - ...clientIdentifier - } -}; - -// Get a session from its values -SessionsManager.getSessionFromRequest = (search) => { - const sessionIndex = SessionsManager.getIdFromRequest(search); - if (sessionIndex === false) +SessionsManager.chooseServer = async (session, ip = false) => { + if (session === false) return (false); - return (sessions[sessionIndex]); + if (urls[session]) + return (urls[session]); + const url = await ServersManager.chooseServer(ip); + urls[session] = url; + return (url); }; -// Get a session position from its values -SessionsManager.getIdFromRequest = (search) => { - - // List of keys could be used to identify a session - let keys = ['session', 'sessionIdentifier']; - - // Reverse session to start by the end - const rsessions = sessions.slice().reverse(); - - // Filter sessions - for (let idx = 0; idx < rsessions.length; idx++) { - for (let i = 0; i < keys.length; i++) { - if (rsessions[idx][keys[i]] && search[keys[i]] && rsessions[idx][keys[i]] === search[keys[i]]) - return (idx); - } +SessionsManager.cacheSessionFromRequest = (req) => { + if (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined' && typeof (req.query.session) !== 'undefined') { + cache[req.query['X-Plex-Session-Identifier']] = req.query.session.toString(); } +} - //keys = ['sessionIdentifier', 'clientIdentifier']; - - /*for (let idx = 0; idx < rsessions.length; idx++) { - for (let i = 0; i < keys.length; i++) { - if (rsessions[idx][keys[i]] && search[keys[i]] && rsessions[idx][keys[i]] === search[keys[i]]) - return (idx); - } - }*/ - - // Android case, no session, only a sessionIdentifier - //if (!search.session && search.sessionIdentifier) - // return (SessionsManager.getIdFromRequest({ ...search, session: search.sessionIdentifier })); - - // Ok, Android really sucks, other case, no session, only a clientIdentifier - /*if (!search.session && search.clientIdentifier) - return (SessionsManager.getIdFromRequest({ ...search, session: search.clientIdentifier }));*/ - - // Not be found +SessionsManager.getCacheSession = (xplexsessionidentifier) => { + if (cache[xplexsessionidentifier]) + return (cache[xplexsessionidentifier]); return (false); -}; - -// Update the session stored -SessionsManager.updateSessionFromRequest = (req) => { - const args = SessionsManager.parseSessionFromRequest(req); - return (SessionsManager.updateSession(args)); -}; - -// Update a session -SessionsManager.updateSession = (args) => { - - // Avoid to create empty session objects (Download case by example) - if (!args.session) - return (false); - - const idx = SessionsManager.getIdFromRequest(args); - - if (idx === false) { - sessions.push({ - unicorn: uniqid(), - session: '', - sessionFull: '', - sessionIdentifier: '', - clientIdentifier: '', - args: [], - env: [], - serverUrl: '', - ...args - }); - return (true); - } - sessions[idx] = { - ...sessions[idx], - ...args - }; +} + +SessionsManager.getSessionFromRequest = (req) => { + if (typeof (req.params.sessionId) !== 'undefined') + return (req.params.sessionId); + if (typeof (req.query.session) !== 'undefined') + return (req.query.session); + if (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined' && typeof (cache[req.query['X-Plex-Session-Identifier']]) !== 'undefined') + return (cache[req.query['X-Plex-Session-Identifier']]); + if (typeof (req.query['X-Plex-Session-Identifier']) !== 'undefined') + return (req.query['X-Plex-Session-Identifier']); + if (typeof (req.query['X-Plex-Client-Identifier']) !== 'undefined') + return (req.query['X-Plex-Client-Identifier']); return (false); -}; +} // Parse FFmpeg parameters with internal bindings SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { @@ -173,19 +120,7 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { // Store the FFMPEG parameters in RedisCache SessionsManager.storeFFmpegParameters = (args, env) => { const parsed = SessionsManager.parseFFmpegParameters(args, env); - - SessionsManager.updateSession(parsed); - - const session = SessionsManager.getSessionFromRequest({ - session: parsed.session, - sessionFull: parsed.sessionFull - }); - - SessionStore.set(session.session, session).then(() => { - - }).catch((err) => { - - }) + SessionStore.set(parsed.session, parsed).then(() => { }).catch(() => { }) return (parsed); }; diff --git a/src/routes/index.js b/src/routes/index.js index df3128b..ff3857b 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -1,9 +1,11 @@ import express from 'express'; +import fetch from 'node-fetch'; import config from '../config'; import RoutesAPI from './api'; import RoutesTranscode from './transcode'; import RoutesProxy from './proxy'; +import SessionsManager from '../core/sessions'; export default (app) => { @@ -20,26 +22,112 @@ export default (app) => { app.get('/api/session/:session', RoutesAPI.session); app.all('/api/plex/*', RoutesAPI.plex); + //--------------------------------------------- + // MPEG Dash support - app.get('/:formatType/:/transcode/universal/start.mpd', RoutesTranscode.redirect); + app.get('/:formatType/:/transcode/universal/start.mpd', (req, res) => { + + // By default we don't have the session identifier + let sessionId = false; + + // If we have a cached X-Plex-Session-Identifier, we use it + if (req.query['X-Plex-Session-Identifier'] && SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier'])) + sessionId = SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier']); + + // Save session + SessionsManager.cacheSessionFromRequest(req); + + // Redirect + RoutesTranscode.redirect(req, res); + }); app.get('/:formatType/:/transcode/universal/dash/:sessionId/:streamId/initial.mp4', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/dash/:sessionId/:streamId/:partId.m4s', RoutesTranscode.redirect); // Long polling support - app.get('/:formatType/:/transcode/universal/start', RoutesTranscode.redirect); + app.get('/:formatType/:/transcode/universal/start', (req, res) => { + // Save session + SessionsManager.cacheSessionFromRequest(req); + + // Redirect + RoutesTranscode.redirect(req, res); + }); app.get('/:formatType/:/transcode/universal/subtitles', RoutesTranscode.redirect); // M3U8 support - app.get('/:formatType/:/transcode/universal/start.m3u8', RoutesTranscode.cleanSession, RoutesProxy.plex); + app.get('/:formatType/:/transcode/universal/start.m3u8', (req, res) => { + // Save session + SessionsManager.cacheSessionFromRequest(req); + + // Redirect + RoutesTranscode.redirect(req, res); + }); app.get('/:formatType/:/transcode/universal/session/:sessionId/base/index.m3u8', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/base-x-mc/index.m3u8', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/:fileType/:partId.ts', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/:fileType/:partId.vtt', RoutesTranscode.redirect); // Control support - app.get('/:formatType/:/transcode/universal/stop', RoutesTranscode.stop); - app.get('/:formatType/:/transcode/universal/ping', RoutesTranscode.ping); - app.get('/:/timeline', RoutesTranscode.timeline); + app.get('/:formatType/:/transcode/universal/stop', (req, res) => { + // Proxy to plex + RoutesProxy.plex(req, res); + + // Extract sessionId from request parameter + const sessionId = SessionsManager.getSessionFromRequest(req); + + // Choose or get the server url + const serverUrl = SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); + + // If a server url is defined, we stop the session + if (serverUrl) + fetch(serverUrl + '/api/stop?session=' + sessionId); + + // Remove session after few seconds to avoid bad redirect + setTimeout(() => { + SessionsManager.cleanSession(sessionId); + }, 1500); + }); + + app.get('/:formatType/:/transcode/universal/ping', (req, res) => { + // Proxy to Plex + RoutesProxy.plex(req, res); + + // Extract sessionId from request parameter + const sessionId = SessionsManager.getSessionFromRequest(req); + + // Choose or get the server url + const serverUrl = SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); + + // If a server url is defined, we ping the session + if (serverUrl) + fetch(serverUrl + '/api/ping?session=' + sessionId); + }); + + app.get('/:/timeline', (req, res) => { + // Proxy to Plex + RoutesProxy.plex(req, res); + + // Extract sessionId from request parameter + const sessionId = SessionsManager.getSessionFromRequest(req); + + // Choose or get the server url + const serverUrl = SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); + + // It's a stop request + if (req.query.state == 'stopped' || (req.query['X-Plex-Session-Identifier'] && SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier']))) { + // If a server url is defined, we stop the session + if (serverUrl) + fetch(serverUrl + '/api/stop?session=' + sessionId); + + // Remove session after few seconds to avoid bad redirect + setTimeout(() => { + SessionsManager.cleanSession(sessionId); + }, 1500); + } + // it's a ping request + else if (serverUrl) { + fetch(serverUrl + '/api/ping?session=' + sessionId); + } + }); // Download app.get('/library/parts/:id1/:id2/file.*', RoutesTranscode.redirect); diff --git a/src/routes/transcode.js b/src/routes/transcode.js index 9c5952e..6bc66e6 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -11,33 +11,24 @@ const D = debug('UnicornLoadBalancer:transcode'); let RoutesTranscode = {}; RoutesTranscode.redirect = (req, res) => { - SessionsManager.updateSessionFromRequest(req); + const session = SessionsManager.getSessionFromRequest(req); + const server = SessionsManager.chooseServer(session, req.headers['x-forwarded-for'] || req.connection.remoteAddress); + if (server) { + res.writeHead(302, { + 'Location': server + req.url + }); + res.end(); + D('Send 302 for ' + session + ' to ' + server); + } else { + res.status(500).send({ error: { code: 'SERVER_UNAVAILABLE', message: 'SERVER_UNAVAILABLE' } }); + D('Fail to 302 for ' + session); + } +}; - const search = SessionsManager.parseSessionFromRequest(req); - const session = SessionsManager.getSessionFromRequest(search); - const redirectRequest = (server) => { - if (server) { - res.writeHead(302, { - 'Location': (server + req.url + (req.url.indexOf('?') === -1 ? '?' : '&') + 'unicorn=' + session.unicorn) - }); - res.end(); - D('Send 302 for ' + session.session + ' to ' + server); - return; - } - D('Fail to 302 for ' + session.session + ' to ' + server); - }; - if (session.serverUrl) { - return (redirectRequest(session.serverUrl)); - } - - ServersManager.chooseServer(req.headers['x-forwarded-for'] || req.connection.remoteAddress).then((server) => { - SessionsManager.updateSession({ ...session, serverUrl: server }); - return (redirectRequest(server)); - }); -}; +/* RoutesTranscode.ping = (req, res) => { SessionsManager.updateSessionFromRequest(req); @@ -128,4 +119,5 @@ RoutesTranscode.cleanSession = (req, res, next) => { } }; +*/ export default RoutesTranscode; From d85b92c985a28d24f7eaa2d97beaab943a10618a Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 23:17:37 +0100 Subject: [PATCH 065/132] async --- src/routes/transcode.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index 6bc66e6..c5d3f3c 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -10,9 +10,9 @@ const D = debug('UnicornLoadBalancer:transcode'); let RoutesTranscode = {}; -RoutesTranscode.redirect = (req, res) => { +RoutesTranscode.redirect = async (req, res) => { const session = SessionsManager.getSessionFromRequest(req); - const server = SessionsManager.chooseServer(session, req.headers['x-forwarded-for'] || req.connection.remoteAddress); + const server = await SessionsManager.chooseServer(session, req.headers['x-forwarded-for'] || req.connection.remoteAddress); if (server) { res.writeHead(302, { 'Location': server + req.url From 10896d5a7c5bf661505126f7754cd10c4cc0bd74 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 23:21:36 +0100 Subject: [PATCH 066/132] async --- src/routes/index.js | 12 +++++------ src/routes/resize.js | 48 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 src/routes/resize.js diff --git a/src/routes/index.js b/src/routes/index.js index ff3857b..48a1954 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -67,7 +67,7 @@ export default (app) => { app.get('/:formatType/:/transcode/universal/session/:sessionId/:fileType/:partId.vtt', RoutesTranscode.redirect); // Control support - app.get('/:formatType/:/transcode/universal/stop', (req, res) => { + app.get('/:formatType/:/transcode/universal/stop', async (req, res) => { // Proxy to plex RoutesProxy.plex(req, res); @@ -75,7 +75,7 @@ export default (app) => { const sessionId = SessionsManager.getSessionFromRequest(req); // Choose or get the server url - const serverUrl = SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); + const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); // If a server url is defined, we stop the session if (serverUrl) @@ -87,7 +87,7 @@ export default (app) => { }, 1500); }); - app.get('/:formatType/:/transcode/universal/ping', (req, res) => { + app.get('/:formatType/:/transcode/universal/ping', async (req, res) => { // Proxy to Plex RoutesProxy.plex(req, res); @@ -95,14 +95,14 @@ export default (app) => { const sessionId = SessionsManager.getSessionFromRequest(req); // Choose or get the server url - const serverUrl = SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); + const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); // If a server url is defined, we ping the session if (serverUrl) fetch(serverUrl + '/api/ping?session=' + sessionId); }); - app.get('/:/timeline', (req, res) => { + app.get('/:/timeline', async (req, res) => { // Proxy to Plex RoutesProxy.plex(req, res); @@ -110,7 +110,7 @@ export default (app) => { const sessionId = SessionsManager.getSessionFromRequest(req); // Choose or get the server url - const serverUrl = SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); + const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); // It's a stop request if (req.query.state == 'stopped' || (req.query['X-Plex-Session-Identifier'] && SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier']))) { diff --git a/src/routes/resize.js b/src/routes/resize.js new file mode 100644 index 0000000..bd3858b --- /dev/null +++ b/src/routes/resize.js @@ -0,0 +1,48 @@ +import sharp from 'sharp'; +import fetch from 'node-fetch'; +import { PassThrough } from 'stream'; +import { plexUrl } from '../utils'; + +let RoutesResize = {}; + +const resizeFromUrl = (url, width = false, height = false, minSize = 0, format = 'jpeg') => { + let pass = new PassThrough(); + + fetch(url).then(res => { + res.body.pipe(pass); + }); + + let transform = sharp(); + + + if (width || height) { + transform = transform.resize(((minSize > 0) ? width : undefined), ((minSize <= 0) ? height : undefined)); + } + + if (format) { + transform = transform.toFormat(sharp.format.webp); // jpeg | webp | png + } + + return pass.pipe(transform); +} + +RoutesResize.resize = (req, res) => { + let url = req.query.url || false; + if (url && url[0] == '/') + url = plexUrl() + url; + + const params = { + width: parseInt(req.query.width) || false, + height: parseInt(req.query.height) || false, + minSize: parseInt(req.query.minSize) || 0, + url + }; + + if (!params.width || !params.height || !params.url) + return (res.status(400).send({ error: { code: 'RESIZE_ERROR', message: 'Invalid parameters, resize request fails' } })); + + res.type(`image/webp`); + resizeFromUrl(params.url, params.width, params.height, params.minSize).pipe(res) +}; + +export default RoutesResize; \ No newline at end of file From 6048fb435d88931b3a320c1ceae1d1f15042dc26 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Mon, 3 Dec 2018 23:35:25 +0100 Subject: [PATCH 067/132] Fixes delete keys --- src/routes/index.js | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/routes/index.js b/src/routes/index.js index 48a1954..4df024a 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -37,6 +37,10 @@ export default (app) => { // Save session SessionsManager.cacheSessionFromRequest(req); + // If session id available + if (sessionId) + SessionsManager.cleanSession(sessionId); + // Redirect RoutesTranscode.redirect(req, res); }); @@ -48,6 +52,13 @@ export default (app) => { // Save session SessionsManager.cacheSessionFromRequest(req); + // Get sessionId + const sessionId = SessionsManager.getSessionFromRequest(req); + + // If sessionId is defined + if (sessionId) + SessionsManager.cleanSession(sessionId); + // Redirect RoutesTranscode.redirect(req, res); }); @@ -58,6 +69,13 @@ export default (app) => { // Save session SessionsManager.cacheSessionFromRequest(req); + // Get sessionId + const sessionId = SessionsManager.getSessionFromRequest(req); + + // If sessionId is defined + if (sessionId) + SessionsManager.cleanSession(sessionId); + // Redirect RoutesTranscode.redirect(req, res); }); @@ -80,11 +98,6 @@ export default (app) => { // If a server url is defined, we stop the session if (serverUrl) fetch(serverUrl + '/api/stop?session=' + sessionId); - - // Remove session after few seconds to avoid bad redirect - setTimeout(() => { - SessionsManager.cleanSession(sessionId); - }, 1500); }); app.get('/:formatType/:/transcode/universal/ping', async (req, res) => { @@ -113,15 +126,10 @@ export default (app) => { const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); // It's a stop request - if (req.query.state == 'stopped' || (req.query['X-Plex-Session-Identifier'] && SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier']))) { + if (req.query.state === 'stopped'/* || (req.query['X-Plex-Session-Identifier'] && SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier']))*/) { // If a server url is defined, we stop the session if (serverUrl) fetch(serverUrl + '/api/stop?session=' + sessionId); - - // Remove session after few seconds to avoid bad redirect - setTimeout(() => { - SessionsManager.cleanSession(sessionId); - }, 1500); } // it's a ping request else if (serverUrl) { From 6a815623787dab3676d1b1f5ee8ebfafe25de324 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Tue, 4 Dec 2018 00:01:05 +0100 Subject: [PATCH 068/132] Clean code --- src/core/servers.js | 1 - src/core/sessions.js | 11 +-- src/routes/api.js | 8 +- src/routes/index.js | 109 ++-------------------------- src/routes/transcode.js | 157 ++++++++++++++++++++++------------------ 5 files changed, 97 insertions(+), 189 deletions(-) diff --git a/src/core/servers.js b/src/core/servers.js index 9908b51..edf732a 100644 --- a/src/core/servers.js +++ b/src/core/servers.js @@ -2,7 +2,6 @@ import fetch from 'node-fetch'; import { time } from '../utils'; import config from '../config'; - let servers = {}; let ServersManager = {}; diff --git a/src/core/sessions.js b/src/core/sessions.js index 9c9dd07..3350ab9 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -10,26 +10,19 @@ const D = debug('UnicornLoadBalancer:SessionsManager'); let SessionsManager = {}; -let sessions = [ -]; - // Plex table to match "session" and "X-Plex-Session-Identifier" let cache = {}; // Table to link session to transcoder url let urls = {} -// list all the sessions -SessionsManager.list = () => { - return (sessions); -}; - SessionsManager.chooseServer = async (session, ip = false) => { if (session === false) return (false); if (urls[session]) return (urls[session]); const url = await ServersManager.chooseServer(ip); + D('Choosed server for ' + session + ': ' + url); urls[session] = url; return (url); }; @@ -120,11 +113,13 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { // Store the FFMPEG parameters in RedisCache SessionsManager.storeFFmpegParameters = (args, env) => { const parsed = SessionsManager.parseFFmpegParameters(args, env); + D('FFMPEG callback for session ' + parsed.session); SessionStore.set(parsed.session, parsed).then(() => { }).catch(() => { }) return (parsed); }; SessionsManager.cleanSession = (sessionId) => { + D('Delete ' + sessionId); return SessionStore.delete(sessionId) }; diff --git a/src/routes/api.js b/src/routes/api.js index 6eb9a99..8b3bef1 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -17,10 +17,6 @@ RoutesAPI.stats = (req, res) => { res.send(ServersManager.list()); }; -RoutesAPI.sessions = (req, res) => { - res.send(SessionsManager.list()); -}; - // Save the stats of a server RoutesAPI.update = (req, res) => { res.send(ServersManager.update(req.body)); @@ -38,7 +34,7 @@ RoutesAPI.ffmpeg = (req, res) => { RoutesAPI.path = (req, res) => { try { const db = new (sqlite3.verbose().Database)(config.plex.path.database); - db.get("SELECT * FROM media_parts WHERE id=? LIMIT 0, 1", req.params.id, (err, row) => { + db.get('SELECT * FROM media_parts WHERE id=? LIMIT 0, 1', req.params.id, (err, row) => { if (row && row.file) res.send(JSON.stringify(row)); else @@ -69,7 +65,7 @@ RoutesAPI.plex = (req, res) => { return (proxy.web(req, res)); }; -// Returns sessions from UnicornID +// Returns session RoutesAPI.session = (req, res) => { SessionStore.get(req.params.session).then((data) => { res.send(data); diff --git a/src/routes/index.js b/src/routes/index.js index 4df024a..b299a4f 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -1,11 +1,9 @@ import express from 'express'; -import fetch from 'node-fetch'; import config from '../config'; import RoutesAPI from './api'; import RoutesTranscode from './transcode'; import RoutesProxy from './proxy'; -import SessionsManager from '../core/sessions'; export default (app) => { @@ -14,7 +12,6 @@ export default (app) => { // UnicornLoadBalancer API app.use('/api/sessions', express.static(config.plex.path.sessions)); - app.use('/api/debug', RoutesAPI.sessions); app.get('/api/stats', RoutesAPI.stats); app.post('/api/ffmpeg', RoutesAPI.ffmpeg); app.get('/api/path/:id', RoutesAPI.path); @@ -22,120 +19,26 @@ export default (app) => { app.get('/api/session/:session', RoutesAPI.session); app.all('/api/plex/*', RoutesAPI.plex); - //--------------------------------------------- - // MPEG Dash support - app.get('/:formatType/:/transcode/universal/start.mpd', (req, res) => { - - // By default we don't have the session identifier - let sessionId = false; - - // If we have a cached X-Plex-Session-Identifier, we use it - if (req.query['X-Plex-Session-Identifier'] && SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier'])) - sessionId = SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier']); - - // Save session - SessionsManager.cacheSessionFromRequest(req); - - // If session id available - if (sessionId) - SessionsManager.cleanSession(sessionId); - - // Redirect - RoutesTranscode.redirect(req, res); - }); + app.get('/:formatType/:/transcode/universal/start.mpd', RoutesTranscode.dashStart); app.get('/:formatType/:/transcode/universal/dash/:sessionId/:streamId/initial.mp4', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/dash/:sessionId/:streamId/:partId.m4s', RoutesTranscode.redirect); // Long polling support - app.get('/:formatType/:/transcode/universal/start', (req, res) => { - // Save session - SessionsManager.cacheSessionFromRequest(req); - - // Get sessionId - const sessionId = SessionsManager.getSessionFromRequest(req); - - // If sessionId is defined - if (sessionId) - SessionsManager.cleanSession(sessionId); - - // Redirect - RoutesTranscode.redirect(req, res); - }); + app.get('/:formatType/:/transcode/universal/start', RoutesTranscode.lpStart); app.get('/:formatType/:/transcode/universal/subtitles', RoutesTranscode.redirect); // M3U8 support - app.get('/:formatType/:/transcode/universal/start.m3u8', (req, res) => { - // Save session - SessionsManager.cacheSessionFromRequest(req); - - // Get sessionId - const sessionId = SessionsManager.getSessionFromRequest(req); - - // If sessionId is defined - if (sessionId) - SessionsManager.cleanSession(sessionId); - - // Redirect - RoutesTranscode.redirect(req, res); - }); + app.get('/:formatType/:/transcode/universal/start.m3u8', RoutesTranscode.hlsStart); app.get('/:formatType/:/transcode/universal/session/:sessionId/base/index.m3u8', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/base-x-mc/index.m3u8', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/:fileType/:partId.ts', RoutesTranscode.redirect); app.get('/:formatType/:/transcode/universal/session/:sessionId/:fileType/:partId.vtt', RoutesTranscode.redirect); // Control support - app.get('/:formatType/:/transcode/universal/stop', async (req, res) => { - // Proxy to plex - RoutesProxy.plex(req, res); - - // Extract sessionId from request parameter - const sessionId = SessionsManager.getSessionFromRequest(req); - - // Choose or get the server url - const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); - - // If a server url is defined, we stop the session - if (serverUrl) - fetch(serverUrl + '/api/stop?session=' + sessionId); - }); - - app.get('/:formatType/:/transcode/universal/ping', async (req, res) => { - // Proxy to Plex - RoutesProxy.plex(req, res); - - // Extract sessionId from request parameter - const sessionId = SessionsManager.getSessionFromRequest(req); - - // Choose or get the server url - const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); - - // If a server url is defined, we ping the session - if (serverUrl) - fetch(serverUrl + '/api/ping?session=' + sessionId); - }); - - app.get('/:/timeline', async (req, res) => { - // Proxy to Plex - RoutesProxy.plex(req, res); - - // Extract sessionId from request parameter - const sessionId = SessionsManager.getSessionFromRequest(req); - - // Choose or get the server url - const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); - - // It's a stop request - if (req.query.state === 'stopped'/* || (req.query['X-Plex-Session-Identifier'] && SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier']))*/) { - // If a server url is defined, we stop the session - if (serverUrl) - fetch(serverUrl + '/api/stop?session=' + sessionId); - } - // it's a ping request - else if (serverUrl) { - fetch(serverUrl + '/api/ping?session=' + sessionId); - } - }); + app.get('/:formatType/:/transcode/universal/stop', RoutesTranscode.stop); + app.get('/:formatType/:/transcode/universal/ping', RoutesTranscode.ping); + app.get('/:/timeline', RoutesTranscode.timeline); // Download app.get('/library/parts/:id1/:id2/file.*', RoutesTranscode.redirect); diff --git a/src/routes/transcode.js b/src/routes/transcode.js index c5d3f3c..b22a772 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -1,15 +1,17 @@ import debug from 'debug'; import fetch from 'node-fetch'; +import RoutesTranscode from './transcode'; +import RoutesProxy from './proxy'; +import SessionsManager from '../core/sessions'; import SessionsManager from '../core/sessions'; -import ServersManager from '../core/servers'; -import Proxy from './proxy'; // Debugger const D = debug('UnicornLoadBalancer:transcode'); let RoutesTranscode = {}; +/* Route to send a 302 to another server */ RoutesTranscode.redirect = async (req, res) => { const session = SessionsManager.getSessionFromRequest(req); const server = await SessionsManager.chooseServer(session, req.headers['x-forwarded-for'] || req.connection.remoteAddress); @@ -25,99 +27,112 @@ RoutesTranscode.redirect = async (req, res) => { } }; +/* Route called when a DASH stream starts */ +RoutesTranscode.dashStart = (req, res) => { + // By default we don't have the session identifier + let sessionId = false; + // If we have a cached X-Plex-Session-Identifier, we use it + if (req.query['X-Plex-Session-Identifier'] && SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier'])) + sessionId = SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier']); -/* -RoutesTranscode.ping = (req, res) => { - SessionsManager.updateSessionFromRequest(req); + // Save session + SessionsManager.cacheSessionFromRequest(req); - const search = SessionsManager.parseSessionFromRequest(req); - const session = SessionsManager.getSessionFromRequest(search); + // If session id available + if (sessionId) + SessionsManager.cleanSession(sessionId); - Proxy.plex(req, res); + // Redirect + RoutesTranscode.redirect(req, res); +} - const pingRequest = (server) => { - fetch(server + '/api/ping?session=' + session.session + '&unicorn=' + session.unicorn); - }; +/* Routes called when a long polling stream starts */ +RoutesTranscode.lpStart = (req, res) => { + // Save session + SessionsManager.cacheSessionFromRequest(req); - if (session.serverUrl) { - return (pingRequest(session.serverUrl)); - } + // Get sessionId + const sessionId = SessionsManager.getSessionFromRequest(req); - ServersManager.chooseServer(req.headers['x-forwarded-for'] || req.connection.remoteAddress).then((server) => { - SessionsManager.updateSession({ ...session, serverUrl: server }); - return (pingRequest(server)); - }); -}; + // If sessionId is defined + if (sessionId) + SessionsManager.cleanSession(sessionId); -RoutesTranscode.timeline = (req, res) => { - SessionsManager.updateSessionFromRequest(req); + // Redirect + RoutesTranscode.redirect(req, res); +} - const search = SessionsManager.parseSessionFromRequest(req); - const session = SessionsManager.getSessionFromRequest(search); +/* Route called when a HLS stream starts */ +RoutesTranscode.hlsStart = (req, res) => { + // Proxy to Plex + RoutesProxy.plex(req, res); - Proxy.plex(req, res); + // Save session + SessionsManager.cacheSessionFromRequest(req); - const pingRequest = (server) => { - fetch(server + '/api/ping?session=' + session.session + '&unicorn=' + session.unicorn); - }; + // Get sessionId + const sessionId = SessionsManager.getSessionFromRequest(req); - const stopRequest = (server) => { - fetch(server + '/api/stop?session=' + session.session + '&unicorn=' + session.unicorn); - }; + // If sessionId is defined + if (sessionId) + SessionsManager.cleanSession(sessionId); +}; - const autoRequest = (server) => { - if (req.query.state == 'stopped') - stopRequest(server); - else - pingRequest(server); - } +/* Route ping */ +RoutesTranscode.ping = async (req, res) => { + // Proxy to Plex + RoutesProxy.plex(req, res); - if (session.serverUrl) { - return (autoRequest(session.serverUrl)); - } + // Extract sessionId from request parameter + const sessionId = SessionsManager.getSessionFromRequest(req); - ServersManager.chooseServer(req.headers['x-forwarded-for'] || req.connection.remoteAddress).then((server) => { - SessionsManager.updateSession({ ...session, serverUrl: server }); - return (autoRequest(server)); - }); -}; + // Choose or get the server url + const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); -RoutesTranscode.stop = (req, res) => { - SessionsManager.updateSessionFromRequest(req); + // If a server url is defined, we ping the session + if (serverUrl) + fetch(serverUrl + '/api/ping?session=' + sessionId); +}; - const search = SessionsManager.parseSessionFromRequest(req); - const session = SessionsManager.getSessionFromRequest(search); +/* Route timeline */ +RoutesTranscode.timeline = async (req, res) => { + // Proxy to Plex + RoutesProxy.plex(req, res); - Proxy.plex(req, res); + // Extract sessionId from request parameter + const sessionId = SessionsManager.getSessionFromRequest(req); - const stopRequest = (server) => { - fetch(server + '/api/stop?session=' + session.session + '&unicorn=' + session.unicorn); - }; + // Choose or get the server url + const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); - if (session.serverUrl) { - return (stopRequest(session.serverUrl)); + // It's a stop request + if (req.query.state === 'stopped'/* || (req.query['X-Plex-Session-Identifier'] && SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier']))*/) { + // If a server url is defined, we stop the session + if (serverUrl) + fetch(serverUrl + '/api/stop?session=' + sessionId); + } + // it's a ping request + else if (serverUrl) { + fetch(serverUrl + '/api/ping?session=' + sessionId); } - - ServersManager.chooseServer(req.headers['x-forwarded-for'] || req.connection.remoteAddress).then((server) => { - SessionsManager.updateSession({ ...session, serverUrl: server }); - return (stopRequest(server)); - }); }; -RoutesTranscode.cleanSession = (req, res, next) => { - if (typeof req.query.session !== 'undefined') { - D('Clean ' + req.query.session + 'from session store'); - SessionsManager.cleanSession(req.query.session) - .then(() => { - next(); - }) - .catch(() => { - res.status(500).send({ error: { code: 'INTERNAL_ERROR', message: 'Internal Server Error' } }); - }) - } +/* Route stop */ +RoutesTranscode.stop = async (req, res) => { + // Proxy to plex + RoutesProxy.plex(req, res); + + // Extract sessionId from request parameter + const sessionId = SessionsManager.getSessionFromRequest(req); + + // Choose or get the server url + const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); + + // If a server url is defined, we stop the session + if (serverUrl) + fetch(serverUrl + '/api/stop?session=' + sessionId); }; -*/ export default RoutesTranscode; From 15c2d6e8fcc1dbd206998730c4aaaa87476fe736 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Tue, 4 Dec 2018 00:02:07 +0100 Subject: [PATCH 069/132] Fix import --- src/routes/transcode.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index b22a772..996075b 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -4,7 +4,6 @@ import fetch from 'node-fetch'; import RoutesTranscode from './transcode'; import RoutesProxy from './proxy'; import SessionsManager from '../core/sessions'; -import SessionsManager from '../core/sessions'; // Debugger const D = debug('UnicornLoadBalancer:transcode'); From 1a44d177d1d8d4ecd903b3ba0763280e8c166bad Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Tue, 4 Dec 2018 00:03:00 +0100 Subject: [PATCH 070/132] Fix import --- src/routes/transcode.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index 996075b..1c1eb97 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -1,7 +1,6 @@ import debug from 'debug'; import fetch from 'node-fetch'; -import RoutesTranscode from './transcode'; import RoutesProxy from './proxy'; import SessionsManager from '../core/sessions'; From 22681301674dc5440f2f550e56f0a6ab8e332c17 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Tue, 4 Dec 2018 00:15:38 +0100 Subject: [PATCH 071/132] Logs --- src/core/sessions.js | 7 +++---- src/routes/transcode.js | 41 +++++++++++++++++++++++++++++++++-------- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 3350ab9..6a353ca 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -22,7 +22,7 @@ SessionsManager.chooseServer = async (session, ip = false) => { if (urls[session]) return (urls[session]); const url = await ServersManager.chooseServer(ip); - D('Choosed server for ' + session + ': ' + url); + D('Choose server for ' + session + ' [' + url + ']'); urls[session] = url; return (url); }; @@ -66,7 +66,7 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { return (false); // Debug - D('Session found: ' + sessionId + ' (' + sessionFull + ')'); + D('FFMPEG start for ' + sessionId + ' [' + sessionFull + ']'); // Parse arguments const parsedArgs = args.map((e) => { @@ -113,13 +113,12 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { // Store the FFMPEG parameters in RedisCache SessionsManager.storeFFmpegParameters = (args, env) => { const parsed = SessionsManager.parseFFmpegParameters(args, env); - D('FFMPEG callback for session ' + parsed.session); SessionStore.set(parsed.session, parsed).then(() => { }).catch(() => { }) return (parsed); }; SessionsManager.cleanSession = (sessionId) => { - D('Delete ' + sessionId); + D('Delete session ' + sessionId); return SessionStore.delete(sessionId) }; diff --git a/src/routes/transcode.js b/src/routes/transcode.js index 1c1eb97..d83296a 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -27,7 +27,6 @@ RoutesTranscode.redirect = async (req, res) => { /* Route called when a DASH stream starts */ RoutesTranscode.dashStart = (req, res) => { - // By default we don't have the session identifier let sessionId = false; @@ -35,6 +34,9 @@ RoutesTranscode.dashStart = (req, res) => { if (req.query['X-Plex-Session-Identifier'] && SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier'])) sessionId = SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier']); + // Log + D('Start stream ' + sessionId + ' [DASH]'); + // Save session SessionsManager.cacheSessionFromRequest(req); @@ -54,6 +56,9 @@ RoutesTranscode.lpStart = (req, res) => { // Get sessionId const sessionId = SessionsManager.getSessionFromRequest(req); + // Log + D('Start stream ' + sessionId + ' [LP]'); + // If sessionId is defined if (sessionId) SessionsManager.cleanSession(sessionId); @@ -73,6 +78,9 @@ RoutesTranscode.hlsStart = (req, res) => { // Get sessionId const sessionId = SessionsManager.getSessionFromRequest(req); + // Log + D('Start stream ' + sessionId + ' [HLS]'); + // If sessionId is defined if (sessionId) SessionsManager.cleanSession(sessionId); @@ -90,8 +98,12 @@ RoutesTranscode.ping = async (req, res) => { const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); // If a server url is defined, we ping the session - if (serverUrl) + if (serverUrl) { + D('Ping for ' + sessionId + ' [' + serverUrl + ']'); fetch(serverUrl + '/api/ping?session=' + sessionId); + } else { + D('Ping for ' + sessionId + ' [UNKNOWN]'); + } }; /* Route timeline */ @@ -106,14 +118,23 @@ RoutesTranscode.timeline = async (req, res) => { const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); // It's a stop request - if (req.query.state === 'stopped'/* || (req.query['X-Plex-Session-Identifier'] && SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier']))*/) { + if (req.query.state === 'stopped') { // If a server url is defined, we stop the session - if (serverUrl) + if (serverUrl) { + D('Stop for ' + sessionId + ' [' + serverUrl + ']'); fetch(serverUrl + '/api/stop?session=' + sessionId); + } else { + D('Stop for ' + sessionId + ' [UNKNOWN]'); + } } - // it's a ping request - else if (serverUrl) { - fetch(serverUrl + '/api/ping?session=' + sessionId); + // It's a ping request + else { + if (serverUrl) { + D('Ping for ' + sessionId + ' [' + serverUrl + ']'); + fetch(serverUrl + '/api/ping?session=' + sessionId); + } else { + D('Ping for ' + sessionId + ' [UNKNOWN]'); + } } }; @@ -129,8 +150,12 @@ RoutesTranscode.stop = async (req, res) => { const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); // If a server url is defined, we stop the session - if (serverUrl) + if (serverUrl) { + D('Stop for ' + sessionId + ' [' + serverUrl + ']'); fetch(serverUrl + '/api/stop?session=' + sessionId); + } else { + D('Stop for ' + sessionId + ' [UNKNOWN]'); + } }; export default RoutesTranscode; From 42f8a8e3c476a6b7057d6a5bc29beb90a5c41cd8 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Tue, 4 Dec 2018 00:20:45 +0100 Subject: [PATCH 072/132] Logs --- src/core/sessions.js | 6 +++--- src/routes/api.js | 1 - src/routes/transcode.js | 26 +++++++++++++------------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 6a353ca..9e2970b 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -22,7 +22,7 @@ SessionsManager.chooseServer = async (session, ip = false) => { if (urls[session]) return (urls[session]); const url = await ServersManager.chooseServer(ip); - D('Choose server for ' + session + ' [' + url + ']'); + D('SERVER ' + session + ' [' + url + ']'); urls[session] = url; return (url); }; @@ -66,7 +66,7 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { return (false); // Debug - D('FFMPEG start for ' + sessionId + ' [' + sessionFull + ']'); + D('FFMPEG ' + sessionId + ' [' + sessionFull + ']'); // Parse arguments const parsedArgs = args.map((e) => { @@ -118,7 +118,7 @@ SessionsManager.storeFFmpegParameters = (args, env) => { }; SessionsManager.cleanSession = (sessionId) => { - D('Delete session ' + sessionId); + D('DELETE ' + sessionId); return SessionStore.delete(sessionId) }; diff --git a/src/routes/api.js b/src/routes/api.js index 8b3bef1..62daf8f 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -56,7 +56,6 @@ RoutesAPI.plex = (req, res) => { } }).on('error', (err) => { if (err.code === 'HPE_UNEXPECTED_CONTENT_LENGTH') { - D('Ignore /progress error'); return (res.status(200).send()); } res.status(400).send({ error: { code: 'PROXY_TIMEOUT', message: 'Plex not respond in time, proxy request fails' } }); diff --git a/src/routes/transcode.js b/src/routes/transcode.js index d83296a..1e61773 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -18,10 +18,10 @@ RoutesTranscode.redirect = async (req, res) => { 'Location': server + req.url }); res.end(); - D('Send 302 for ' + session + ' to ' + server); + D('REDIRECT ' + session + ' [' + server + ']'); } else { res.status(500).send({ error: { code: 'SERVER_UNAVAILABLE', message: 'SERVER_UNAVAILABLE' } }); - D('Fail to 302 for ' + session); + D('REDIRECT ' + session + ' [UNKNOWN]'); } }; @@ -35,7 +35,7 @@ RoutesTranscode.dashStart = (req, res) => { sessionId = SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier']); // Log - D('Start stream ' + sessionId + ' [DASH]'); + D('START ' + sessionId + ' [DASH]'); // Save session SessionsManager.cacheSessionFromRequest(req); @@ -57,7 +57,7 @@ RoutesTranscode.lpStart = (req, res) => { const sessionId = SessionsManager.getSessionFromRequest(req); // Log - D('Start stream ' + sessionId + ' [LP]'); + D('START ' + sessionId + ' [LP]'); // If sessionId is defined if (sessionId) @@ -79,7 +79,7 @@ RoutesTranscode.hlsStart = (req, res) => { const sessionId = SessionsManager.getSessionFromRequest(req); // Log - D('Start stream ' + sessionId + ' [HLS]'); + D('START ' + sessionId + ' [HLS]'); // If sessionId is defined if (sessionId) @@ -99,10 +99,10 @@ RoutesTranscode.ping = async (req, res) => { // If a server url is defined, we ping the session if (serverUrl) { - D('Ping for ' + sessionId + ' [' + serverUrl + ']'); + D('PING ' + sessionId + ' [' + serverUrl + ']'); fetch(serverUrl + '/api/ping?session=' + sessionId); } else { - D('Ping for ' + sessionId + ' [UNKNOWN]'); + D('PING ' + sessionId + ' [UNKNOWN]'); } }; @@ -121,19 +121,19 @@ RoutesTranscode.timeline = async (req, res) => { if (req.query.state === 'stopped') { // If a server url is defined, we stop the session if (serverUrl) { - D('Stop for ' + sessionId + ' [' + serverUrl + ']'); + D('STOP ' + sessionId + ' [' + serverUrl + ']'); fetch(serverUrl + '/api/stop?session=' + sessionId); } else { - D('Stop for ' + sessionId + ' [UNKNOWN]'); + D('STOP ' + sessionId + ' [UNKNOWN]'); } } // It's a ping request else { if (serverUrl) { - D('Ping for ' + sessionId + ' [' + serverUrl + ']'); + D('PING ' + sessionId + ' [' + serverUrl + ']'); fetch(serverUrl + '/api/ping?session=' + sessionId); } else { - D('Ping for ' + sessionId + ' [UNKNOWN]'); + D('PING ' + sessionId + ' [UNKNOWN]'); } } }; @@ -151,10 +151,10 @@ RoutesTranscode.stop = async (req, res) => { // If a server url is defined, we stop the session if (serverUrl) { - D('Stop for ' + sessionId + ' [' + serverUrl + ']'); + D('STOP ' + sessionId + ' [' + serverUrl + ']'); fetch(serverUrl + '/api/stop?session=' + sessionId); } else { - D('Stop for ' + sessionId + ' [UNKNOWN]'); + D('STOP ' + sessionId + ' [UNKNOWN]'); } }; From 15404a8747d6758735bdb7769ea7f5df082bb1da Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Tue, 4 Dec 2018 00:23:01 +0100 Subject: [PATCH 073/132] Logs --- src/core/sessions.js | 2 +- src/routes/api.js | 2 +- src/routes/transcode.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 9e2970b..9b79fad 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -6,7 +6,7 @@ import SessionStore from '../store'; import ServersManager from './servers'; // Debugger -const D = debug('UnicornLoadBalancer:SessionsManager'); +const D = debug('UnicornLoadBalancer'); let SessionsManager = {}; diff --git a/src/routes/api.js b/src/routes/api.js index 62daf8f..2bdf754 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -8,7 +8,7 @@ import SessionsManager from '../core/sessions'; import ServersManager from '../core/servers'; // Debugger -const D = debug('UnicornLoadBalancer:api'); +const D = debug('UnicornLoadBalancer'); let RoutesAPI = {}; diff --git a/src/routes/transcode.js b/src/routes/transcode.js index 1e61773..dc5e94f 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -5,7 +5,7 @@ import RoutesProxy from './proxy'; import SessionsManager from '../core/sessions'; // Debugger -const D = debug('UnicornLoadBalancer:transcode'); +const D = debug('UnicornLoadBalancer'); let RoutesTranscode = {}; From fc8e03a88c28ddb694165523a3a24c50fbb58252 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Tue, 4 Dec 2018 00:23:35 +0100 Subject: [PATCH 074/132] Logs --- src/store/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/store/index.js b/src/store/index.js index 2581d53..b3cad86 100644 --- a/src/store/index.js +++ b/src/store/index.js @@ -4,7 +4,7 @@ import LocalSessionStore from './local'; import debug from 'debug'; // Debugger -const D = debug('UnicornLoadBalancer:SessionStore'); +const D = debug('UnicornLoadBalancer'); let SessionStore; From 6e8238dc6b813a41bb40eb15c04aedaf6fff2c8b Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Tue, 4 Dec 2018 00:35:27 +0100 Subject: [PATCH 075/132] Log and fixes --- src/config.js | 2 +- src/core/sessions.js | 2 -- src/routes/transcode.js | 2 +- src/utils.js | 4 ++-- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/config.js b/src/config.js index 5517b7f..90096a4 100644 --- a/src/config.js +++ b/src/config.js @@ -6,7 +6,7 @@ export default { version: '2.0.0', server: { port: env.int('SERVER_PORT', 3001), - public_port: env.int('SERVER_PUBLIC_PORT', 443), + public: env.int('SERVER_PUBLIC', 443), host: env.string('SERVER_HOST', '127.0.0.1'), ssl: env.bool('SERVER_SSL', false) }, diff --git a/src/core/sessions.js b/src/core/sessions.js index 9b79fad..bd0bba5 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -17,8 +17,6 @@ let cache = {}; let urls = {} SessionsManager.chooseServer = async (session, ip = false) => { - if (session === false) - return (false); if (urls[session]) return (urls[session]); const url = await ServersManager.chooseServer(ip); diff --git a/src/routes/transcode.js b/src/routes/transcode.js index dc5e94f..9ff422b 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -35,7 +35,7 @@ RoutesTranscode.dashStart = (req, res) => { sessionId = SessionsManager.getCacheSession(req.query['X-Plex-Session-Identifier']); // Log - D('START ' + sessionId + ' [DASH]'); + D('START ' + SessionsManager.getSessionFromRequest(req) + ' [DASH]'); // Save session SessionsManager.cacheSessionFromRequest(req); diff --git a/src/utils.js b/src/utils.js index ddd3317..4ea5030 100644 --- a/src/utils.js +++ b/src/utils.js @@ -3,11 +3,11 @@ import redisClient from 'redis'; import config from './config'; export const publicUrl = () => { - return ('http' + ((config.server.ssl) ? 's' : '') + '://' + config.server.host + (([80, 443].indexOf(config.server.public_port) === -1) ? ':' + config.server.public_port : '') + '/') + return ('http' + ((config.server.ssl) ? 's' : '') + '://' + config.server.host + (([80, 443].indexOf(config.server.public) === -1) ? ':' + config.server.public : '') + '/') }; export const internalUrl = () => { - return ('http' + ((config.server.ssl) ? 's' : '') + '://' + config.server.host + ':' + config.server.port + '/') + return ('http://127.0.0.1:' + config.server.port + '/') }; export const plexUrl = () => { From 3c9ca2a633e26d8770d1559a09ec20fdc2d98f07 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Wed, 16 Jan 2019 21:40:50 +0100 Subject: [PATCH 076/132] Custom Image Resizer to improve Plex resize performances (and support WebP) --- package.json | 6 ++- src/core/images.js | 107 +++++++++++++++++++++++++++++++++++++++++++ src/routes/index.js | 4 ++ src/routes/resize.js | 79 +++++++++++++++++++------------- 4 files changed, 161 insertions(+), 35 deletions(-) create mode 100644 src/core/images.js diff --git a/package.json b/package.json index 7a47a8c..fd5a9af 100644 --- a/package.json +++ b/package.json @@ -4,14 +4,15 @@ "description": "", "main": "index.js", "scripts": { - "start": "node index.js", - "test": "echo \"Error: no test specified\" && exit 1" + "start": "node index.js" }, "author": "Maxime Baconnais", "license": "MIT", "dependencies": { + "color": "^3.1.0", "cors": "^2.8.4", "debug": "^4.0.1", + "detect-browser": "^4.0.3", "esm": "^3.0.84", "express": "^4.16.3", "getenv": "^0.7.0", @@ -19,6 +20,7 @@ "node-fetch": "^2.2.1", "node-pre-gyp": "^0.11.0", "redis": "^2.8.0", + "sharp": "^0.21.2", "sqlite3": "^4.0.2", "uniqid": "^5.0.3" } diff --git a/src/core/images.js b/src/core/images.js new file mode 100644 index 0000000..520b4c1 --- /dev/null +++ b/src/core/images.js @@ -0,0 +1,107 @@ +import fetch from 'node-fetch'; +import sharp from 'sharp'; +import color from 'color'; + +export default (link, parameters, needAlpha = false) => { + return new Promise(async (resolve, reject) => { + try { + const params = { + + // Width of the image (px value) + width: false, + + // Height of the image (px value) + height: false, + + // Background color + background: false, + + // Background opacity + opacity: false, + + // Resize constraint (0:height / 1:width) + minSize: 0, + + // Blur on picture (between 0 and 10000) + blur: 0, + + // Output format + format: false, // png / jpg / webp + + // Force upscale + upscale: false, + + // User parameters + ...parameters + } + + if (!params.width || !params.height) + return reject('Size not provided'); + + // Get image content + const body = await fetch(link).then(res => res.buffer()); + + // Load body + let s = sharp(body); + + // Resize parameters + const opt = { + ...((params.upscale) ? { withoutEnlargement: !!params.upscale } : {}) + } + + // Resize based on width + if (params.minSize === 1) + s.resize(params.width, null, opt); + else + s.resize(null, params.height, opt); + + // Background & opacity support + if (params.background && params.opacity) { + const buff = await s.png().toBuffer(); + s = sharp(buff); + const meta = await s.metadata(); + const bgd = await sharp({ + create: { + width: meta.width, + height: meta.height, + channels: 4, + background: { + r: color(`#${params.background}`).r, + g: color(`#${params.background}`).g, + b: color(`#${params.background}`).b, + alpha: ((100 - params.opacity) / 100) + } + } + }).png().toBuffer(); + s.overlayWith(bgd); + } + + // Blur + if (params.blur > 0 && params.blur <= 1000) + s.blur(params.blur * 1.25).gamma(2); + + // Output format + if (params.format === 'jpg') + s.jpeg({ + quality: 70 + }) + else if (params.format === 'png') + s.png({ + quality: 70, + progressive: true, + compressionLevel: 9 + }) + else if (params.format === 'webp') + s.webp({ + quality: 70, + ...((needAlpha) ? {} : { alphaQuality: 0 }) + }) + + // Return stream + resolve(s); + } + catch (err) { + reject(err); + } + }); +} \ No newline at end of file diff --git a/src/routes/index.js b/src/routes/index.js index b299a4f..aaa739a 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -4,6 +4,7 @@ import config from '../config'; import RoutesAPI from './api'; import RoutesTranscode from './transcode'; import RoutesProxy from './proxy'; +import RoutesResize from './resize'; export default (app) => { @@ -43,6 +44,9 @@ export default (app) => { // Download app.get('/library/parts/:id1/:id2/file.*', RoutesTranscode.redirect); + // Image Resizer + app.get('/photo/:/transcode', RoutesResize.resize); + // Forward other to Plex app.all('*', RoutesProxy.plex); }; diff --git a/src/routes/resize.js b/src/routes/resize.js index bd3858b..224ef87 100644 --- a/src/routes/resize.js +++ b/src/routes/resize.js @@ -1,48 +1,61 @@ -import sharp from 'sharp'; -import fetch from 'node-fetch'; -import { PassThrough } from 'stream'; -import { plexUrl } from '../utils'; - -let RoutesResize = {}; - -const resizeFromUrl = (url, width = false, height = false, minSize = 0, format = 'jpeg') => { - let pass = new PassThrough(); +import debug from 'debug'; +import { parseUserAgent } from 'detect-browser'; - fetch(url).then(res => { - res.body.pipe(pass); - }); - - let transform = sharp(); - - - if (width || height) { - transform = transform.resize(((minSize > 0) ? width : undefined), ((minSize <= 0) ? height : undefined)); - } +import { plexUrl } from '../utils'; +import optimizeImage from '../core/images'; - if (format) { - transform = transform.toFormat(sharp.format.webp); // jpeg | webp | png - } +// Debugger +const D = debug('UnicornLoadBalancer'); - return pass.pipe(transform); -} +let RoutesResize = {}; RoutesResize.resize = (req, res) => { + + // Parse url let url = req.query.url || false; - if (url && url[0] == '/') - url = plexUrl() + url; + if (url && url[0] === '/') + url = plexUrl() + url.substring(1); + // Extract parameters const params = { - width: parseInt(req.query.width) || false, - height: parseInt(req.query.height) || false, - minSize: parseInt(req.query.minSize) || 0, - url + ...((req.query.width) ? { width: parseInt(req.query.width) } : {}), + ...((req.query.height) ? { height: parseInt(req.query.height) } : {}), + ...((req.query.background) ? { background: req.query.background } : {}), + ...((req.query.opacity) ? { opacity: parseInt(req.query.opacity) } : {}), + ...((req.query.minSize) ? { minSize: parseInt(req.query.minSize) } : {}), + ...((req.query.blur) ? { blur: parseInt(req.query.blur) } : {}), + ...((req.query.format && (req.query.format === 'webp' || req.query.format === 'png')) ? { format: req.query.format } : { format: 'jpg' }), + ...((req.query.upscale) ? { upscale: parseInt(req.query.upscale) } : {}), }; - if (!params.width || !params.height || !params.url) + // Check size + if (!params.width || !params.height || !url) return (res.status(400).send({ error: { code: 'RESIZE_ERROR', message: 'Invalid parameters, resize request fails' } })); - res.type(`image/webp`); - resizeFromUrl(params.url, params.width, params.height, params.minSize).pipe(res) + // Auto select WebP if user-agent support it + const browser = parseUserAgent(req.get('User-Agent')); + const needAlpha = params.format === 'png'; + if (browser.name === 'chrome') { + params.format = 'webp'; + } + + // Debug + D('IMAGE ' + url + ' [' + params.format + ']'); + + // Mime type + if (params.format === 'webp') + res.type(`image/webp`); + else if (params.format === 'png') + res.type(`image/png`); + else + res.type(`image/jpeg`); + + // Process image + optimizeImage(url, params, needAlpha).then((stream) => { + return stream.pipe(res); + }).catch(err => { + return (res.status(400).send({ error: { code: 'RESIZE_ERROR', message: 'Invalid parameters, resize request fails' } })); + }) }; export default RoutesResize; \ No newline at end of file From d3114300114591ed2e723f4f1cf2a097ff5d00b3 Mon Sep 17 00:00:00 2001 From: Maxime BACONNAIS Date: Mon, 28 Jan 2019 13:00:43 +0100 Subject: [PATCH 077/132] Fix IP detection with cloudflare and multi-proxy --- src/routes/transcode.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index 9ff422b..c3c4c4e 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -9,10 +9,19 @@ const D = debug('UnicornLoadBalancer'); let RoutesTranscode = {}; +/* Extract IP */ +const getIp = (req) => { + if (req.get('CF-Connecting-IP')) + return req.get('CF-Connecting-IP'); + if (req.get('x-forwarded-for')) + return req.get('x-forwarded-for').split(',')[0]; + return req.connection.remoteAddress +}; + /* Route to send a 302 to another server */ RoutesTranscode.redirect = async (req, res) => { const session = SessionsManager.getSessionFromRequest(req); - const server = await SessionsManager.chooseServer(session, req.headers['x-forwarded-for'] || req.connection.remoteAddress); + const server = await SessionsManager.chooseServer(session, getIp(req)); if (server) { res.writeHead(302, { 'Location': server + req.url @@ -95,7 +104,7 @@ RoutesTranscode.ping = async (req, res) => { const sessionId = SessionsManager.getSessionFromRequest(req); // Choose or get the server url - const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); + const serverUrl = await SessionsManager.chooseServer(sessionId, getIp(req)); // If a server url is defined, we ping the session if (serverUrl) { @@ -115,7 +124,7 @@ RoutesTranscode.timeline = async (req, res) => { const sessionId = SessionsManager.getSessionFromRequest(req); // Choose or get the server url - const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); + const serverUrl = await SessionsManager.chooseServer(sessionId, getIp(req)); // It's a stop request if (req.query.state === 'stopped') { @@ -147,7 +156,7 @@ RoutesTranscode.stop = async (req, res) => { const sessionId = SessionsManager.getSessionFromRequest(req); // Choose or get the server url - const serverUrl = await SessionsManager.chooseServer(sessionId, req.headers['x-forwarded-for'] || req.connection.remoteAddress); + const serverUrl = await SessionsManager.chooseServer(sessionId, getIp(req)); // If a server url is defined, we stop the session if (serverUrl) { From 96c264e6dba898c33c33388d7651797ae4cafd88 Mon Sep 17 00:00:00 2001 From: Maxime BACONNAIS Date: Tue, 5 Feb 2019 14:42:13 +0100 Subject: [PATCH 078/132] Ignore websocket proxy timeout --- src/routes/proxy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/proxy.js b/src/routes/proxy.js index be3f590..624d675 100644 --- a/src/routes/proxy.js +++ b/src/routes/proxy.js @@ -27,7 +27,7 @@ RoutesProxy.ws = (req, res) => { port: config.plex.port } }).on('error', () => { - res.status(400).send({ error: { code: 'PROXY_TIMEOUT', message: 'Plex not respond in time, proxy request fails' } }); + // Fail silently }); return (proxy.ws(req, res)); }; From 31a5d5a392bd22c695063a57686b55de64c22ebd Mon Sep 17 00:00:00 2001 From: Maxime BACONNAIS Date: Sun, 10 Feb 2019 18:21:04 +0100 Subject: [PATCH 079/132] Fix sort --- src/core/servers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/servers.js b/src/core/servers.js index edf732a..73a4717 100644 --- a/src/core/servers.js +++ b/src/core/servers.js @@ -54,7 +54,7 @@ ServersManager.chooseServer = (ip = false) => { Object.keys(list).forEach((i) => { tab.push(list[i]); }); - tab.sort((e) => (e.score)); + tab.sort((a, b) => (a.score - b.score)); if (typeof (tab[0]) === 'undefined') return resolve(false); fetch(tab[0].url + '/api/resolve?ip=' + ip) From e44d1314b9db022f07df8d9c8878d3d77e911eca Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Sun, 10 Feb 2019 19:47:37 +0100 Subject: [PATCH 080/132] Send session to transcoder --- src/core/servers.js | 4 ++-- src/core/sessions.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/servers.js b/src/core/servers.js index 73a4717..b114cc2 100644 --- a/src/core/servers.js +++ b/src/core/servers.js @@ -47,7 +47,7 @@ ServersManager.list = () => { } // Chose best server -ServersManager.chooseServer = (ip = false) => { +ServersManager.chooseServer = (session, ip = false) => { return (new Promise((resolve, reject) => { let tab = []; const list = ServersManager.list(); @@ -57,7 +57,7 @@ ServersManager.chooseServer = (ip = false) => { tab.sort((a, b) => (a.score - b.score)); if (typeof (tab[0]) === 'undefined') return resolve(false); - fetch(tab[0].url + '/api/resolve?ip=' + ip) + fetch(`${tab[0].url}/api/resolve?session=${session}&ip=${ip}`) .then(res => res.json()) .then(body => { return resolve(body.client) diff --git a/src/core/sessions.js b/src/core/sessions.js index bd0bba5..ab9ad12 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -19,7 +19,7 @@ let urls = {} SessionsManager.chooseServer = async (session, ip = false) => { if (urls[session]) return (urls[session]); - const url = await ServersManager.chooseServer(ip); + const url = await ServersManager.chooseServer(session, ip); D('SERVER ' + session + ' [' + url + ']'); urls[session] = url; return (url); From 723c15d71a87234c00c413117ff5b1b2c2fe0f3c Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Sun, 10 Feb 2019 19:55:42 +0100 Subject: [PATCH 081/132] Allow download forward and image proxy configuration --- src/config.js | 8 ++++++++ src/routes/index.js | 12 ++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/config.js b/src/config.js index 90096a4..d10f1ab 100644 --- a/src/config.js +++ b/src/config.js @@ -27,5 +27,13 @@ export default { }, scores: { timeout: env.int('SCORES_TIMEOUT', 10) + }, + custom: { + image: { + resizer: env.bool('CUSTOM_IMAGE_RESIZER', true) + }, + download: { + forward: env.bool('CUSTOM_DOWNLOAD_FORWARD', true) + } } }; \ No newline at end of file diff --git a/src/routes/index.js b/src/routes/index.js index aaa739a..f4e149d 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -42,11 +42,15 @@ export default (app) => { app.get('/:/timeline', RoutesTranscode.timeline); // Download - app.get('/library/parts/:id1/:id2/file.*', RoutesTranscode.redirect); + if (config.custom.download.forward) { + app.get('/library/parts/:id1/:id2/file.*', RoutesTranscode.redirect); + } + + // Image Resizer + if (config.custom.image.resizer) { + app.get('/photo/:/transcode', RoutesResize.resize); + } - // Image Resizer - app.get('/photo/:/transcode', RoutesResize.resize); - // Forward other to Plex app.all('*', RoutesProxy.plex); }; From 856e35687d7e79f5ebcfd347909b7be28a5dc6de Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Sun, 10 Feb 2019 20:44:29 +0100 Subject: [PATCH 082/132] Change SERVER_PUBLIC value and send it to /resolve url --- src/config.js | 2 +- src/core/servers.js | 5 +++-- src/utils.js | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/config.js b/src/config.js index d10f1ab..3fde3d3 100644 --- a/src/config.js +++ b/src/config.js @@ -6,7 +6,7 @@ export default { version: '2.0.0', server: { port: env.int('SERVER_PORT', 3001), - public: env.int('SERVER_PUBLIC', 443), + public: env.string('SERVER_PUBLIC', 'http://127.0.0.1:3001/'), host: env.string('SERVER_HOST', '127.0.0.1'), ssl: env.bool('SERVER_SSL', false) }, diff --git a/src/core/servers.js b/src/core/servers.js index b114cc2..d61ad52 100644 --- a/src/core/servers.js +++ b/src/core/servers.js @@ -1,5 +1,5 @@ import fetch from 'node-fetch'; -import { time } from '../utils'; +import { time, publicUrl } from '../utils'; import config from '../config'; let servers = {}; @@ -57,7 +57,8 @@ ServersManager.chooseServer = (session, ip = false) => { tab.sort((a, b) => (a.score - b.score)); if (typeof (tab[0]) === 'undefined') return resolve(false); - fetch(`${tab[0].url}/api/resolve?session=${session}&ip=${ip}`) + const origin = encodeURIComponent(publicUrl()) + fetch(`${tab[0].url}/api/resolve?session=${session}&ip=${ip}&origin=${origin}`) .then(res => res.json()) .then(body => { return resolve(body.client) diff --git a/src/utils.js b/src/utils.js index 4ea5030..1f39ad0 100644 --- a/src/utils.js +++ b/src/utils.js @@ -3,7 +3,7 @@ import redisClient from 'redis'; import config from './config'; export const publicUrl = () => { - return ('http' + ((config.server.ssl) ? 's' : '') + '://' + config.server.host + (([80, 443].indexOf(config.server.public) === -1) ? ':' + config.server.public : '') + '/') + return (config.server.public) }; export const internalUrl = () => { From 95c69aca2688b04e97d3b26fd75f373a9e4d6752 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Sun, 10 Feb 2019 21:03:09 +0100 Subject: [PATCH 083/132] Fix 302 redirect --- src/routes/transcode.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index c3c4c4e..cb6cf0a 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -23,10 +23,7 @@ RoutesTranscode.redirect = async (req, res) => { const session = SessionsManager.getSessionFromRequest(req); const server = await SessionsManager.chooseServer(session, getIp(req)); if (server) { - res.writeHead(302, { - 'Location': server + req.url - }); - res.end(); + res.redirect(302, server + req.url); D('REDIRECT ' + session + ' [' + server + ']'); } else { res.status(500).send({ error: { code: 'SERVER_UNAVAILABLE', message: 'SERVER_UNAVAILABLE' } }); From 8cb76c8c1420eb7337c0305c1d9d954b9f5db0b8 Mon Sep 17 00:00:00 2001 From: Maxime BACONNAIS Date: Tue, 12 Feb 2019 10:12:54 +0100 Subject: [PATCH 084/132] Change bool to boolish and add default servers list --- src/config.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/config.js b/src/config.js index 3fde3d3..19b2f9a 100644 --- a/src/config.js +++ b/src/config.js @@ -8,7 +8,7 @@ export default { port: env.int('SERVER_PORT', 3001), public: env.string('SERVER_PUBLIC', 'http://127.0.0.1:3001/'), host: env.string('SERVER_HOST', '127.0.0.1'), - ssl: env.bool('SERVER_SSL', false) + ssl: env.boolish('SERVER_SSL', false) }, plex: { host: env.string('PLEX_HOST', '127.0.0.1'), @@ -30,10 +30,13 @@ export default { }, custom: { image: { - resizer: env.bool('CUSTOM_IMAGE_RESIZER', true) + resizer: env.boolish('CUSTOM_IMAGE_RESIZER', true) }, download: { - forward: env.bool('CUSTOM_DOWNLOAD_FORWARD', true) + forward: env.boolish('CUSTOM_DOWNLOAD_FORWARD', true) + }, + servers: { + list: env.array('CUSTOM_SERVERS_LIST', 'string', []) } } -}; \ No newline at end of file +}; From 02f2778230e7889f0b0321630180954e30626efc Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Tue, 12 Feb 2019 19:40:57 +0100 Subject: [PATCH 085/132] Update README, support default servers and changes in configuration file --- README.md | 6 +++++- src/app.js | 15 +++++++++++++++ src/config.js | 6 +++--- src/core/servers.js | 2 +- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7743bc5..7c40acc 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer | **SERVER_HOST** | Host to access to the *UnicornLoadBalancer* | `string` | `127.0.0.1` | | **SERVER_PORT** | Port used by the *UnicornLoadBalancer* | `int` | `3001` | | **SERVER_SSL** | If HTTPS is enabled or not on the *UnicornLoadBalancer* | `bool` | `false` | +| **SERVER_PUBLIC** | Public url where the *UnicornLoadBalancer* can be called, with a slash at the end | `string` | `http://127.0.0.1:3001/` | | **PLEX_HOST** | Host to access to Plex | `string` | `127.0.0.1` | | **PLEX_PORT** | Port used by Plex | `int` | `32400` | | **PLEX_PATH_USR** | The Plex's path | `string` | `/usr/lib/plexmediaserver/` | @@ -36,7 +37,10 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer | **REDIS_PORT** | Port used by Redis | `int` | `6379` | | **REDIS_PASSWORD** | The password of the redis database | `string` | ` ` | | **REDIS_DB** | The index of the redis database | `int` | `0` | -| **SCORES_TIMEOUT** | Seconds to consider a not-pinged server as unavailable | `int` | `10` | +| **CUSTOM_SCORES_TIMEOUT** | Seconds to consider a not-pinged server as unavailable | `int` | `10` | +| **CUSTOM_IMAGE_RESIZER** | Enable or disable the custom image resizer (most efficient than Plex one) | `bool` | `true` | +| **CUSTOM_DOWNLOAD_FORWARD** | Enable or disable 302 for download links and direct play | `bool` | `true` | +| **CUSTOM_SERVERS_LIST** | Servers set by default | `string array` | `[]` | * Configure Plex Media Server access address * In Settings -> Server -> Network diff --git a/src/app.js b/src/app.js index 39ebf36..b4a76a8 100644 --- a/src/app.js +++ b/src/app.js @@ -6,6 +6,7 @@ import config from './config'; import Router from './routes'; import Proxy from './routes/proxy'; import { internalUrl } from './utils'; +import ServersManager from './core/servers'; import debug from 'debug'; @@ -38,6 +39,20 @@ D('Initializing API routes...'); // Routes Router(app); +// Load servers available in configuration +((Array.isArray(config.custom.servers.list)) ? config.custom.servers.list : []).map(e => ({ + name: e, + url: e, + sessions: [], + settings: { + maxSessions: 0, + maxDownloads: 0, + maxTranscodes: 0 + } +})).forEach(e => { + ServersManager.update(e); +}); + // Create HTTP server const httpServer = app.listen(config.server.port); diff --git a/src/config.js b/src/config.js index 19b2f9a..882d79b 100644 --- a/src/config.js +++ b/src/config.js @@ -25,10 +25,10 @@ export default { password: env.string('REDIS_PASSWORD', ''), db: env.int('REDIS_DB', 0) }, - scores: { - timeout: env.int('SCORES_TIMEOUT', 10) - }, custom: { + scores: { + timeout: env.int('CUSTOM_SCORES_TIMEOUT', 10) + }, image: { resizer: env.boolish('CUSTOM_IMAGE_RESIZER', true) }, diff --git a/src/core/servers.js b/src/core/servers.js index d61ad52..1c702c5 100644 --- a/src/core/servers.js +++ b/src/core/servers.js @@ -69,7 +69,7 @@ ServersManager.chooseServer = (session, ip = false) => { // Calculate server score ServersManager.score = (e) => { // The configuration wasn't updated since X seconds, the server is probably unavailable - if (time() - e.time > config.scores.timeout) + if (time() - e.time > config.custom.scores.timeout) return (100); // Default load 0 From b9cb6dabdd75bc73310cfa16d6a2b745e32601b5 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Tue, 12 Feb 2019 19:45:04 +0100 Subject: [PATCH 086/132] README update --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7c40acc..bcb7311 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer | **SERVER_HOST** | Host to access to the *UnicornLoadBalancer* | `string` | `127.0.0.1` | | **SERVER_PORT** | Port used by the *UnicornLoadBalancer* | `int` | `3001` | | **SERVER_SSL** | If HTTPS is enabled or not on the *UnicornLoadBalancer* | `bool` | `false` | -| **SERVER_PUBLIC** | Public url where the *UnicornLoadBalancer* can be called, with a slash at the end | `string` | `http://127.0.0.1:3001/` | +| **SERVER_PUBLIC** | Public url where the *UnicornLoadBalancer* can be called, **with** a slash at the end | `string` | `http://127.0.0.1:3001/` | | **PLEX_HOST** | Host to access to Plex | `string` | `127.0.0.1` | | **PLEX_PORT** | Port used by Plex | `int` | `32400` | | **PLEX_PATH_USR** | The Plex's path | `string` | `/usr/lib/plexmediaserver/` | @@ -40,7 +40,7 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer | **CUSTOM_SCORES_TIMEOUT** | Seconds to consider a not-pinged server as unavailable | `int` | `10` | | **CUSTOM_IMAGE_RESIZER** | Enable or disable the custom image resizer (most efficient than Plex one) | `bool` | `true` | | **CUSTOM_DOWNLOAD_FORWARD** | Enable or disable 302 for download links and direct play | `bool` | `true` | -| **CUSTOM_SERVERS_LIST** | Servers set by default | `string array` | `[]` | +| **CUSTOM_SERVERS_LIST** | Transcoder servers set by default, **without** a slash at the end, separate servers with a **comma** | `string array` | `[]` | * Configure Plex Media Server access address * In Settings -> Server -> Network From 49c9df7b7f1c4af1942aea0d0f7b6c72b2defeb0 Mon Sep 17 00:00:00 2001 From: Maxime BACONNAIS Date: Tue, 12 Feb 2019 19:51:59 +0100 Subject: [PATCH 087/132] Changes in notes --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bcb7311..69497c5 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer ### 2. Notes -All the requests to this Plex Media Server should pass through the *UnicornLoadBalancer*, if someone reach the server directly he will not be able to start a stream, since FFMPEG binary has been replaced. It is recomended to setup a nginx as reverse proxy in front to setup a SSL certificate and to have an iptable to direct access to the users on port **32400**. +All requests to the Plex Media Server should pass through the *UnicornLoadBalancer*, if someone reach the server directly he will not be able to start a stream, since FFMPEG binary has been replaced. To solve this problem it is recomended to configure an iptable to drop direct access on port **32400**. +It is also recomended to setup a nginx reverse proxy in front of the *UnicornLoadBalancer* to setup a SSL certificate. ``` #Example iptable From 820503fba40871f813327929e43f5d81e61ad941 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 21 Feb 2019 18:02:21 +0100 Subject: [PATCH 088/132] Rework Image proxy, rename arguments, bugfixes --- README.md | 12 +++++++++-- package.json | 1 + src/config.js | 18 ++++++++++++++--- src/core/images.js | 37 +++++++++++++++++++++++++++++++--- src/database/index.js | 19 ++++++++++++++++++ src/database/postgresql.js | 9 +++++++++ src/database/sqlite.js | 22 ++++++++++++++++++++ src/routes/api.js | 18 +++++------------ src/routes/index.js | 7 +++++-- src/routes/resize.js | 41 ++++++++++++-------------------------- 10 files changed, 133 insertions(+), 51 deletions(-) create mode 100644 src/database/index.js create mode 100644 src/database/postgresql.js create mode 100644 src/database/sqlite.js diff --git a/README.md b/README.md index bcb7311..93a95bd 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer * Plex Media Server * NodeJS * RedisCache (Optionnal) +* Postgresql (Optionnal) ## Setup @@ -32,13 +33,20 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer | **PLEX_PORT** | Port used by Plex | `int` | `32400` | | **PLEX_PATH_USR** | The Plex's path | `string` | `/usr/lib/plexmediaserver/` | | **PLEX_PATH_SESSIONS** | The path where Plex store sessions (to grab external subtitles) | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Cache/Transcode/Sessions` | -| **PLEX_PATH_DATABASE** | The path of the Plex database | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db` | +| **DATABASE_MODE** | Kind of database to use with Plex, can be `sqlite` or `postgresql` | `string` | `sqlite` | +| **DATABASE_SQLITE_PATH** | The path of the Plex d +atabase | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db` | +| **DATABASE_POSTGRESQL_HOST** | Host of the postgresql server | `string` | ` ` | +| **DATABASE_POSTGRESQL_DATABASE** | Name of the postgresql database | `string` | ` ` | +| **DATABASE_POSTGRESQL_USER** | User to use to the Postgresql database| `string` | ` ` | +| **DATABASE_POSTGRESQL_PASSWORD** | Password to use to the Postgresql database | `string` | `sqlite` | | **REDIS_HOST** | The host of the redis database | `string` `undefined` | `undefined` | | **REDIS_PORT** | Port used by Redis | `int` | `6379` | | **REDIS_PASSWORD** | The password of the redis database | `string` | ` ` | | **REDIS_DB** | The index of the redis database | `int` | `0` | | **CUSTOM_SCORES_TIMEOUT** | Seconds to consider a not-pinged server as unavailable | `int` | `10` | -| **CUSTOM_IMAGE_RESIZER** | Enable or disable the custom image resizer (most efficient than Plex one) | `bool` | `true` | +| **CUSTOM_IMAGE_RESIZER** | Enable or disable the custom (Unicorn) image resizer (most efficient than Plex one) | `bool` | `true` | +| **CUSTOM_IMAGE_PROXY** | Use a proxy to convert images | `string` | ` ` | | **CUSTOM_DOWNLOAD_FORWARD** | Enable or disable 302 for download links and direct play | `bool` | `true` | | **CUSTOM_SERVERS_LIST** | Transcoder servers set by default, **without** a slash at the end, separate servers with a **comma** | `string array` | `[]` | diff --git a/package.json b/package.json index fd5a9af..08d7610 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "http-proxy": "^1.17.0", "node-fetch": "^2.2.1", "node-pre-gyp": "^0.11.0", + "query-string": "^6.2.0", "redis": "^2.8.0", "sharp": "^0.21.2", "sqlite3": "^4.0.2", diff --git a/src/config.js b/src/config.js index 882d79b..9f62ebd 100644 --- a/src/config.js +++ b/src/config.js @@ -15,8 +15,19 @@ export default { port: env.int('PLEX_PORT', 32400), path: { usr: env.string('PLEX_PATH_USR', '/usr/lib/plexmediaserver/'), - sessions: env.string('PLEX_PATH_SESSIONS', '/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Cache/Transcode/Sessions/'), - database: env.string('PLEX_PATH_DATABASE', '/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db') + sessions: env.string('PLEX_PATH_SESSIONS', '/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Cache/Transcode/Sessions/') + } + }, + database: { + mode: env.string('DATABASE_MODE', 'sqlite'), + sqlite: { + path: env.string('DATABASE_SQLITE_PATH', '/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db') + }, + postgresql: { + host: env.string('DATABASE_POSTGRESQL_HOST', ''), + database: env.string('DATABASE_POSTGRESQL_DATABASE', ''), + user: env.string('DATABASE_POSTGRESQL_USER', ''), + password: env.string('DATABASE_POSTGRESQL_PASSWORD', '') } }, redis: { @@ -30,7 +41,8 @@ export default { timeout: env.int('CUSTOM_SCORES_TIMEOUT', 10) }, image: { - resizer: env.boolish('CUSTOM_IMAGE_RESIZER', true) + resizer: env.boolish('CUSTOM_IMAGE_RESIZER', true), + proxy: env.string('CUSTOM_IMAGE_PROXY', '') }, download: { forward: env.boolish('CUSTOM_DOWNLOAD_FORWARD', true) diff --git a/src/core/images.js b/src/core/images.js index 520b4c1..8b6e9a4 100644 --- a/src/core/images.js +++ b/src/core/images.js @@ -2,7 +2,38 @@ import fetch from 'node-fetch'; import sharp from 'sharp'; import color from 'color'; -export default (link, parameters, needAlpha = false) => { +export const parseArguments = (query, basepath = '') => { + + // Parse url + let url = query.url || false; + if (url && url[0] === '/') + url = basepath + url.substring(1); + + // Extract parameters + const params = { + ...((query.width) ? { width: parseInt(query.width) } : {}), + ...((query.height) ? { height: parseInt(query.height) } : {}), + ...((query.background) ? { background: query.background } : {}), + ...((query.opacity) ? { opacity: parseInt(query.opacity) } : {}), + ...((query.minSize) ? { minSize: parseInt(query.minSize) } : {}), + ...((query.blur) ? { blur: parseInt(query.blur) } : {}), + ...((query.format && (query.format === 'webp' || query.format === 'png')) ? { format: query.format } : { format: 'jpg' }), + ...((query.upscale) ? { upscale: parseInt(query.upscale) } : {}), + alpha: (query.format === 'png'), + url + }; + + // Auto select WebP if user-agent support it + const browser = parseUserAgent(req.get('User-Agent')); + if (browser.name === 'chrome') { + params.format = 'webp'; + } + + // Return params + return params; +} + +export const resize = (parameters) => { return new Promise(async (resolve, reject) => { try { const params = { @@ -39,7 +70,7 @@ export default (link, parameters, needAlpha = false) => { return reject('Size not provided'); // Get image content - const body = await fetch(link).then(res => res.buffer()); + const body = await fetch(parameters.link).then(res => res.buffer()); // Load body let s = sharp(body); @@ -94,7 +125,7 @@ export default (link, parameters, needAlpha = false) => { else if (params.format === 'webp') s.webp({ quality: 70, - ...((needAlpha) ? {} : { alphaQuality: 0 }) + ...((parameters.alpha) ? {} : { alphaQuality: 0 }) }) // Return stream diff --git a/src/database/index.js b/src/database/index.js new file mode 100644 index 0000000..50c9162 --- /dev/null +++ b/src/database/index.js @@ -0,0 +1,19 @@ +import config from '../config'; +import SqliteDatabase from './sqlite'; +import PostgresqlDatabase from './postgresql'; +import debug from 'debug'; + +// Debugger +const D = debug('UnicornLoadBalancer'); + +let Database; + +if (config.database.mode === 'sqlite') { + D('Using sqlite as database'); + Database = new SqliteDatabase(); +} else if (config.database.mode === 'postgresql') { + D('Using postgresql as database'); + Database = new PostgresqlDatabase(); +} + +export default Database; \ No newline at end of file diff --git a/src/database/postgresql.js b/src/database/postgresql.js new file mode 100644 index 0000000..f7adce4 --- /dev/null +++ b/src/database/postgresql.js @@ -0,0 +1,9 @@ +import config from '../config'; + +let PostgresqlDatabase = {}; + +PostgresqlDatabase.getPart = (part_id) => (new Promise((resolve, reject) => { + return reject('FILE_NOT_FOUND'); +})) + +export default PostgresqlDatabase; \ No newline at end of file diff --git a/src/database/sqlite.js b/src/database/sqlite.js new file mode 100644 index 0000000..82f6df1 --- /dev/null +++ b/src/database/sqlite.js @@ -0,0 +1,22 @@ +import sqlite3 from 'sqlite3'; +import config from '../config'; + +let SqliteDatabase = {}; + +SqliteDatabase.getPart = (part_id) => (new Promise((resolve, reject) => { + try { + const db = new (sqlite3.verbose().Database)(config.database.sqlite.path); + db.get('SELECT * FROM media_parts WHERE id=? LIMIT 0, 1', part_id, (err, row) => { + if (row && row.file) + resolve(row); + else + reject('FILE_NOT_FOUND'); + db.close(); + }); + } + catch (err) { + return reject('FILE_NOT_FOUND'); + } +})) + +export default SqliteDatabase; \ No newline at end of file diff --git a/src/routes/api.js b/src/routes/api.js index 2bdf754..a37b221 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -1,11 +1,11 @@ import httpProxy from 'http-proxy'; -import sqlite3 from 'sqlite3'; import debug from 'debug'; import config from '../config'; import SessionStore from '../store'; import SessionsManager from '../core/sessions'; import ServersManager from '../core/servers'; +import Database from '../database'; // Debugger const D = debug('UnicornLoadBalancer'); @@ -32,19 +32,11 @@ RoutesAPI.ffmpeg = (req, res) => { // Resolve path from file id RoutesAPI.path = (req, res) => { - try { - const db = new (sqlite3.verbose().Database)(config.plex.path.database); - db.get('SELECT * FROM media_parts WHERE id=? LIMIT 0, 1', req.params.id, (err, row) => { - if (row && row.file) - res.send(JSON.stringify(row)); - else - res.status(400).send({ error: { code: 'FILE_NOT_FOUND', message: 'File not found in Plex Database' } }); - db.close(); - }); - } - catch (err) { + Database.getPart(part_id).then((data) => { + res.send(JSON.stringify(data)); + }).reject((err) => { res.status(400).send({ error: { code: 'FILE_NOT_FOUND', message: 'File not found in Plex Database' } }); - } + }) }; // Proxy to Plex diff --git a/src/routes/index.js b/src/routes/index.js index f4e149d..6d6b152 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -46,8 +46,11 @@ export default (app) => { app.get('/library/parts/:id1/:id2/file.*', RoutesTranscode.redirect); } - // Image Resizer - if (config.custom.image.resizer) { + // Image Proxy or Iamge Resizer + if (config.custom.image.proxy) { + app.get('/photo/:/transcode', RoutesResize.proxy); + } + else if (config.custom.image.resizer) { app.get('/photo/:/transcode', RoutesResize.resize); } diff --git a/src/routes/resize.js b/src/routes/resize.js index 224ef87..a4785d7 100644 --- a/src/routes/resize.js +++ b/src/routes/resize.js @@ -1,46 +1,31 @@ import debug from 'debug'; -import { parseUserAgent } from 'detect-browser'; import { plexUrl } from '../utils'; -import optimizeImage from '../core/images'; +import { parseArguments, resize } from '../core/images'; // Debugger const D = debug('UnicornLoadBalancer'); let RoutesResize = {}; +RoutesResize.proxy = (req, res) => { + const params = parseArguments(req.query, plexUrl()); + // TODO + // Fallback + RoutesResize.resize(req, res); +} + RoutesResize.resize = (req, res) => { - // Parse url - let url = req.query.url || false; - if (url && url[0] === '/') - url = plexUrl() + url.substring(1); - - // Extract parameters - const params = { - ...((req.query.width) ? { width: parseInt(req.query.width) } : {}), - ...((req.query.height) ? { height: parseInt(req.query.height) } : {}), - ...((req.query.background) ? { background: req.query.background } : {}), - ...((req.query.opacity) ? { opacity: parseInt(req.query.opacity) } : {}), - ...((req.query.minSize) ? { minSize: parseInt(req.query.minSize) } : {}), - ...((req.query.blur) ? { blur: parseInt(req.query.blur) } : {}), - ...((req.query.format && (req.query.format === 'webp' || req.query.format === 'png')) ? { format: req.query.format } : { format: 'jpg' }), - ...((req.query.upscale) ? { upscale: parseInt(req.query.upscale) } : {}), - }; + // Parse params + const params = parseArguments(req.query, plexUrl()); // Check size - if (!params.width || !params.height || !url) + if (!params.width || !params.height || !params.url) return (res.status(400).send({ error: { code: 'RESIZE_ERROR', message: 'Invalid parameters, resize request fails' } })); - // Auto select WebP if user-agent support it - const browser = parseUserAgent(req.get('User-Agent')); - const needAlpha = params.format === 'png'; - if (browser.name === 'chrome') { - params.format = 'webp'; - } - // Debug - D('IMAGE ' + url + ' [' + params.format + ']'); + D('IMAGE ' + params.url + ' [' + params.format + ']'); // Mime type if (params.format === 'webp') @@ -51,7 +36,7 @@ RoutesResize.resize = (req, res) => { res.type(`image/jpeg`); // Process image - optimizeImage(url, params, needAlpha).then((stream) => { + resize(params).then((stream) => { return stream.pipe(res); }).catch(err => { return (res.status(400).send({ error: { code: 'RESIZE_ERROR', message: 'Invalid parameters, resize request fails' } })); From 93b16acb8b1f05dbc27762d6ab6543cd5285fb56 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 21 Feb 2019 22:48:48 +0100 Subject: [PATCH 089/132] Image Resizer proxy and minor changes --- README.md | 1 - package.json | 2 ++ src/config.js | 3 +-- src/core/images.js | 29 ++++++++++++++++++++++------- src/database/postgresql.js | 6 +++++- src/database/sqlite.js | 18 +++++++++++++++++- src/routes/api.js | 2 +- src/routes/resize.js | 33 +++++++++++++++++++-------------- 8 files changed, 67 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index d73dc27..977b3f2 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer | ----------------- | ------------------------------------------------------------ | ------| ------- | | **SERVER_HOST** | Host to access to the *UnicornLoadBalancer* | `string` | `127.0.0.1` | | **SERVER_PORT** | Port used by the *UnicornLoadBalancer* | `int` | `3001` | -| **SERVER_SSL** | If HTTPS is enabled or not on the *UnicornLoadBalancer* | `bool` | `false` | | **SERVER_PUBLIC** | Public url where the *UnicornLoadBalancer* can be called, **with** a slash at the end | `string` | `http://127.0.0.1:3001/` | | **PLEX_HOST** | Host to access to Plex | `string` | `127.0.0.1` | | **PLEX_PORT** | Port used by Plex | `int` | `32400` | diff --git a/package.json b/package.json index 08d7610..08969cc 100644 --- a/package.json +++ b/package.json @@ -17,8 +17,10 @@ "express": "^4.16.3", "getenv": "^0.7.0", "http-proxy": "^1.17.0", + "md5": "^2.2.1", "node-fetch": "^2.2.1", "node-pre-gyp": "^0.11.0", + "pg": "^7.8.1", "query-string": "^6.2.0", "redis": "^2.8.0", "sharp": "^0.21.2", diff --git a/src/config.js b/src/config.js index 9f62ebd..65d98ed 100644 --- a/src/config.js +++ b/src/config.js @@ -7,8 +7,7 @@ export default { server: { port: env.int('SERVER_PORT', 3001), public: env.string('SERVER_PUBLIC', 'http://127.0.0.1:3001/'), - host: env.string('SERVER_HOST', '127.0.0.1'), - ssl: env.boolish('SERVER_SSL', false) + host: env.string('SERVER_HOST', '127.0.0.1') }, plex: { host: env.string('PLEX_HOST', '127.0.0.1'), diff --git a/src/core/images.js b/src/core/images.js index 8b6e9a4..39a4743 100644 --- a/src/core/images.js +++ b/src/core/images.js @@ -1,11 +1,13 @@ import fetch from 'node-fetch'; import sharp from 'sharp'; import color from 'color'; +import md5 from 'md5'; +import { parseUserAgent } from 'detect-browser'; + +export const parseArguments = (query, basepath = '', useragent = '') => { -export const parseArguments = (query, basepath = '') => { - // Parse url - let url = query.url || false; + let url = query.url || ''; if (url && url[0] === '/') url = basepath + url.substring(1); @@ -24,11 +26,14 @@ export const parseArguments = (query, basepath = '') => { }; // Auto select WebP if user-agent support it - const browser = parseUserAgent(req.get('User-Agent')); - if (browser.name === 'chrome') { + const browser = parseUserAgent(useragent); + if (browser.name === 'chrome' && !query.format) { params.format = 'webp'; } + // Generate key + params.key = md5(`${(query.url || '').split('?')[0]}|${params.width || ''}|${params.height || ''}|${params.background || ''}|${params.opacity || ''}|${params.minSize || ''}|${params.blur || ''}|${params.format || ''}|${params.upscale || ''}`.toLowerCase()) + // Return params return params; } @@ -70,10 +75,18 @@ export const resize = (parameters) => { return reject('Size not provided'); // Get image content - const body = await fetch(parameters.link).then(res => res.buffer()); + const body = await fetch(parameters.url).then(res => res.buffer()); // Load body - let s = sharp(body); + let s = false; + try { + s = sharp(body); + } + catch (e) { + return reject(e) + } + if (!s) + return reject(e) // Resize parameters const opt = { @@ -88,6 +101,7 @@ export const resize = (parameters) => { // Background & opacity support if (params.background && params.opacity) { + const buff = await s.png().toBuffer(); s = sharp(buff); const meta = await s.metadata(); @@ -105,6 +119,7 @@ export const resize = (parameters) => { } }).png().toBuffer(); s.overlayWith(bgd); + } // Blur diff --git a/src/database/postgresql.js b/src/database/postgresql.js index f7adce4..f79f459 100644 --- a/src/database/postgresql.js +++ b/src/database/postgresql.js @@ -2,7 +2,11 @@ import config from '../config'; let PostgresqlDatabase = {}; -PostgresqlDatabase.getPart = (part_id) => (new Promise((resolve, reject) => { +PostgresqlDatabase.getPartFromId = (part_id) => (new Promise((resolve, reject) => { + return reject('FILE_NOT_FOUND'); +})) + +PostgresqlDatabase.getPartFromPath = (path) => (new Promise((resolve, reject) => { return reject('FILE_NOT_FOUND'); })) diff --git a/src/database/sqlite.js b/src/database/sqlite.js index 82f6df1..755f13d 100644 --- a/src/database/sqlite.js +++ b/src/database/sqlite.js @@ -3,7 +3,7 @@ import config from '../config'; let SqliteDatabase = {}; -SqliteDatabase.getPart = (part_id) => (new Promise((resolve, reject) => { +SqliteDatabase.getPartFromId = (part_id) => (new Promise((resolve, reject) => { try { const db = new (sqlite3.verbose().Database)(config.database.sqlite.path); db.get('SELECT * FROM media_parts WHERE id=? LIMIT 0, 1', part_id, (err, row) => { @@ -19,4 +19,20 @@ SqliteDatabase.getPart = (part_id) => (new Promise((resolve, reject) => { } })) +SqliteDatabase.getPartFromPath = (path) => (new Promise((resolve, reject) => { + try { + const db = new (sqlite3.verbose().Database)(config.database.sqlite.path); + db.get('SELECT * FROM media_parts WHERE file=? LIMIT 0, 1', path, (err, row) => { + if (row && row.file) + resolve(row); + else + reject('FILE_NOT_FOUND'); + db.close(); + }); + } + catch (err) { + return reject('FILE_NOT_FOUND'); + } +})) + export default SqliteDatabase; \ No newline at end of file diff --git a/src/routes/api.js b/src/routes/api.js index a37b221..44fb9fe 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -32,7 +32,7 @@ RoutesAPI.ffmpeg = (req, res) => { // Resolve path from file id RoutesAPI.path = (req, res) => { - Database.getPart(part_id).then((data) => { + Database.getPartFromId(part_id).then((data) => { res.send(JSON.stringify(data)); }).reject((err) => { res.status(400).send({ error: { code: 'FILE_NOT_FOUND', message: 'File not found in Plex Database' } }); diff --git a/src/routes/resize.js b/src/routes/resize.js index a4785d7..590a2ee 100644 --- a/src/routes/resize.js +++ b/src/routes/resize.js @@ -1,24 +1,28 @@ import debug from 'debug'; - +import httpProxy from 'http-proxy'; import { plexUrl } from '../utils'; import { parseArguments, resize } from '../core/images'; +import config from '../config'; // Debugger const D = debug('UnicornLoadBalancer'); let RoutesResize = {}; +/* Forward image request to the image transcode */ RoutesResize.proxy = (req, res) => { - const params = parseArguments(req.query, plexUrl()); - // TODO - // Fallback - RoutesResize.resize(req, res); + const params = parseArguments(req.query, plexUrl(), req.get('User-Agent')); + const path = Object.keys(params).map(e => (`${e}=${encodeURIComponent(params[e])}`)).join('&'); + req.url = config.custom.image.proxy + 'photo/:/transcode?' + path; + const proxy = httpProxy.createProxyServer({ target: config.custom.image.proxy, changeOrigin: true }); + proxy.web(req, res); } +/* Custom image transcoder */ RoutesResize.resize = (req, res) => { // Parse params - const params = parseArguments(req.query, plexUrl()); + const params = parseArguments(req.query, plexUrl(), req.get('User-Agent')); // Check size if (!params.width || !params.height || !params.url) @@ -27,16 +31,17 @@ RoutesResize.resize = (req, res) => { // Debug D('IMAGE ' + params.url + ' [' + params.format + ']'); - // Mime type - if (params.format === 'webp') - res.type(`image/webp`); - else if (params.format === 'png') - res.type(`image/png`); - else - res.type(`image/jpeg`); - // Process image resize(params).then((stream) => { + + // Mime type + if (params.format === 'webp') + res.type(`image/webp`); + else if (params.format === 'png') + res.type(`image/png`); + else + res.type(`image/jpeg`); + return stream.pipe(res); }).catch(err => { return (res.status(400).send({ error: { code: 'RESIZE_ERROR', message: 'Invalid parameters, resize request fails' } })); From 154c71307268075719cf089a2e0b36bd03af2358 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 21 Feb 2019 22:50:41 +0100 Subject: [PATCH 090/132] Catch proxy errors --- src/routes/resize.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/routes/resize.js b/src/routes/resize.js index 590a2ee..bd88ed9 100644 --- a/src/routes/resize.js +++ b/src/routes/resize.js @@ -15,6 +15,9 @@ RoutesResize.proxy = (req, res) => { const path = Object.keys(params).map(e => (`${e}=${encodeURIComponent(params[e])}`)).join('&'); req.url = config.custom.image.proxy + 'photo/:/transcode?' + path; const proxy = httpProxy.createProxyServer({ target: config.custom.image.proxy, changeOrigin: true }); + proxy.on('error', (e) => { + return (res.status(400).send({ error: { code: 'RESIZE_ERROR', message: 'Invalid parameters, resize request fails' } })); + }); proxy.web(req, res); } From aed4db1c02498d24579e45387c2c170c99defcb2 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 21 Feb 2019 23:01:43 +0100 Subject: [PATCH 091/132] Consistency --- README.md | 4 ++-- src/app.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 977b3f2..5b89a59 100644 --- a/README.md +++ b/README.md @@ -45,9 +45,9 @@ atabase | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex | **REDIS_DB** | The index of the redis database | `int` | `0` | | **CUSTOM_SCORES_TIMEOUT** | Seconds to consider a not-pinged server as unavailable | `int` | `10` | | **CUSTOM_IMAGE_RESIZER** | Enable or disable the custom (Unicorn) image resizer (most efficient than Plex one) | `bool` | `true` | -| **CUSTOM_IMAGE_PROXY** | Use a proxy to convert images | `string` | ` ` | +| **CUSTOM_IMAGE_PROXY** | Use a proxy to convert images, **with** a slash at the end | `string` | ` ` | | **CUSTOM_DOWNLOAD_FORWARD** | Enable or disable 302 for download links and direct play | `bool` | `true` | -| **CUSTOM_SERVERS_LIST** | Transcoder servers set by default, **without** a slash at the end, separate servers with a **comma** | `string array` | `[]` | +| **CUSTOM_SERVERS_LIST** | Transcoder servers set by default, **with** a slash at the end, separate servers with a **comma** | `string array` | `[]` | * Configure Plex Media Server access address * In Settings -> Server -> Network diff --git a/src/app.js b/src/app.js index b4a76a8..2757928 100644 --- a/src/app.js +++ b/src/app.js @@ -42,7 +42,7 @@ Router(app); // Load servers available in configuration ((Array.isArray(config.custom.servers.list)) ? config.custom.servers.list : []).map(e => ({ name: e, - url: e, + url: ((e.substr(-1) === '/') ? e.substr(0, e.length - 1) : e), sessions: [], settings: { maxSessions: 0, From b6649e25846a2601140e07edc6033fcb67429909 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 21 Feb 2019 23:09:02 +0100 Subject: [PATCH 092/132] Disable iamge resizer by default --- README.md | 2 +- src/config.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5b89a59..af2693b 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ atabase | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex | **REDIS_PASSWORD** | The password of the redis database | `string` | ` ` | | **REDIS_DB** | The index of the redis database | `int` | `0` | | **CUSTOM_SCORES_TIMEOUT** | Seconds to consider a not-pinged server as unavailable | `int` | `10` | -| **CUSTOM_IMAGE_RESIZER** | Enable or disable the custom (Unicorn) image resizer (most efficient than Plex one) | `bool` | `true` | +| **CUSTOM_IMAGE_RESIZER** | Enable or disable the custom (Unicorn) image resizer (most efficient than Plex one) | `bool` | `false` | | **CUSTOM_IMAGE_PROXY** | Use a proxy to convert images, **with** a slash at the end | `string` | ` ` | | **CUSTOM_DOWNLOAD_FORWARD** | Enable or disable 302 for download links and direct play | `bool` | `true` | | **CUSTOM_SERVERS_LIST** | Transcoder servers set by default, **with** a slash at the end, separate servers with a **comma** | `string array` | `[]` | diff --git a/src/config.js b/src/config.js index 65d98ed..74ff541 100644 --- a/src/config.js +++ b/src/config.js @@ -40,7 +40,7 @@ export default { timeout: env.int('CUSTOM_SCORES_TIMEOUT', 10) }, image: { - resizer: env.boolish('CUSTOM_IMAGE_RESIZER', true), + resizer: env.boolish('CUSTOM_IMAGE_RESIZER', false), proxy: env.string('CUSTOM_IMAGE_PROXY', '') }, download: { From 1639803e3dd4f6587e3e01368eadd2f7f2df7140 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 21 Feb 2019 23:09:15 +0100 Subject: [PATCH 093/132] Fix major issue --- src/database/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/index.js b/src/database/index.js index 50c9162..37641dd 100644 --- a/src/database/index.js +++ b/src/database/index.js @@ -10,10 +10,10 @@ let Database; if (config.database.mode === 'sqlite') { D('Using sqlite as database'); - Database = new SqliteDatabase(); + Database = SqliteDatabase; } else if (config.database.mode === 'postgresql') { D('Using postgresql as database'); - Database = new PostgresqlDatabase(); + Database = PostgresqlDatabase; } export default Database; \ No newline at end of file From b19b1483c1cd84072ad8abca0bca14596c313c11 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 21 Feb 2019 23:15:32 +0100 Subject: [PATCH 094/132] Fix image proxy --- src/routes/resize.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/resize.js b/src/routes/resize.js index bd88ed9..0a04d0c 100644 --- a/src/routes/resize.js +++ b/src/routes/resize.js @@ -1,6 +1,6 @@ import debug from 'debug'; import httpProxy from 'http-proxy'; -import { plexUrl } from '../utils'; +import { publicUrl } from '../utils'; import { parseArguments, resize } from '../core/images'; import config from '../config'; @@ -25,7 +25,7 @@ RoutesResize.proxy = (req, res) => { RoutesResize.resize = (req, res) => { // Parse params - const params = parseArguments(req.query, plexUrl(), req.get('User-Agent')); + const params = parseArguments(req.query, publicUrl(), req.get('User-Agent')); // Check size if (!params.width || !params.height || !params.url) From c49182bd0a6fa52969db1fe58410c96e527c78ac Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 21 Feb 2019 23:16:12 +0100 Subject: [PATCH 095/132] Fix image proxy --- src/routes/resize.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/resize.js b/src/routes/resize.js index 0a04d0c..1d357b2 100644 --- a/src/routes/resize.js +++ b/src/routes/resize.js @@ -11,7 +11,7 @@ let RoutesResize = {}; /* Forward image request to the image transcode */ RoutesResize.proxy = (req, res) => { - const params = parseArguments(req.query, plexUrl(), req.get('User-Agent')); + const params = parseArguments(req.query, publicUrl(), req.get('User-Agent')); const path = Object.keys(params).map(e => (`${e}=${encodeURIComponent(params[e])}`)).join('&'); req.url = config.custom.image.proxy + 'photo/:/transcode?' + path; const proxy = httpProxy.createProxyServer({ target: config.custom.image.proxy, changeOrigin: true }); From a2b4f7a275d6dc1604d1f5bfb1892ecd4990cf73 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 21 Feb 2019 23:29:07 +0100 Subject: [PATCH 096/132] Fix crash --- src/core/images.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/images.js b/src/core/images.js index 39a4743..2310e98 100644 --- a/src/core/images.js +++ b/src/core/images.js @@ -27,7 +27,7 @@ export const parseArguments = (query, basepath = '', useragent = '') => { // Auto select WebP if user-agent support it const browser = parseUserAgent(useragent); - if (browser.name === 'chrome' && !query.format) { + if (browser.name && browser.name === 'chrome' && !query.format) { params.format = 'webp'; } From f6fe0209a4a76f0f57d268cc7ef3a256085e6582 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 21 Feb 2019 23:32:48 +0100 Subject: [PATCH 097/132] Fix crash --- src/core/images.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/images.js b/src/core/images.js index 2310e98..a3e8f66 100644 --- a/src/core/images.js +++ b/src/core/images.js @@ -27,7 +27,7 @@ export const parseArguments = (query, basepath = '', useragent = '') => { // Auto select WebP if user-agent support it const browser = parseUserAgent(useragent); - if (browser.name && browser.name === 'chrome' && !query.format) { + if (browser && browser.name && browser.name === 'chrome' && !query.format) { params.format = 'webp'; } From 87938ca13459032695fb8dcf8765dc6b32f47f40 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 21 Feb 2019 23:56:46 +0100 Subject: [PATCH 098/132] src --- src/routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/api.js b/src/routes/api.js index 44fb9fe..4bbeb47 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -32,7 +32,7 @@ RoutesAPI.ffmpeg = (req, res) => { // Resolve path from file id RoutesAPI.path = (req, res) => { - Database.getPartFromId(part_id).then((data) => { + Database.getPartFromId(req.params.id).then((data) => { res.send(JSON.stringify(data)); }).reject((err) => { res.status(400).send({ error: { code: 'FILE_NOT_FOUND', message: 'File not found in Plex Database' } }); From e332a8aabba62deed8cc27d80cefae9480f2791c Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Thu, 21 Feb 2019 23:59:35 +0100 Subject: [PATCH 099/132] Fix bug --- src/routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/api.js b/src/routes/api.js index 4bbeb47..20054c9 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -34,7 +34,7 @@ RoutesAPI.ffmpeg = (req, res) => { RoutesAPI.path = (req, res) => { Database.getPartFromId(req.params.id).then((data) => { res.send(JSON.stringify(data)); - }).reject((err) => { + }).catch((err) => { res.status(400).send({ error: { code: 'FILE_NOT_FOUND', message: 'File not found in Plex Database' } }); }) }; From cf4eaaf69e29055240c05696f226ee5ed7f36024 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 00:03:21 +0100 Subject: [PATCH 100/132] Fix resolve error --- src/core/servers.js | 2 +- src/core/sessions.js | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/core/servers.js b/src/core/servers.js index 1c702c5..a6c3e63 100644 --- a/src/core/servers.js +++ b/src/core/servers.js @@ -62,7 +62,7 @@ ServersManager.chooseServer = (session, ip = false) => { .then(res => res.json()) .then(body => { return resolve(body.client) - }); + }).catch((err) => { return reject(err) }); })); }; diff --git a/src/core/sessions.js b/src/core/sessions.js index ab9ad12..511243f 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -19,9 +19,14 @@ let urls = {} SessionsManager.chooseServer = async (session, ip = false) => { if (urls[session]) return (urls[session]); - const url = await ServersManager.chooseServer(session, ip); + let url = ''; + try { + url = await ServersManager.chooseServer(session, ip); + } + catch (err) { } D('SERVER ' + session + ' [' + url + ']'); - urls[session] = url; + if (url.length) + urls[session] = url; return (url); }; From f94f57712e83c67ca304d813180da014fd6f8152 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 11:00:33 +0100 Subject: [PATCH 101/132] Fix local images and support headers on images --- src/core/images.js | 12 +++++++++--- src/routes/resize.js | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/core/images.js b/src/core/images.js index a3e8f66..24d92bb 100644 --- a/src/core/images.js +++ b/src/core/images.js @@ -4,10 +4,11 @@ import color from 'color'; import md5 from 'md5'; import { parseUserAgent } from 'detect-browser'; -export const parseArguments = (query, basepath = '', useragent = '') => { +export const parseArguments = (query, basepath = '/', useragent = '') => { // Parse url let url = query.url || ''; + url.replace('http://127.0.0.1/', basepath); if (url && url[0] === '/') url = basepath + url.substring(1); @@ -38,7 +39,7 @@ export const parseArguments = (query, basepath = '', useragent = '') => { return params; } -export const resize = (parameters) => { +export const resize = (parameters, headers = {}) => { return new Promise(async (resolve, reject) => { try { const params = { @@ -74,8 +75,13 @@ export const resize = (parameters) => { if (!params.width || !params.height) return reject('Size not provided'); + // Erase previous host header + delete headers.host; + // Get image content - const body = await fetch(parameters.url).then(res => res.buffer()); + const body = await fetch(parameters.url, { + headers + }).then(res => res.buffer()); // Load body let s = false; diff --git a/src/routes/resize.js b/src/routes/resize.js index 1d357b2..2c726a6 100644 --- a/src/routes/resize.js +++ b/src/routes/resize.js @@ -35,7 +35,7 @@ RoutesResize.resize = (req, res) => { D('IMAGE ' + params.url + ' [' + params.format + ']'); // Process image - resize(params).then((stream) => { + resize(params, req.headers).then((stream) => { // Mime type if (params.format === 'webp') From 3c49770fc9826353055d77c90e26b54b33434e34 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 11:28:20 +0100 Subject: [PATCH 102/132] Support Postgresql as database --- README.md | 7 ++++--- src/config.js | 3 ++- src/database/postgresql.js | 42 ++++++++++++++++++++++++++++++++++++-- src/database/sqlite.js | 4 ++-- 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index af2693b..d287694 100644 --- a/README.md +++ b/README.md @@ -35,10 +35,11 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer | **DATABASE_MODE** | Kind of database to use with Plex, can be `sqlite` or `postgresql` | `string` | `sqlite` | | **DATABASE_SQLITE_PATH** | The path of the Plex d atabase | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db` | -| **DATABASE_POSTGRESQL_HOST** | Host of the postgresql server | `string` | ` ` | +| **DATABASE_POSTGRESQL_HOST** | Host of the Postgresql server | `string` | ` ` | | **DATABASE_POSTGRESQL_DATABASE** | Name of the postgresql database | `string` | ` ` | -| **DATABASE_POSTGRESQL_USER** | User to use to the Postgresql database| `string` | ` ` | -| **DATABASE_POSTGRESQL_PASSWORD** | Password to use to the Postgresql database | `string` | `sqlite` | +| **DATABASE_POSTGRESQL_USER** | User used by the Postgresql database| `string` | ` ` | +| **DATABASE_POSTGRESQL_PASSWORD** | Password used by the Postgresql database | `string` | `sqlite` | +| **DATABASE_POSTGRESQL_PORT** | Port used by the Postgresql database | `string` | `sqlite` | | **REDIS_HOST** | The host of the redis database | `string` `undefined` | `undefined` | | **REDIS_PORT** | Port used by Redis | `int` | `6379` | | **REDIS_PASSWORD** | The password of the redis database | `string` | ` ` | diff --git a/src/config.js b/src/config.js index 74ff541..5f7f6e8 100644 --- a/src/config.js +++ b/src/config.js @@ -26,7 +26,8 @@ export default { host: env.string('DATABASE_POSTGRESQL_HOST', ''), database: env.string('DATABASE_POSTGRESQL_DATABASE', ''), user: env.string('DATABASE_POSTGRESQL_USER', ''), - password: env.string('DATABASE_POSTGRESQL_PASSWORD', '') + password: env.string('DATABASE_POSTGRESQL_PASSWORD', ''), + port: env.string('DATABASE_POSTGRESQL_PORT', 5432) } }, redis: { diff --git a/src/database/postgresql.js b/src/database/postgresql.js index f79f459..31f5c34 100644 --- a/src/database/postgresql.js +++ b/src/database/postgresql.js @@ -1,13 +1,51 @@ +import { Client } from 'pg'; import config from '../config'; let PostgresqlDatabase = {}; +const _getClient = () => (new Promise(async (resolve, reject) => { + const client = new Client({ + user: config.database.postgresql.user, + host: config.database.postgresql.host, + database: config.database.postgresql.database, + password: config.database.postgresql.password, + port: config.database.postgresql.port, + }) + client.on('error', (err) => { + return reject(err); + }) + await client.connect(); + return resolve(client); +})) + PostgresqlDatabase.getPartFromId = (part_id) => (new Promise((resolve, reject) => { - return reject('FILE_NOT_FOUND'); + _getClient.then((client) => { + client.query('SELECT * FROM media_parts WHERE id=$1 LIMIT 0, 1', [part_id], (err, res) => { + client.end() + if (res.rows.length) { + return resolve(res.rows[0]) + } else { + return reject('FILE_NOT_FOUND'); + } + }) + }).catch((err) => { + return reject('DATABASE_ERROR'); + }) })) PostgresqlDatabase.getPartFromPath = (path) => (new Promise((resolve, reject) => { - return reject('FILE_NOT_FOUND'); + _getClient.then((client) => { + client.query('SELECT * FROM media_parts WHERE file=$1 LIMIT 0, 1', [path], (err, res) => { + client.end() + if (res.rows.length) { + return resolve(res.rows[0]) + } else { + return reject('FILE_NOT_FOUND'); + } + }) + }).catch((err) => { + return reject('DATABASE_ERROR'); + }) })) export default PostgresqlDatabase; \ No newline at end of file diff --git a/src/database/sqlite.js b/src/database/sqlite.js index 755f13d..79b7b7f 100644 --- a/src/database/sqlite.js +++ b/src/database/sqlite.js @@ -15,7 +15,7 @@ SqliteDatabase.getPartFromId = (part_id) => (new Promise((resolve, reject) => { }); } catch (err) { - return reject('FILE_NOT_FOUND'); + return reject('DATABASE_ERROR'); } })) @@ -31,7 +31,7 @@ SqliteDatabase.getPartFromPath = (path) => (new Promise((resolve, reject) => { }); } catch (err) { - return reject('FILE_NOT_FOUND'); + return reject('DATABASE_ERROR'); } })) From 2309f285ae6d69150de2d8c5edb34526b3df2f8b Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 11:30:04 +0100 Subject: [PATCH 103/132] Fix postgresql port type --- README.md | 2 +- src/config.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d287694..9d1e86e 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ atabase | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex | **DATABASE_POSTGRESQL_DATABASE** | Name of the postgresql database | `string` | ` ` | | **DATABASE_POSTGRESQL_USER** | User used by the Postgresql database| `string` | ` ` | | **DATABASE_POSTGRESQL_PASSWORD** | Password used by the Postgresql database | `string` | `sqlite` | -| **DATABASE_POSTGRESQL_PORT** | Port used by the Postgresql database | `string` | `sqlite` | +| **DATABASE_POSTGRESQL_PORT** | Port used by the Postgresql database | `int` | `5432` | | **REDIS_HOST** | The host of the redis database | `string` `undefined` | `undefined` | | **REDIS_PORT** | Port used by Redis | `int` | `6379` | | **REDIS_PASSWORD** | The password of the redis database | `string` | ` ` | diff --git a/src/config.js b/src/config.js index 5f7f6e8..5540c32 100644 --- a/src/config.js +++ b/src/config.js @@ -27,7 +27,7 @@ export default { database: env.string('DATABASE_POSTGRESQL_DATABASE', ''), user: env.string('DATABASE_POSTGRESQL_USER', ''), password: env.string('DATABASE_POSTGRESQL_PASSWORD', ''), - port: env.string('DATABASE_POSTGRESQL_PORT', 5432) + port: env.int('DATABASE_POSTGRESQL_PORT', 5432) } }, redis: { From f5f513bfa4fc8152474bed4d7b2042709070f2ef Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 12:12:23 +0100 Subject: [PATCH 104/132] Support link resolver --- README.md | 2 +- src/config.js | 2 +- src/core/sessions.js | 21 ++++++++++++++++++--- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9d1e86e..03e2f18 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ atabase | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex | **CUSTOM_SCORES_TIMEOUT** | Seconds to consider a not-pinged server as unavailable | `int` | `10` | | **CUSTOM_IMAGE_RESIZER** | Enable or disable the custom (Unicorn) image resizer (most efficient than Plex one) | `bool` | `false` | | **CUSTOM_IMAGE_PROXY** | Use a proxy to convert images, **with** a slash at the end | `string` | ` ` | -| **CUSTOM_DOWNLOAD_FORWARD** | Enable or disable 302 for download links and direct play | `bool` | `true` | +| **CUSTOM_DOWNLOAD_FORWARD** | Enable or disable 302 for download links and direct play, if enabled, transcoders need to have access to media files | `bool` | `false` | | **CUSTOM_SERVERS_LIST** | Transcoder servers set by default, **with** a slash at the end, separate servers with a **comma** | `string array` | `[]` | * Configure Plex Media Server access address diff --git a/src/config.js b/src/config.js index 5540c32..cf24cf0 100644 --- a/src/config.js +++ b/src/config.js @@ -45,7 +45,7 @@ export default { proxy: env.string('CUSTOM_IMAGE_PROXY', '') }, download: { - forward: env.boolish('CUSTOM_DOWNLOAD_FORWARD', true) + forward: env.boolish('CUSTOM_DOWNLOAD_FORWARD', false) }, servers: { list: env.array('CUSTOM_SERVERS_LIST', 'string', []) diff --git a/src/core/sessions.js b/src/core/sessions.js index 511243f..97b8443 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -1,9 +1,9 @@ import debug from 'debug'; - import config from '../config'; import { publicUrl, plexUrl } from '../utils'; import SessionStore from '../store'; import ServersManager from './servers'; +import Database from '../database'; // Debugger const D = debug('UnicornLoadBalancer'); @@ -86,11 +86,12 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { return (e.replace(plexUrl(), publicUrl()).replace(config.plex.path.sessions, publicUrl() + 'api/sessions/').replace(config.plex.path.usr, '{INTERNAL_RESOURCES}')); }); - // Add seglist to arguments if needed + // Add seglist to arguments if needed and resolve links if needed const segList = '{INTERNAL_TRANSCODER}video/:/transcode/session/' + sessionFull + '/seglist'; let finalArgs = []; let segListMode = false; - parsedArgs.forEach((e, i) => { + parsedArgs.forEach(async (e, i) => { + // Seglist if (e === '-segment_list') { segListMode = true; finalArgs.push(e); @@ -103,6 +104,20 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { segListMode = false; return (true); } + + // Link resolver (Replace filepath to http plex path) + if (i > 0 && parsedArgs[i - 1] === '-i' && !config.custom.download.forward) { + file = parsedArgs[i] + try { + const data = await Database.getPartFromPath(parsedArgs[i]); + if (typeof(data.id) !== 'undefined') + file = `${publicUrl()}library/parts/${data.id}/0/file.stream?download=1`; + } catch (e) { } + finalArgs.push(file); + return (true); + } + + // Ignore aprameter finalArgs.push(e); }); return ({ From d8697f4a65b342f8ccccd45b5215dade14f01403 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 12:16:24 +0100 Subject: [PATCH 105/132] Fix undefined var --- src/core/sessions.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 97b8443..f087254 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -107,10 +107,10 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { // Link resolver (Replace filepath to http plex path) if (i > 0 && parsedArgs[i - 1] === '-i' && !config.custom.download.forward) { - file = parsedArgs[i] + let file = parsedArgs[i] try { const data = await Database.getPartFromPath(parsedArgs[i]); - if (typeof(data.id) !== 'undefined') + if (typeof (data.id) !== 'undefined') file = `${publicUrl()}library/parts/${data.id}/0/file.stream?download=1`; } catch (e) { } finalArgs.push(file); From 2c5c9b69721a80e8a174ddf2a09a3a26c0d666f3 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 12:40:47 +0100 Subject: [PATCH 106/132] Support copy codec in scoring calc --- src/core/servers.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/servers.js b/src/core/servers.js index a6c3e63..0f478aa 100644 --- a/src/core/servers.js +++ b/src/core/servers.js @@ -84,6 +84,9 @@ ServersManager.score = (e) => { if (s.codec === 'hevc') { load += 1.5; } + if (s.codec === 'copy') { + load -= 0.5; + } } // Serving streams From 9e40617f909b16113fa363946470cb9af931ca1b Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 12:41:45 +0100 Subject: [PATCH 107/132] Change timeout session --- src/store/local.js | 2 +- src/store/redis.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/store/local.js b/src/store/local.js index 448b7f5..efa6993 100644 --- a/src/store/local.js +++ b/src/store/local.js @@ -29,7 +29,7 @@ class LocalSessionStore { reject('timeout'); }; - timeout = setTimeout(timeoutCb, 10000); + timeout = setTimeout(timeoutCb, 20000); this.sessionEvents.on(sessionId, eventCb); }) } diff --git a/src/store/redis.js b/src/store/redis.js index 6b616f0..c6fc52c 100644 --- a/src/store/redis.js +++ b/src/store/redis.js @@ -35,7 +35,7 @@ class RedisSessionStore { let timeout = setTimeout(() => { this.redisSubscriber.unsubscribe(redisSubKey); reject('timeout'); - }, 10000); + }, 20000); this.redisSubscriber.on("message", (eventKey, action) => { if (action !== 'set' || eventKey !== redisSubKey) From 7e3a77d33d31308537f867f991035fd55caf86ff Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 13:03:45 +0100 Subject: [PATCH 108/132] Fix readme --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 03e2f18..67c6b95 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,7 @@ This software is a part of __UnicornTranscoder__ project, it's the LoadBalancer | **PLEX_PATH_USR** | The Plex's path | `string` | `/usr/lib/plexmediaserver/` | | **PLEX_PATH_SESSIONS** | The path where Plex store sessions (to grab external subtitles) | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Cache/Transcode/Sessions` | | **DATABASE_MODE** | Kind of database to use with Plex, can be `sqlite` or `postgresql` | `string` | `sqlite` | -| **DATABASE_SQLITE_PATH** | The path of the Plex d -atabase | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db` | +| **DATABASE_SQLITE_PATH** | The path of the Plex database | `string` | `/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db` | | **DATABASE_POSTGRESQL_HOST** | Host of the Postgresql server | `string` | ` ` | | **DATABASE_POSTGRESQL_DATABASE** | Name of the postgresql database | `string` | ` ` | | **DATABASE_POSTGRESQL_USER** | User used by the Postgresql database| `string` | ` ` | From 334c575de31ecd6cd59bb436350fd22490cddbd2 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 13:19:41 +0100 Subject: [PATCH 109/132] Fix image resizer --- src/core/images.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/images.js b/src/core/images.js index 24d92bb..5b1f3d2 100644 --- a/src/core/images.js +++ b/src/core/images.js @@ -9,6 +9,7 @@ export const parseArguments = (query, basepath = '/', useragent = '') => { // Parse url let url = query.url || ''; url.replace('http://127.0.0.1/', basepath); + url.replace('http://127.0.0.1:32400/', basepath); if (url && url[0] === '/') url = basepath + url.substring(1); From da3b3f0b36e51738195269370609f6fc31ab44d7 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 13:22:12 +0100 Subject: [PATCH 110/132] Fix replace --- src/core/images.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/images.js b/src/core/images.js index 5b1f3d2..baad980 100644 --- a/src/core/images.js +++ b/src/core/images.js @@ -8,8 +8,8 @@ export const parseArguments = (query, basepath = '/', useragent = '') => { // Parse url let url = query.url || ''; - url.replace('http://127.0.0.1/', basepath); - url.replace('http://127.0.0.1:32400/', basepath); + url = url.replace('http://127.0.0.1/', basepath); + url = url.replace('http://127.0.0.1:32400/', basepath); if (url && url[0] === '/') url = basepath + url.substring(1); From 0d64b38fbbbad9216e7675a167ff04976574c851 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 13:48:40 +0100 Subject: [PATCH 111/132] Fix resizer --- src/core/images.js | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/core/images.js b/src/core/images.js index baad980..10a3fb6 100644 --- a/src/core/images.js +++ b/src/core/images.js @@ -12,6 +12,9 @@ export const parseArguments = (query, basepath = '/', useragent = '') => { url = url.replace('http://127.0.0.1:32400/', basepath); if (url && url[0] === '/') url = basepath + url.substring(1); + if (query['X-Plex-Token']) { + url = (url.indexOf('?') === -1) ? `?X-Plex-Token=${query['X-Plex-Token']}` : `&X-Plex-Token=${query['X-Plex-Token']}` + } // Extract parameters const params = { @@ -24,6 +27,7 @@ export const parseArguments = (query, basepath = '/', useragent = '') => { ...((query.format && (query.format === 'webp' || query.format === 'png')) ? { format: query.format } : { format: 'jpg' }), ...((query.upscale) ? { upscale: parseInt(query.upscale) } : {}), alpha: (query.format === 'png'), + ...((query['X-Plex-Token']) ? { "X-Plex-Token": query['X-Plex-Token'] } : {}), url }; @@ -108,25 +112,29 @@ export const resize = (parameters, headers = {}) => { // Background & opacity support if (params.background && params.opacity) { - - const buff = await s.png().toBuffer(); - s = sharp(buff); - const meta = await s.metadata(); - const bgd = await sharp({ - create: { - width: meta.width, - height: meta.height, - channels: 4, - background: { - r: color(`#${params.background}`).r, - g: color(`#${params.background}`).g, - b: color(`#${params.background}`).b, - alpha: ((100 - params.opacity) / 100) + let bgd = false; + try { + const buff = await s.png().toBuffer(); + s = sharp(buff); + const meta = await s.metadata(); + bgd = await sharp({ + create: { + width: meta.width, + height: meta.height, + channels: 4, + background: { + r: color(`#${params.background}`).r, + g: color(`#${params.background}`).g, + b: color(`#${params.background}`).b, + alpha: ((100 - params.opacity) / 100) + } } - } - }).png().toBuffer(); + }).png().toBuffer(); + } + catch (e) { + return reject(e) + } s.overlayWith(bgd); - } // Blur From 8b35146617e627725b6d2f510fd0866ad4a9e385 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 13:50:39 +0100 Subject: [PATCH 112/132] Fix --- src/core/images.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/images.js b/src/core/images.js index 10a3fb6..53a64e8 100644 --- a/src/core/images.js +++ b/src/core/images.js @@ -13,7 +13,7 @@ export const parseArguments = (query, basepath = '/', useragent = '') => { if (url && url[0] === '/') url = basepath + url.substring(1); if (query['X-Plex-Token']) { - url = (url.indexOf('?') === -1) ? `?X-Plex-Token=${query['X-Plex-Token']}` : `&X-Plex-Token=${query['X-Plex-Token']}` + url += (url.indexOf('?') === -1) ? `?X-Plex-Token=${query['X-Plex-Token']}` : `&X-Plex-Token=${query['X-Plex-Token']}` } // Extract parameters From 3835ead7e76cc2fb39e30f59df8d465a9c6838f0 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 14:23:13 +0100 Subject: [PATCH 113/132] Fix resize --- src/core/sessions.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index f087254..122d453 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -107,12 +107,15 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { // Link resolver (Replace filepath to http plex path) if (i > 0 && parsedArgs[i - 1] === '-i' && !config.custom.download.forward) { - let file = parsedArgs[i] + let file = parsedArgs[i]; try { const data = await Database.getPartFromPath(parsedArgs[i]); if (typeof (data.id) !== 'undefined') file = `${publicUrl()}library/parts/${data.id}/0/file.stream?download=1`; - } catch (e) { } + } catch (e) { + console.log('CRASH request', e) + file = parsedArgs[i] + } finalArgs.push(file); return (true); } From 9585683917811dcbe9574f0743c4ba29672e0ba5 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 14:38:45 +0100 Subject: [PATCH 114/132] Fix postgresql --- src/database/postgresql.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/postgresql.js b/src/database/postgresql.js index 31f5c34..8e6352d 100644 --- a/src/database/postgresql.js +++ b/src/database/postgresql.js @@ -19,7 +19,7 @@ const _getClient = () => (new Promise(async (resolve, reject) => { })) PostgresqlDatabase.getPartFromId = (part_id) => (new Promise((resolve, reject) => { - _getClient.then((client) => { + _getClient().then((client) => { client.query('SELECT * FROM media_parts WHERE id=$1 LIMIT 0, 1', [part_id], (err, res) => { client.end() if (res.rows.length) { @@ -34,7 +34,7 @@ PostgresqlDatabase.getPartFromId = (part_id) => (new Promise((resolve, reject) = })) PostgresqlDatabase.getPartFromPath = (path) => (new Promise((resolve, reject) => { - _getClient.then((client) => { + _getClient().then((client) => { client.query('SELECT * FROM media_parts WHERE file=$1 LIMIT 0, 1', [path], (err, res) => { client.end() if (res.rows.length) { From a9abf1e6b4ecfbc0d70e724b9c38994f9125601b Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 14:46:27 +0100 Subject: [PATCH 115/132] Fixes --- src/database/postgresql.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/database/postgresql.js b/src/database/postgresql.js index 8e6352d..c5b35c2 100644 --- a/src/database/postgresql.js +++ b/src/database/postgresql.js @@ -21,6 +21,8 @@ const _getClient = () => (new Promise(async (resolve, reject) => { PostgresqlDatabase.getPartFromId = (part_id) => (new Promise((resolve, reject) => { _getClient().then((client) => { client.query('SELECT * FROM media_parts WHERE id=$1 LIMIT 0, 1', [part_id], (err, res) => { + if (err) + return reject(err); client.end() if (res.rows.length) { return resolve(res.rows[0]) @@ -36,6 +38,8 @@ PostgresqlDatabase.getPartFromId = (part_id) => (new Promise((resolve, reject) = PostgresqlDatabase.getPartFromPath = (path) => (new Promise((resolve, reject) => { _getClient().then((client) => { client.query('SELECT * FROM media_parts WHERE file=$1 LIMIT 0, 1', [path], (err, res) => { + if (err) + return reject(err); client.end() if (res.rows.length) { return resolve(res.rows[0]) From e1c08488f4fb59462c035e864a69dd197cd7d6b1 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 14:54:15 +0100 Subject: [PATCH 116/132] fixes --- src/database/postgresql.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/postgresql.js b/src/database/postgresql.js index c5b35c2..a4fb019 100644 --- a/src/database/postgresql.js +++ b/src/database/postgresql.js @@ -20,7 +20,7 @@ const _getClient = () => (new Promise(async (resolve, reject) => { PostgresqlDatabase.getPartFromId = (part_id) => (new Promise((resolve, reject) => { _getClient().then((client) => { - client.query('SELECT * FROM media_parts WHERE id=$1 LIMIT 0, 1', [part_id], (err, res) => { + client.query('SELECT * FROM media_parts WHERE id=$1 LIMIT 1', [part_id], (err, res) => { if (err) return reject(err); client.end() @@ -37,7 +37,7 @@ PostgresqlDatabase.getPartFromId = (part_id) => (new Promise((resolve, reject) = PostgresqlDatabase.getPartFromPath = (path) => (new Promise((resolve, reject) => { _getClient().then((client) => { - client.query('SELECT * FROM media_parts WHERE file=$1 LIMIT 0, 1', [path], (err, res) => { + client.query('SELECT * FROM media_parts WHERE file=$1 LIMIT 1', [path], (err, res) => { if (err) return reject(err); client.end() From b502832f627208b5d2384b388de899dc316496c2 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 15:02:18 +0100 Subject: [PATCH 117/132] Fixes --- src/core/sessions.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 122d453..4783089 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -113,9 +113,11 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { if (typeof (data.id) !== 'undefined') file = `${publicUrl()}library/parts/${data.id}/0/file.stream?download=1`; } catch (e) { - console.log('CRASH request', e) + console.log(e); file = parsedArgs[i] - } + finalArgs.push(file); + return (true); + } finalArgs.push(file); return (true); } From 4a6299228884e09d3793819a7e7dcec6dbca33f0 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 15:13:44 +0100 Subject: [PATCH 118/132] Fixes --- src/database/postgresql.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/database/postgresql.js b/src/database/postgresql.js index a4fb019..52513a9 100644 --- a/src/database/postgresql.js +++ b/src/database/postgresql.js @@ -36,10 +36,14 @@ PostgresqlDatabase.getPartFromId = (part_id) => (new Promise((resolve, reject) = })) PostgresqlDatabase.getPartFromPath = (path) => (new Promise((resolve, reject) => { + console.log('PARSING PATH:', path); _getClient().then((client) => { client.query('SELECT * FROM media_parts WHERE file=$1 LIMIT 1', [path], (err, res) => { - if (err) + console.log('CALLBACK DB', res) + if (err) { + console.log('ERROR', err) return reject(err); + } client.end() if (res.rows.length) { return resolve(res.rows[0]) From 87f5817637257868b881c5f237db398bbf1fdbaa Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 15:21:56 +0100 Subject: [PATCH 119/132] Fixes --- src/core/sessions.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 4783089..0a1319a 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -107,17 +107,22 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { // Link resolver (Replace filepath to http plex path) if (i > 0 && parsedArgs[i - 1] === '-i' && !config.custom.download.forward) { + console.log("REPLACE MODE") let file = parsedArgs[i]; try { const data = await Database.getPartFromPath(parsedArgs[i]); + console.log('GET DATA', data); if (typeof (data.id) !== 'undefined') file = `${publicUrl()}library/parts/${data.id}/0/file.stream?download=1`; + + console.log('FETCH', file) } catch (e) { - console.log(e); + console.log('ERROR', e); file = parsedArgs[i] finalArgs.push(file); return (true); } + console.log('DBG'); finalArgs.push(file); return (true); } From a2f381b64e85efc2856cd5f353e25bb92a77694e Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 15:29:56 +0100 Subject: [PATCH 120/132] Fix --- src/core/sessions.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 0a1319a..149c5fb 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -57,7 +57,7 @@ SessionsManager.getSessionFromRequest = (req) => { } // Parse FFmpeg parameters with internal bindings -SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { +SessionsManager.parseFFmpegParameters = async (args = [], env = {}) => { // Extract Session ID const regex = /^http\:\/\/127.0.0.1:32400\/video\/:\/transcode\/session\/(.*)\/progress$/; const sessions = args.filter(e => (regex.test(e))).map(e => (e.match(regex)[1])) @@ -90,7 +90,9 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { const segList = '{INTERNAL_TRANSCODER}video/:/transcode/session/' + sessionFull + '/seglist'; let finalArgs = []; let segListMode = false; - parsedArgs.forEach(async (e, i) => { + for (let i = 0; i < parsedArgs.length; i++) { + let e = parsedArgs[i]; + // Seglist if (e === '-segment_list') { segListMode = true; @@ -129,7 +131,7 @@ SessionsManager.parseFFmpegParameters = (args = [], env = {}) => { // Ignore aprameter finalArgs.push(e); - }); + }; return ({ args: finalArgs, env, From 337e96ce12f62720a50d2dea045312bd72405b56 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 15:35:25 +0100 Subject: [PATCH 121/132] Fix --- src/core/sessions.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/sessions.js b/src/core/sessions.js index 149c5fb..cfa0253 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -132,6 +132,7 @@ SessionsManager.parseFFmpegParameters = async (args = [], env = {}) => { // Ignore aprameter finalArgs.push(e); }; + console.log('ffmpeg', finalArgs); return ({ args: finalArgs, env, From 5ce1e5260dc5b5a6dc24a7e31acafaae10744cda Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 15:39:35 +0100 Subject: [PATCH 122/132] Fix --- src/core/sessions.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index cfa0253..99a36bf 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -97,14 +97,14 @@ SessionsManager.parseFFmpegParameters = async (args = [], env = {}) => { if (e === '-segment_list') { segListMode = true; finalArgs.push(e); - return (true); + continue; } if (segListMode) { finalArgs.push(segList); if (parsedArgs[i + 1] !== '-segment_list_type') finalArgs.push('-segment_list_type', 'csv', '-segment_list_size', '2147483647'); segListMode = false; - return (true); + continue; } // Link resolver (Replace filepath to http plex path) @@ -122,11 +122,11 @@ SessionsManager.parseFFmpegParameters = async (args = [], env = {}) => { console.log('ERROR', e); file = parsedArgs[i] finalArgs.push(file); - return (true); + continue; } console.log('DBG'); finalArgs.push(file); - return (true); + continue; } // Ignore aprameter From b302d7fdcfca748502ae82af888d1b424d155c28 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 15:54:26 +0100 Subject: [PATCH 123/132] Download direct route --- src/core/sessions.js | 11 +---------- src/routes/index.js | 5 ++++- src/routes/transcode.js | 17 ++++++++++++++++- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 99a36bf..7ea5c81 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -109,30 +109,21 @@ SessionsManager.parseFFmpegParameters = async (args = [], env = {}) => { // Link resolver (Replace filepath to http plex path) if (i > 0 && parsedArgs[i - 1] === '-i' && !config.custom.download.forward) { - console.log("REPLACE MODE") let file = parsedArgs[i]; try { const data = await Database.getPartFromPath(parsedArgs[i]); - console.log('GET DATA', data); if (typeof (data.id) !== 'undefined') file = `${publicUrl()}library/parts/${data.id}/0/file.stream?download=1`; - - console.log('FETCH', file) } catch (e) { - console.log('ERROR', e); file = parsedArgs[i] - finalArgs.push(file); - continue; } - console.log('DBG'); finalArgs.push(file); continue; } - // Ignore aprameter + // Ignore parameter finalArgs.push(e); }; - console.log('ffmpeg', finalArgs); return ({ args: finalArgs, env, diff --git a/src/routes/index.js b/src/routes/index.js index 6d6b152..bab9649 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -45,8 +45,11 @@ export default (app) => { if (config.custom.download.forward) { app.get('/library/parts/:id1/:id2/file.*', RoutesTranscode.redirect); } + if (!config.custom.download.forward) { + app.get('/library/parts/:id1/:id2/file.*', RoutesTranscode.download); + } - // Image Proxy or Iamge Resizer + // Image Proxy or Image Resizer if (config.custom.image.proxy) { app.get('/photo/:/transcode', RoutesResize.proxy); } diff --git a/src/routes/transcode.js b/src/routes/transcode.js index cb6cf0a..a7a5ce1 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -1,7 +1,7 @@ import debug from 'debug'; import fetch from 'node-fetch'; - import RoutesProxy from './proxy'; +import Database from '../database'; import SessionsManager from '../core/sessions'; // Debugger @@ -164,4 +164,19 @@ RoutesTranscode.stop = async (req, res) => { } }; +/* Route download */ +RoutesTranscode.download = (req, res) => { + D('DOWNLOAD ' + sessionId + ' [LB]'); + Database.getPartFromId(req.params.id1).then((data) => { + res.sendFile(data.file, {}, (err) => { + if (err) { + res.status(400).send({ error: { code: 'DOWNLOAD_ERROR', message: 'An error has occured during download' } }); + return; + } + }) + }).catch((err) => { + res.status(400).send({ error: { code: 'NOT_FOUND', message: 'File not available' } }); + }) +} + export default RoutesTranscode; From aefd65f5f916c8ea5e8cb74ac76754ea408d1719 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 15:54:53 +0100 Subject: [PATCH 124/132] Fix debug --- src/routes/transcode.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index a7a5ce1..e1bfcc2 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -166,7 +166,7 @@ RoutesTranscode.stop = async (req, res) => { /* Route download */ RoutesTranscode.download = (req, res) => { - D('DOWNLOAD ' + sessionId + ' [LB]'); + D('DOWNLOAD ' + req.params.id1 + ' [LB]'); Database.getPartFromId(req.params.id1).then((data) => { res.sendFile(data.file, {}, (err) => { if (err) { From 551e284e67c4738765ea0c17c3bf75bd45fd3c1f Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 16:06:11 +0100 Subject: [PATCH 125/132] Fix debug --- src/database/postgresql.js | 3 --- src/routes/transcode.js | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/database/postgresql.js b/src/database/postgresql.js index 52513a9..1e07090 100644 --- a/src/database/postgresql.js +++ b/src/database/postgresql.js @@ -36,12 +36,9 @@ PostgresqlDatabase.getPartFromId = (part_id) => (new Promise((resolve, reject) = })) PostgresqlDatabase.getPartFromPath = (path) => (new Promise((resolve, reject) => { - console.log('PARSING PATH:', path); _getClient().then((client) => { client.query('SELECT * FROM media_parts WHERE file=$1 LIMIT 1', [path], (err, res) => { - console.log('CALLBACK DB', res) if (err) { - console.log('ERROR', err) return reject(err); } client.end() diff --git a/src/routes/transcode.js b/src/routes/transcode.js index e1bfcc2..b0a7be1 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -170,6 +170,7 @@ RoutesTranscode.download = (req, res) => { Database.getPartFromId(req.params.id1).then((data) => { res.sendFile(data.file, {}, (err) => { if (err) { + console.log(err); res.status(400).send({ error: { code: 'DOWNLOAD_ERROR', message: 'An error has occured during download' } }); return; } From feb17229c18c48bd3dddb3b300e3bdd80d51a345 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 16:06:29 +0100 Subject: [PATCH 126/132] Fix debug --- src/routes/transcode.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index b0a7be1..832385f 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -170,7 +170,7 @@ RoutesTranscode.download = (req, res) => { Database.getPartFromId(req.params.id1).then((data) => { res.sendFile(data.file, {}, (err) => { if (err) { - console.log(err); + console.log('ERROR DOWNLOAD', err); res.status(400).send({ error: { code: 'DOWNLOAD_ERROR', message: 'An error has occured during download' } }); return; } From 8f33aa29ab73c964821745b4af61e1a900ed5b33 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 16:10:21 +0100 Subject: [PATCH 127/132] Fix debug --- src/routes/transcode.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index 832385f..c1d4eda 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -169,11 +169,7 @@ RoutesTranscode.download = (req, res) => { D('DOWNLOAD ' + req.params.id1 + ' [LB]'); Database.getPartFromId(req.params.id1).then((data) => { res.sendFile(data.file, {}, (err) => { - if (err) { - console.log('ERROR DOWNLOAD', err); - res.status(400).send({ error: { code: 'DOWNLOAD_ERROR', message: 'An error has occured during download' } }); - return; - } + D('DOWNLOAD FAILED ' + req.params.id1 + ' [LB]'); }) }).catch((err) => { res.status(400).send({ error: { code: 'NOT_FOUND', message: 'File not available' } }); From 0b48f861199fe1b94bed7c1eb4dcdb91fde9052d Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 16:15:45 +0100 Subject: [PATCH 128/132] Fix debug --- src/routes/transcode.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/routes/transcode.js b/src/routes/transcode.js index c1d4eda..9ceb0a2 100644 --- a/src/routes/transcode.js +++ b/src/routes/transcode.js @@ -169,7 +169,8 @@ RoutesTranscode.download = (req, res) => { D('DOWNLOAD ' + req.params.id1 + ' [LB]'); Database.getPartFromId(req.params.id1).then((data) => { res.sendFile(data.file, {}, (err) => { - D('DOWNLOAD FAILED ' + req.params.id1 + ' [LB]'); + if (err && err.code !== 'ECONNABORTED') + D('DOWNLOAD FAILED ' + req.params.id1 + ' [LB]'); }) }).catch((err) => { res.status(400).send({ error: { code: 'NOT_FOUND', message: 'File not available' } }); From 38318bf845194e8c64d2db1d73ae6af34dcde24c Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 19:00:18 +0100 Subject: [PATCH 129/132] Avoid leak Plex token on iamge resizer --- src/core/images.js | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/core/images.js b/src/core/images.js index 53a64e8..22d5918 100644 --- a/src/core/images.js +++ b/src/core/images.js @@ -8,13 +8,14 @@ export const parseArguments = (query, basepath = '/', useragent = '') => { // Parse url let url = query.url || ''; - url = url.replace('http://127.0.0.1/', basepath); - url = url.replace('http://127.0.0.1:32400/', basepath); - if (url && url[0] === '/') - url = basepath + url.substring(1); - if (query['X-Plex-Token']) { + url = url.replace('http://127.0.0.1/', '/'); + url = url.replace('http://127.0.0.1:32400/', '/'); + url = url.replace(basepath, '/'); + if (query['X-Plex-Token'] && url && url[0] === '/') { url += (url.indexOf('?') === -1) ? `?X-Plex-Token=${query['X-Plex-Token']}` : `&X-Plex-Token=${query['X-Plex-Token']}` } + if (url && url[0] === '/') + url = basepath + url.substring(1); // Extract parameters const params = { @@ -105,10 +106,15 @@ export const resize = (parameters, headers = {}) => { } // Resize based on width - if (params.minSize === 1) - s.resize(params.width, null, opt); - else - s.resize(null, params.height, opt); + try { + if (params.minSize === 1) + s.resize(params.width, null, opt); + else + s.resize(null, params.height, opt); + } + catch (e) { + return reject(e) + } // Background & opacity support if (params.background && params.opacity) { From 63034e15aa13415a6dc9614580d48054829c7ff5 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 19:03:20 +0100 Subject: [PATCH 130/132] Add debug --- src/core/sessions.js | 1 + src/routes/api.js | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/core/sessions.js b/src/core/sessions.js index 7ea5c81..35551d4 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -135,6 +135,7 @@ SessionsManager.parseFFmpegParameters = async (args = [], env = {}) => { // Store the FFMPEG parameters in RedisCache SessionsManager.storeFFmpegParameters = (args, env) => { const parsed = SessionsManager.parseFFmpegParameters(args, env); + console.log('FFMPEG', parsed.session, parsed); SessionStore.set(parsed.session, parsed).then(() => { }).catch(() => { }) return (parsed); }; diff --git a/src/routes/api.js b/src/routes/api.js index 20054c9..edf9bd4 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -25,8 +25,10 @@ RoutesAPI.update = (req, res) => { // Save the FFMPEG arguments // Body: {args: [], env: []} RoutesAPI.ffmpeg = (req, res) => { + console.log('FFMPEG CALLED 1') if (!req.body || !req.body.arg || !req.body.env) return (res.status(400).send({ error: { code: 'INVALID_ARGUMENTS', message: 'Invalid UnicornFFMPEG parameters' } })); + console.log('FFMPEG CALLED 2') return (res.send(SessionsManager.storeFFmpegParameters(req.body.arg, req.body.env))); }; From c209c32b715ac405b4697b4b14bcd3ea844f8c93 Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Fri, 22 Feb 2019 19:09:43 +0100 Subject: [PATCH 131/132] Fix ffmpeg --- src/core/sessions.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/sessions.js b/src/core/sessions.js index 35551d4..fe43081 100644 --- a/src/core/sessions.js +++ b/src/core/sessions.js @@ -133,8 +133,8 @@ SessionsManager.parseFFmpegParameters = async (args = [], env = {}) => { }; // Store the FFMPEG parameters in RedisCache -SessionsManager.storeFFmpegParameters = (args, env) => { - const parsed = SessionsManager.parseFFmpegParameters(args, env); +SessionsManager.storeFFmpegParameters = async (args, env) => { + const parsed = await SessionsManager.parseFFmpegParameters(args, env); console.log('FFMPEG', parsed.session, parsed); SessionStore.set(parsed.session, parsed).then(() => { }).catch(() => { }) return (parsed); From 3a48827ce2aae396702550b0081b6e2519a7423e Mon Sep 17 00:00:00 2001 From: Maxime Baconnais Date: Sat, 23 Feb 2019 20:39:31 +0100 Subject: [PATCH 132/132] Fix images crash --- src/core/images.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/images.js b/src/core/images.js index 22d5918..c2746cd 100644 --- a/src/core/images.js +++ b/src/core/images.js @@ -92,7 +92,7 @@ export const resize = (parameters, headers = {}) => { // Load body let s = false; try { - s = sharp(body); + s = sharp(body).on('error', err => { return reject(err); }); } catch (e) { return reject(e) @@ -121,7 +121,7 @@ export const resize = (parameters, headers = {}) => { let bgd = false; try { const buff = await s.png().toBuffer(); - s = sharp(buff); + s = sharp(buff).on('error', err => { return reject(err); }); const meta = await s.metadata(); bgd = await sharp({ create: { @@ -135,7 +135,7 @@ export const resize = (parameters, headers = {}) => { alpha: ((100 - params.opacity) / 100) } } - }).png().toBuffer(); + }).on('error', err => { return reject(err); }).png().toBuffer(); } catch (e) { return reject(e)