Merge pull request #224 from pablouser1/scraper_rewrite

v2.5.0.0: "Hopefully fixing the mess" edition (BREAKING CHANGES!)
pablouser1 · Aug 20, 2024 · 672e752 · 672e752
2 parents 358c6ad + 591bb2c
commit 672e752
Show file tree

Hide file tree

Showing 25 changed files with 1,696 additions and 405 deletions.
diff --git a/.env.example b/.env.example
@@ -2,17 +2,17 @@
 # LATTE_CACHE=/tmp/proxitok_api # Path for Latte cache, leave commented for ./cache/latte
 
 # API CONFIG
-# API_SIGNER="remote"
-# API_SIGNER_URL="http://localhost:8080"
-# API_TEST_ENDPOINTS=true # Uncomment for usage of testing TikTok endpoints, may help sometimes
+# API_CHROMEDRIVER=http://localhost:4444
+# API_VERIFYFP=verify_...
+# API_DEVICE_ID=586094...
 # API_CACHE=redis
 
-# Redis cache, used on Helpers\CacheEngines\RedisCache (CHOOSE ONE)
+# Redis cache, used on App\Cache\RedisCache (CHOOSE ONE)
 # REDIS_HOST=localhost # Host or path to unix socket
 # REDIS_PORT=6379 # Set to -1 to use unix socket
 # REDIS_PASSWORD=example # Leave commented for no password
 
-# JSON cache, used on Helpers\CacheEngines\JSONCache
+# JSON cache, used on App\Cache\JSONCache
 # API_CACHE_JSON=/tmp/proxitok_api # Path for JSON API Cache, leave commented for ./cache/api
 
 # PROXY="https://user:password@host:port" # Host to be used as proxy, make sure to type a valid protocol. Available: http:// | https:// | socks4:// | socks5://

diff --git a/Dockerfile b/Dockerfile
@@ -1,18 +1,18 @@
-FROM trafex/php-nginx:3.4.0
+FROM trafex/php-nginx:3.6.0
 
 # Add composer
 COPY --from=composer /usr/bin/composer /usr/bin/composer
 
 # Copy config
-COPY ./misc/setup/docker/php.ini /etc/php82/conf.d/settings.ini
+COPY ./misc/setup/docker/php.ini /etc/php83/conf.d/settings.ini
 
 USER root
 # Create cache folder
 RUN mkdir /cache && chown -R nobody:nogroup /cache
 # Install deps
-RUN apk add --no-cache php82-redis php82-zip php82-tokenizer
-USER nobody
+RUN apk add --no-cache php83-redis php83-zip php83-tokenizer
 
+USER nobody
 # Copy source
 COPY --chown=nobody . /var/www/html
 

diff --git a/README.md b/README.md
@@ -4,11 +4,10 @@ Use Tiktok with an alternative frontend, inspired by Nitter.
 ## Features
 * Privacy: All requests made to TikTok are server-side, so you will never connect to their servers
 * See user's feed
-* See trending and discovery tab
 * See tags
 * See video by id
 * Themes
-* RSS Feed for user, trending and tag (just add /rss to the url)
+* RSS Feed for user and tag (just add /rss to the url)
 
 ## Self-hosting
 Please check [this](https://github.com/pablouser1/ProxiTok/wiki/Self-hosting) wiki article for info on how to self-host your own instance
@@ -33,6 +32,7 @@ Apply to: Main window (address bar)
 ```
 
 ## TODO / Known issues
+* Fix Heroku, still using old signer method
 * Replace placeholder favicon
 * Make video on /video fit screen and don't overflow
 * Fix embed styling

diff --git a/app/Cache/ApcuCache.php b/app/Cache/ApcuCache.php
@@ -1,9 +1,9 @@
 <?php
 namespace App\Cache;
 
-use TikScraper\Interfaces\CacheInterface;
+use TikScraper\Interfaces\ICache;
 
-class ApcuCache implements CacheInterface {
+class ApcuCache implements ICache {
     function __construct() {
         if (!(extension_loaded('apcu') && apcu_enabled())) {
             throw new \Exception('APCu not enabled');
@@ -19,7 +19,7 @@ public function exists(string $cache_key): bool {
         return apcu_exists($cache_key);
     }
 
-    public function set(string $cache_key, string $data, $timeout = 3600): void {
+    public function set(string $cache_key, string $data, int $timeout = 3600): void {
         apcu_store($cache_key, $data, $timeout);
     }
 }
diff --git a/app/Cache/JSONCache.php b/app/Cache/JSONCache.php
@@ -1,9 +1,9 @@
 <?php
 namespace App\Cache;
 
-use TikScraper\Interfaces\CacheInterface;
+use TikScraper\Interfaces\ICache;
 
-class JSONCache implements CacheInterface {
+class JSONCache implements ICache {
     private string $cache_path = __DIR__ . '/../../cache/api';
 
     function __construct() {
@@ -26,7 +26,7 @@ public function exists(string $cache_key): bool  {
         return is_file($filename);
     }
 
-    public function set(string $cache_key, string $data, $timeout = 3600) {
+    public function set(string $cache_key, string $data, int $timeout = 3600) {
         file_put_contents($this->cache_path . '/' . $cache_key . '.json', $data);
     }
 }
diff --git a/app/Cache/RedisCache.php b/app/Cache/RedisCache.php
@@ -1,9 +1,9 @@
 <?php
 namespace App\Cache;
 
-use TikScraper\Interfaces\CacheInterface;
+use TikScraper\Interfaces\ICache;
 
-class RedisCache implements CacheInterface {
+class RedisCache implements ICache {
     private \Redis $client;
     function __construct(string $host, int $port, ?string $password) {
         $this->client = new \Redis();
@@ -30,7 +30,7 @@ public function exists(string $cache_key): bool {
         return $this->client->exists($cache_key);
     }
 
-    public function set(string $cache_key, string $data, $timeout = 3600) {
+    public function set(string $cache_key, string $data, int $timeout = 3600) {
         $this->client->set($cache_key, $data, $timeout);
     }
 }
diff --git a/app/Controllers/DiscoverController.php b/app/Controllers/DiscoverController.php
diff --git a/app/Controllers/ProxyController.php b/app/Controllers/ProxyController.php
@@ -3,7 +3,6 @@
 
 use App\Helpers\Cookies;
 use App\Helpers\Misc;
-use TikScraper\Constants\UserAgents as TikScraperUserAgents;
 
 class ProxyController {
     const VALID_TIKTOK_DOMAINS = [
@@ -13,15 +12,16 @@ class ProxyController {
     public static function stream() {
         self::checkUrl();
         $url = $_GET['url'];
-        $config['user_agent'] = Misc::env("USER_AGENT", TikScraperUserAgents::DEFAULT);
-        $streamer = new \TikScraper\Stream($config);
+
+        $options = Misc::getScraperOptions();
+        $streamer = new \TikScraper\Stream($options);
         $streamer->url($url);
     }
 
     public static function download() {
         self::checkUrl();
         $method = Cookies::downloader();
-        $downloader = new \TikScraper\Download($method);
+        $downloader = new \TikScraper\Download($method, Misc::getScraperOptions());
 
         // Params
         $id = $_GET['id'] ?? '';

diff --git a/app/Controllers/SettingsController.php b/app/Controllers/SettingsController.php
@@ -21,11 +21,6 @@ public static function general() {
 
     public static function api() {
         // TODO, ADD COUNT
-        if (isset($_POST['api-test_endpoints'])) {
-            $test_endpoints = $_POST['api-test_endpoints'];
-            Cookies::set('api-test_endpoints', $test_endpoints);
-        }
-
         if (isset($_POST['api-downloader'])) {
             $downloader = $_POST['api-downloader'];
             Cookies::set("api-downloader", $downloader);

diff --git a/app/Controllers/TrendingController.php b/app/Controllers/TrendingController.php
diff --git a/app/Helpers/Misc.php b/app/Helpers/Misc.php
@@ -25,6 +25,31 @@ public static function getView(string $template): string {
         return __DIR__ . "/../../templates/views/{$template}.latte";
     }
 
+    public static function getScraperOptions(): array {
+        $url = self::env('API_CHROMEDRIVER', 'http://localhost:4444');
+        $verifyFp = self::env('API_VERIFYFP', '');
+        $device_id = self::env('API_DEVICE_ID', '');
+        $ua = self::env('USER_AGENT', '');
+
+        $options = [
+            'browser' => [
+                'url' => $url,
+                'close_when_done' => false
+            ],
+            'verify_fp' => $verifyFp,
+            'device_id' => $device_id,
+            'user_agent' => $ua
+        ];
+
+        $proxy = Misc::env('PROXY', '');
+
+        if ($proxy !== '') {
+            $options['proxy'] = $proxy;
+        }
+
+        return $options;
+    }
+
     /**
      * Common method for rss feeds
      */

diff --git a/app/Helpers/Wrappers.php b/app/Helpers/Wrappers.php
@@ -24,7 +24,7 @@ public static function latte(string $template, BaseTemplate $base) {
         });
 
         // Static assets
-        $latte->addFunction('static', function (string $type, string $file, bool $isVendor = false): string {
+        $latte->addFunction('assets', function (string $type, string $file, bool $isVendor = false): string {
             $endpoint = '';
             switch ($type) {
                 case 'js':
@@ -34,7 +34,7 @@ public static function latte(string $template, BaseTemplate $base) {
                     $endpoint .= '/styles';
                     break;
                 default:
-                    throw new \Exception('Invalid static asset type');
+                    throw new \Exception('Invalid asset type');
             }
 
             if ($isVendor) $endpoint .= '/vendor';
@@ -127,35 +127,7 @@ public static function latte(string $template, BaseTemplate $base) {
      * Setup of TikTok Api wrapper
      */
     public static function api(): \TikScraper\Api {
-        $method = Misc::env('API_SIGNER', '');
-        $url = Misc::env('API_SIGNER_URL', '');
-        if (!$method) {
-            // Legacy support
-            $browser_url = Misc::env('API_BROWSER_URL', '');
-            if ($url) {
-                $method = 'remote';
-            } elseif ($browser_url) {
-                $url = $browser_url;
-                $method = 'browser';
-            }
-        }
-
-        $options = [
-            'use_test_endpoints' => Misc::env('API_TEST_ENDPOINTS', false) || isset($_COOKIE['api-test_endpoints']) && $_COOKIE['api-test_endpoints'] === 'yes',
-            'signer' => [
-                'method' => $method,
-                'url' => $url,
-                'close_when_done' => false
-            ]
-        ];
-
-        // -- PROXY CONFIG -- //
-        $proxy = Misc::env('PROXY', '');
-
-        if ($proxy !== '') {
-            $options['proxy'] = $proxy;
-        }
-
+        $options = Misc::getScraperOptions();
         // Cache config
         $cacheEngine = null;
         if (isset($_ENV['API_CACHE'])) {
@@ -179,19 +151,13 @@ public static function api(): \TikScraper\Api {
                     } else {
                         $host = $_ENV['REDIS_HOST'];
                         $port = intval($_ENV['REDIS_PORT']);
-                        $password = isset($_ENV['REDIS_PASSWORD']) ? $_ENV['REDIS_PASSWORD'] : null;
+                        $password = $_ENV['REDIS_PASSWORD'] ?? null;
                     }
                     $cacheEngine = new RedisCache($host, $port, $password);
                     break;
             }
         }
 
-        $customUa = Misc::env("USER_AGENT", '');
-
-        if ($customUa) {
-            $options['user_agent'] = $customUa;
-        }
-
         return new \TikScraper\Api($options, $cacheEngine);
     }
 }
diff --git a/app/Models/SettingsTemplate.php b/app/Models/SettingsTemplate.php
@@ -11,7 +11,6 @@
 class SettingsTemplate extends BaseTemplate {
     public array $downloaders = [];
     public array $themes = [];
-    public bool $isTestEndpoints = false;
     public string $currentDownloader;
     public string $currentTheme;
     public bool $isServiceWorker = false;
@@ -25,7 +24,6 @@ function __construct() {
         $ref = new \ReflectionClass(Themes::class);
         $this->themes = $ref->getConstants();
         // Cookies data
-        $this->isTestEndpoints = Cookies::check('api-test_endpoints', 'yes');
         $this->currentDownloader = Cookies::downloader();
         $this->currentTheme = Cookies::theme();
         $this->isServiceWorker = Cookies::check('misc-sw', 'yes');

diff --git a/composer.json b/composer.json
@@ -1,7 +1,7 @@
 {
     "name": "pablouser1/proxitok",
     "description": "An alternative frontend for TikTok",
-    "version": "2.4.9.4",
+    "version": "2.5.0.0",
     "license": "AGPL-3.0-or-later",
     "type": "project",
     "authors": [
@@ -14,18 +14,20 @@
 	"config": {
 		"optimize-autoloader": true,
 		"platform": {
-			"php": "7.4"
+			"php": "8.1"
 		}
 	},
     "require": {
-        "php": ">=7.4|^8.0",
-        "ext-redis": "^5.3.2|^6.0.0",
+        "php": "^8.1",
         "ext-mbstring": "*",
-        "latte/latte": "^2.11",
+        "latte/latte": "^3.0",
         "bramus/router": "^1.6",
-        "pablouser1/tikscraper": "^2.5",
+        "pablouser1/tikscraper": "^2.6",
         "josegonzalez/dotenv": "^4.0"
     },
+    "suggest": {
+        "ext-redis": "For using redis as a caching engine"
+    },
     "autoload": {
         "psr-4": {
             "App\\": "app/"