From 12a8eadd7f1fc0e3f7edfa571f624e451885ac5e Mon Sep 17 00:00:00 2001
From: udhayarajan <mudhayarajan.2013@gmail.com>
Date: Tue, 27 Jun 2023 05:36:10 +0530
Subject: [PATCH] Added way to test HTML & forcing all Instagram request with
 header

---
 build.gradle.kts                              |  2 +-
 .../vidsnapkit/extractor/DailyMotion.kt       |  4 +++
 .../mugames/vidsnapkit/extractor/Extractor.kt | 13 +++++----
 .../mugames/vidsnapkit/extractor/Facebook.kt  |  2 +-
 .../mugames/vidsnapkit/extractor/Instagram.kt | 27 ++++++++++---------
 .../com/mugames/vidsnapkit/extractor/Likee.kt |  4 +++
 .../mugames/vidsnapkit/extractor/LinkedIn.kt  |  4 +++
 .../mugames/vidsnapkit/extractor/Periscope.kt |  4 +++
 .../mugames/vidsnapkit/extractor/ShareChat.kt |  4 +++
 .../mugames/vidsnapkit/extractor/TikTok.kt    |  2 +-
 .../mugames/vidsnapkit/extractor/Twitch.kt    |  4 +++
 .../mugames/vidsnapkit/extractor/Twitter.kt   |  4 +++
 .../com/mugames/vidsnapkit/extractor/Vimeo.kt |  4 +++
 .../mugames/vidsnapkit/network/HttpRequest.kt | 10 +++----
 14 files changed, 61 insertions(+), 27 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 2238dee0..a703f988 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -27,7 +27,7 @@ plugins {
 }
 
 group = "io.github.udhayarajan"
-version = "5.3.9"
+version = "5.3.10"
 //Version Naming incremented if "<NEW_FEATURE_ADDED>.<WORKED_ON_BUG>.<BETA_VERSION_COUNT>"
 //Priority on incrementing Feature > BugFix > Beta
 
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/DailyMotion.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/DailyMotion.kt
index dc9c050c..c663428f 100644
--- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/DailyMotion.kt
+++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/DailyMotion.kt
@@ -107,6 +107,10 @@ class DailyMotion(url: String) : Extractor(url) {
         }
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun extractFromM3U8(response: String) {
         fun valueForKey(key: String, line: String): String? {
             val matcher = Pattern.compile("$key=(?:\"(.*?)\"|(.*?),)").matcher(line)
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Extractor.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Extractor.kt
index baa8bc33..104d2b5a 100644
--- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Extractor.kt
+++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Extractor.kt
@@ -119,11 +119,12 @@ abstract class Extractor(
 
     private suspend fun safeAnalyze() {
         try {
-            if (inputUrl.contains("instagram"))
+            if (inputUrl.contains("instagram")) {
                 inputUrl = if (cookies == null) {
                     inputUrl.replace("/reels/", "/reel/")
                 } else inputUrl.replace("/reel/", "/reels/")
-
+                headers["User-Agent"] = "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Instagram 105.0.0.11.118 (iPhone11,8; iOS 12_3_1; en_US; en-US; scale=2.00; 828x1792; 165586599)"
+            }
             if (HttpRequest(inputUrl, headers).isAvailable())
                 analyze()
             else if (inputUrl.contains("instagram") && cookies != null) {
@@ -180,7 +181,7 @@ abstract class Extractor(
         val sizes = mutableListOf<Deferred<Long>>()
         coroutineScope {
             for (videoData in format.videoData) {
-                sizes.add(async { HttpRequest(videoData.url).getSize() })
+                sizes.add(async { HttpRequest(videoData.url, headers).getSize() })
             }
         }
         return sizes.awaitAll()
@@ -190,7 +191,7 @@ abstract class Extractor(
         val sizes = mutableListOf<Deferred<Long>>()
         coroutineScope {
             for (audioData in format.audioData) {
-                sizes.add(async { HttpRequest(audioData.url).getSize() })
+                sizes.add(async { HttpRequest(audioData.url, headers).getSize() })
             }
         }
         return sizes.awaitAll()
@@ -200,7 +201,7 @@ abstract class Extractor(
         val sizes = mutableListOf<Deferred<Long>>()
         coroutineScope {
             for (imageData in format.imageData) {
-                sizes.add(async { HttpRequest(imageData.url).getSize() })
+                sizes.add(async { HttpRequest(imageData.url, headers).getSize() })
             }
         }
         return sizes.awaitAll()
@@ -213,4 +214,6 @@ abstract class Extractor(
     protected inline fun internalError(msg: String, e: Exception? = null) {
         onProgress(Result.Failed(Error.InternalError(msg, e)))
     }
+
+    abstract suspend fun testWebpage(string: String)
 }
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Facebook.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Facebook.kt
index 9e6f1b7e..d50b9b66 100644
--- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Facebook.kt
+++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Facebook.kt
@@ -523,7 +523,7 @@ class Facebook internal constructor(url: String) : Extractor(url) {
         return null
     }
 
-    suspend fun testWebpage(string: String) {
+    override suspend fun testWebpage(string: String) {
         onProgress = {
 
         }
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Instagram.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Instagram.kt
index ec6ab136..6cc0ff8a 100644
--- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Instagram.kt
+++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Instagram.kt
@@ -156,14 +156,14 @@ class Instagram internal constructor(url: String) : Extractor(url) {
                 inputUrl = inputUrl.replace("https://instagram.com", "https://www.instagram.com")
                 logger.info("The new url is $inputUrl&__a=1&__d=dis")
                 val items =
-                    HttpRequest(inputUrl.plus("&__a=1&__d=dis"), getHeadersWithUserAgent()).getResponse(true) ?: run {
+                    HttpRequest(inputUrl.plus("&__a=1&__d=dis"), headers).getResponse(true) ?: run {
                         clientRequestError("unable to get post event with __a=1&__d=dis")
                         return
                     }
                 extractFromItems(items.toJSONObject().getJSONArray("items"))
                 return
             }
-            extractInfoShared(HttpRequest(inputUrl, getHeadersWithUserAgent()).getResponse() ?: run {
+            extractInfoShared(HttpRequest(inputUrl, headers).getResponse() ?: run {
                 clientRequestError()
                 return
             })
@@ -182,7 +182,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
     }
 
     private suspend fun extractHighlights(highlightsId: String) {
-        val highlights = HttpRequest(HIGHLIGHTS_API.format("%3A$highlightsId"), getHeadersWithUserAgent()).getResponse()
+        val highlights = HttpRequest(HIGHLIGHTS_API.format("%3A$highlightsId"), headers).getResponse()
             ?.toJSONObjectOrNull()
         highlights?.let {
             if (it.getNullable("login_required") == "true") {
@@ -197,8 +197,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
 
     @Suppress("UNCHECKED_CAST")
     private suspend fun extractStories(userId: String) {
-        val dupHeader = getHeadersWithUserAgent()
-        val stories = HttpRequest(STORIES_API.format(userId), dupHeader).getResponse()
+        val stories = HttpRequest(STORIES_API.format(userId), headers).getResponse()
         val reel = JSONObject(stories).getNullableJSONObject("reel")
         reel?.let { extractFromItems(it.getJSONArray("items")) } ?: onProgress(
             Result.Failed(
@@ -209,12 +208,6 @@ class Instagram internal constructor(url: String) : Extractor(url) {
         )
     }
 
-    private fun getHeadersWithUserAgent(): Hashtable<String, String> {
-        val dupHeader: Hashtable<String, String> = headers.clone() as Hashtable<String, String>
-        dupHeader["User-Agent"] =
-            "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Instagram 105.0.0.11.118 (iPhone11,8; iOS 12_3_1; en_US; en-US; scale=2.00; 828x1792; 165586599)"
-        return dupHeader
-    }
 
 
     private suspend fun extractInfoShared(page: String) {
@@ -228,7 +221,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
                 if (mediaId == null) throw JSONException("mediaId is null purposely thrown wrong error")
                 val url = POST_API.format(mediaId)
                 extractFromItems(
-                    HttpRequest(url, getHeadersWithUserAgent())
+                    HttpRequest(url, headers)
                         .getResponse()
                         .toString()
                         .toJSONObject()
@@ -378,7 +371,7 @@ class Instagram internal constructor(url: String) : Extractor(url) {
         val matcher = Pattern.compile("<link rel=\"preload\" href=\"(.*?)\" as=\"script\"").matcher(page)
         while (matcher.find()) {
             ids.addAll(
-                getQueryHash(HttpRequest(matcher.group(1)).getResponse() ?: run {
+                getQueryHash(HttpRequest(matcher.group(1), headers).getResponse() ?: run {
                     return null
                 })
             )
@@ -560,4 +553,12 @@ class Instagram internal constructor(url: String) : Extractor(url) {
         } else
             finalize()
     }
+
+    override suspend fun testWebpage(string: String) {
+        onProgress = {
+            println(it)
+        }
+        extractInfoShared(string)
+    }
+
 }
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Likee.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Likee.kt
index 2ac1ac3c..00b1fa42 100644
--- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Likee.kt
+++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Likee.kt
@@ -57,6 +57,10 @@ class Likee internal constructor(url: String) : Extractor(url) {
         }
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun extractVideoList(jsonArray: JSONArray) {
         for (i in 0 until jsonArray.length()) {
             val localFormats = formats.copy(title = "", videoData = mutableListOf(), imageData = mutableListOf())
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/LinkedIn.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/LinkedIn.kt
index 8701a3f9..9a50b6e2 100644
--- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/LinkedIn.kt
+++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/LinkedIn.kt
@@ -42,6 +42,10 @@ class LinkedIn internal constructor(url: String) : Extractor(url) {
         })
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun scratchWebpage(page: String) {
         val matcher = Pattern.compile("data-sources=\"(.*?)\"").matcher(page)
         if (matcher.find()) {
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Periscope.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Periscope.kt
index 451f86aa..4632d1e0 100644
--- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Periscope.kt
+++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Periscope.kt
@@ -43,6 +43,10 @@ class Periscope internal constructor(url: String) : Extractor(url) {
         TODO("Not yet implemented")
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     var manifest: ArrayList<ArrayList<String>>? = ArrayList()
 
 
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/ShareChat.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/ShareChat.kt
index 09e15e07..81e935a3 100644
--- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/ShareChat.kt
+++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/ShareChat.kt
@@ -42,6 +42,10 @@ class ShareChat internal constructor(url: String) : Extractor(url) {
         })
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun scratchWebPage(response: String) {
         val matcher =
             Pattern.compile("""<script data-rh="true" type="application\/ld\+json">(\{"@context":"http:\/\/schema\.org","@type":"(?:Image|Video)Object".*?\})<\/script>""")
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/TikTok.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/TikTok.kt
index e19da07f..ca7a6e75 100644
--- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/TikTok.kt
+++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/TikTok.kt
@@ -80,7 +80,7 @@ class TikTok internal constructor(url: String) : Extractor(url) {
             onProgress(Result.Failed(Error.MethodMissingLogic))
     }
 
-    suspend fun testWithWebPage(string: String){
+    override suspend fun testWebpage(string: String) {
         onProgress = {
             println(it)
         }
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitch.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitch.kt
index 4e8ee735..0667a833 100644
--- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitch.kt
+++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitch.kt
@@ -38,6 +38,10 @@ class Twitch internal constructor(url: String) : Extractor(url) {
         TODO("Not yet implemented")
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private fun getId(s: String?): String? {
         return s?.run {
             val matcher =
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitter.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitter.kt
index ba455cc2..3d2d1800 100644
--- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitter.kt
+++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Twitter.kt
@@ -235,6 +235,10 @@ class Twitter internal constructor(url: String) : Extractor(url) {
         super.finalize()
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
 
     fun resolution(url: String?): String {
         val pattern: Pattern = Pattern.compile("/(\\d+x\\d+)/")
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Vimeo.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Vimeo.kt
index 67746b57..197ac8aa 100644
--- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Vimeo.kt
+++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Vimeo.kt
@@ -64,6 +64,10 @@ class Vimeo internal constructor(url: String) : Extractor(url) {
         }
     }
 
+    override suspend fun testWebpage(string: String) {
+        TODO("Not yet implemented")
+    }
+
     private suspend fun parseConfigRequest(response: String) {
         val json = response.toJSONObject()
         val hls = json.getJSONObject("request").getJSONObject("files").getJSONObject("dash")
diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/network/HttpRequest.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/network/HttpRequest.kt
index ab78ad05..18a65a4d 100644
--- a/src/commonMain/kotlin/com/mugames/vidsnapkit/network/HttpRequest.kt
+++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/network/HttpRequest.kt
@@ -42,13 +42,11 @@ class HttpRequest(
     companion object {
         private var prefixUrl = ""
         private var additionHeader: Hashtable<String, String>? = null
-        private fun defaultClient(requiresRedirection: Boolean = true) = HttpClient(Android).also {
-            it.config {
+        private fun defaultClient(requiresRedirection: Boolean = true) = HttpInterfaceImpl(
+            HttpClient(Android).config {
                 followRedirects = requiresRedirection
             }
-        }.run {
-            HttpInterfaceImpl(this)
-        }
+        )
 
         private var clientGenerator: () -> HttpClient = {
             HttpClient(Android)
@@ -104,7 +102,7 @@ class HttpRequest(
      * @return bytes count of given [url]
      */
     suspend fun getSize(useCustomClient: Boolean = true) =
-        (if (useCustomClient) createClient() else defaultClient()).getSize(url)
+        (if (useCustomClient) createClient() else defaultClient()).getSize(url, getHeader())
 
     suspend fun postRequest(postData: Hashtable<String, Any>? = null, useCustomClient: Boolean = true): String =
         withContext(Dispatchers.IO) {