diff --git a/build.gradle.kts b/build.gradle.kts index 2b99cca0..830c8142 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -28,7 +28,7 @@ plugins { } group = "io.github.udhayarajan" -version = "5.6.10" +version = "5.6.11" // Version Naming incremented if ".." // Priority on incrementing Feature > BugFix > Beta diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/M3u8.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/M3u8.kt index 08d1ab9e..328b8a40 100644 --- a/src/commonMain/kotlin/com/mugames/vidsnapkit/M3u8.kt +++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/M3u8.kt @@ -35,11 +35,16 @@ class M3u8(private val url: String, private val onProgressCallback: ProgressCall var info: JSONObject? = null - val httpRequestService = HttpRequestService.create() + val httpRequestService by lazy { + HttpRequestService.create() + } - private val localFormats = Formats() + private val localFormats by lazy { + Formats() + } private fun nonFatalError(msg: String) { + httpRequestService.close() onProgressCallback.onProgress(Result.Failed(Error.NonFatalError(msg))) } diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/DailyMotion.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/DailyMotion.kt index 4c879f7a..474fe674 100644 --- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/DailyMotion.kt +++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/DailyMotion.kt @@ -95,7 +95,7 @@ class DailyMotion(url: String) : Extractor(url) { } } } ?: run { - onProgress(Result.Failed(Error.InvalidUrl)) + failed(Error.InvalidUrl) } } diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Extractor.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Extractor.kt index 2f028f6e..f918fe29 100644 --- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Extractor.kt +++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Extractor.kt @@ -98,7 +98,9 @@ abstract class Extractor( protected lateinit var onProgress: (Result) -> Unit protected var headers: Hashtable = Hashtable() - private val store = AcceptAllCookiesStorage() + private val store by lazy { + AcceptAllCookiesStorage() + } protected var httpRequestService = run { val str = if (inputUrl.contains(Regex("/reels/audio/|tiktok"))) store else null @@ -215,6 +217,7 @@ abstract class Extractor( filteredFormats.forEach { it.cookies.addAll(store.get(Url(inputUrl))) } + httpRequestService.close() onProgress(Result.Success(filteredFormats)) } } @@ -250,18 +253,27 @@ abstract class Extractor( } protected fun clientRequestError(msg: String = "error making request") { + httpRequestService.close() onProgress(Result.Failed(Error.NonFatalError(msg))) } + public fun failed(error: Error) { + httpRequestService.close() + onProgress(Result.Failed(error)) + } + protected fun loginRequired() { + httpRequestService.close() onProgress(Result.Failed(Error.LoginRequired)) } protected fun internalError(msg: String, e: Exception? = null) { + httpRequestService.close() onProgress(Result.Failed(Error.InternalError(msg, e))) } protected fun missingLogic() { + httpRequestService.close() onProgress(Result.Failed(Error.MethodMissingLogic)) } diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Facebook.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Facebook.kt index aa08551a..bf3d77a1 100644 --- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Facebook.kt +++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Facebook.kt @@ -105,7 +105,7 @@ class Facebook internal constructor(url: String) : Extractor(url) { extractInfo() } catch (e: JSONException) { e.printStackTrace() - onProgress(Result.Failed(Error.InternalError("Something went wrong", e))) + internalError("Something went wrong", e) } catch (e: Exception) { logger.warn("$TAG+ analyze: ", e) throw e @@ -168,7 +168,7 @@ class Facebook internal constructor(url: String) : Extractor(url) { return } if (webPage.contains("You must log in to continue")) { - onProgress(Result.Failed(Error.LoginRequired)) + loginRequired() return } } @@ -226,7 +226,7 @@ class Facebook internal constructor(url: String) : Extractor(url) { } ?: apply { val uuid = "fb." + UUID.randomUUID().toString() + ".html" File(uuid).writeText(webPage) - onProgress(Result.Failed(Error.NonFatalError("Sorry! we can't see the page, refer=$uuid"))) + clientRequestError("Sorry! we can't see the page, refer=$uuid") } } @@ -387,7 +387,9 @@ class Facebook internal constructor(url: String) : Extractor(url) { return extractFromCreationStory(media) } val scopedFormats = localFormats.copy( - title = "", videoData = mutableListOf(), audioData = mutableListOf(), imageData = mutableListOf() + videoData = mutableListOf(), + audioData = mutableListOf(), + imageData = mutableListOf() ) if (media.getNullableJSONObject("video_grid_renderer") != null) { return getVideoFromVideoGridRenderer(media) @@ -402,10 +404,14 @@ class Facebook internal constructor(url: String) : Extractor(url) { url = thumbnailUrl ) ) - scopedFormats.title = media.getNullableString("name") ?: media.getNullableJSONObject("savable_description") - ?.getNullableString("text") ?: media.getNullableJSONObject("title")?.getString("text")?.ifEmpty { - "Facebook_Video" - } ?: "Facebook_Video" + val title = media.getNullableString("name") ?: media.getNullableJSONObject("savable_description") + ?.getNullableString("text") ?: media.getNullableJSONObject("title")?.getString("text") + title?.let { + if (scopedFormats.title.isEmpty() || it != scopedFormats.title) + scopedFormats.title = title + } ?: run { + if (scopedFormats.title.isEmpty()) scopedFormats.title = "Facebook_Video" + } val dashXml = media.getNullableString("dash_manifest") dashXml?.let { @@ -463,6 +469,9 @@ class Facebook internal constructor(url: String) : Extractor(url) { val playbackVideo = media.getNullableJSONObject("creation_story")?.getNullableJSONObject("short_form_video_context") ?.getNullableJSONObject("playback_video") + localFormats.title = media.getNullableJSONObject("creation_story") + ?.getNullableJSONObject("message") + ?.getNullableString("text") ?: "" return if (playbackVideo != null) parseGraphqlVideo(playbackVideo) else parseGraphqlVideo(media, false) } diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Instagram.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Instagram.kt index 913e60c7..a5d92138 100644 --- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Instagram.kt +++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Instagram.kt @@ -46,6 +46,7 @@ class Instagram internal constructor(url: String) : Extractor(url) { const val GRAPHQL_URL = "https://www.instagram.com/graphql/query/?query_hash=%s&variables={\"shortcode\":\"%s\"}&__a=1&__d=dis" const val DEFAULT_QUERY_HASH = "b3055c01b4b222b8a47dc12b090e4e64" + const val DEFAULT_APP_ID = "936619743392459" const val AUDIO_API = "https://www.instagram.com/api/v1/clips/music/" private val logger = LoggerFactory.getLogger(Instagram::class.java) } @@ -94,6 +95,22 @@ class Instagram internal constructor(url: String) : Extractor(url) { } } + private fun getAppID(page: String?): String { + if (page == null) + return DEFAULT_APP_ID + val appIdRegex = listOf( + "\"app_id\":\"(\\d.*?)\"".toRegex(), + "\"appId\":\"(\\d.*?)\"".toRegex(), + "\"APP_ID\":\"(\\d.*?)\"".toRegex(), + "\"X-IG-App-ID\":\"(.*?)\"".toRegex() + ) + for (regex in appIdRegex) { + val matcher = regex.find(page) + return matcher?.groups?.get(1)?.value ?: DEFAULT_APP_ID + } + return DEFAULT_APP_ID + } + private fun getMediaId(page: String? = null): String? { if (page == null) { val matcher = Pattern.compile("/([0-9]{19})(?:/|)").matcher(inputUrl) @@ -162,12 +179,12 @@ class Instagram internal constructor(url: String) : Extractor(url) { return null } if (!isAccessible(response, it)) { - onProgress(Result.Failed(Error.InvalidCookies)) + loginRequired() return null } return response.getJSONObject("graphql")?.getJSONObject("user")?.getString("id") } ?: run { - onProgress(Result.Failed(Error.InvalidUrl)) + failed(Error.InvalidUrl) } return null } ?: run { @@ -254,24 +271,12 @@ class Instagram internal constructor(url: String) : Extractor(url) { loginRequired() return } - var appId = "936619743392459" - withTimeoutOrNull(2000) { - httpRequestService.getResponse(inputUrl, headers)?.let { - val appIdRegex = listOf( - "\"app_id\":\"(\\d.*?)\"".toRegex(), - "\"appId\":\"(\\d.*?)\"".toRegex(), - "\"APP_ID\":\"(\\d.*?)\"".toRegex(), - "\"X-IG-App-ID\":\"(.*?)\"".toRegex() - ) - for (regex in appIdRegex) { - val matcher = regex.find(it) - matcher?.groups?.get(1)?.let { - appId = it.value - } - } - } - } + val appID = withTimeoutOrNull(1500) { + getAppID(httpRequestService.getResponse(inputUrl, headers)) + } ?: DEFAULT_APP_ID + + headers["X-Ig-App-Id"] = appID val tempHeader = headers.clone() as Hashtable pre.headers.getAll("set-cookie")?.forEach { @@ -281,7 +286,6 @@ class Instagram internal constructor(url: String) : Extractor(url) { } } tempHeader.remove("User-Agent") - tempHeader["X-Ig-App-Id"] = appId val res = httpRequestService.postRequest(AUDIO_API, tempHeader, audioPayload) val metadata = res?.toJSONObject()?.getJSONObject("metadata") metadata?.run { @@ -328,29 +332,7 @@ class Instagram internal constructor(url: String) : Extractor(url) { return } if (res == "429" && isCookieValid()) { - val mediaID = shortcodeToMediaID(getShortcode()) - mediaID?.let { - val items = - httpRequestService - .getResponse( - POST_API.format(shortcodeToMediaID(getShortcode())), - headers - )?.let { - it.toJSONObjectOrNull()?.getNullableJSONArray("items") ?: run { - loginRequired() - return - } - } ?: run { - loginRequired() - return - } - extractFromItems(items) - return - } ?: run { - logger.error("unable to find mediaID for url $inputUrl") - loginRequired() - return - } + shortcodeExtraction() } extractFromItems( res.toJSONObjectOrNull()?.getNullableJSONArray("items") ?: run { @@ -360,6 +342,33 @@ class Instagram internal constructor(url: String) : Extractor(url) { ) } + // Works only with valid cookies + private suspend fun shortcodeExtraction() { + val mediaID = shortcodeToMediaID(getShortcode()) + mediaID?.let { + val items = + httpRequestService + .getResponse( + POST_API.format(shortcodeToMediaID(getShortcode())), + headers + )?.let { + it.toJSONObjectOrNull()?.getNullableJSONArray("items") ?: run { + loginRequired() + return + } + } ?: run { + loginRequired() + return + } + extractFromItems(items) + return + } ?: run { + logger.error("unable to find mediaID for url $inputUrl") + loginRequired() + return + } + } + private suspend fun extractHighlights(highlightsId: String, isStory: Boolean = false) { val highlights = httpRequestService.getResponse( HIGHLIGHTS_API.format(if (!isStory) "highlight%3A$highlightsId" else highlightsId), @@ -455,15 +464,15 @@ class Instagram internal constructor(url: String) : Extractor(url) { if (isObjectPresentInEntryData("LoginAndSignupPage")) { loginRequired() } else if (isObjectPresentInEntryData("HttpErrorPage")) { - onProgress(Result.Failed(Error.Instagram404Error(cookies != null))) + failed(Error.Instagram404Error(cookies != null)) } else { val user0 = jsonObject.getJSONObject("entry_data").getNullableJSONArray("ProfilePage")?.getJSONObject(0) ?: run { newApiRequest() return } - if (!isAccessible(user0)) onProgress(Result.Failed(Error.InvalidCookies)) - else onProgress(Result.Failed(Error.InternalError("can't find problem"))) + if (!isAccessible(user0)) loginRequired() + else internalError("can't find problem") } } } @@ -655,7 +664,7 @@ class Instagram internal constructor(url: String) : Extractor(url) { media?.let { mediaIt -> setInfo(mediaIt) } ?: run { - onProgress(Result.Failed(Error.InternalError("MediaNotFound"))) + internalError("MediaNotFound") } } ?: run { extractFromItems(jsonObject.getJSONArray("items")) @@ -666,6 +675,8 @@ class Instagram internal constructor(url: String) : Extractor(url) { val queryHash = withTimeoutOrNull(2000) { getQueryHashFromAllJSInPage(page) } ?: DEFAULT_QUERY_HASH + val appID = getAppID(page) + headers["X-Ig-App-Id"] = appID val res = httpRequestService.getResponse(GRAPHQL_URL.format(queryHash, getShortcode()), headers) logger.info("graphQL response = $res") val shortcodeMedia = @@ -742,7 +753,7 @@ class Instagram internal constructor(url: String) : Extractor(url) { } } if (!isPostUrl() && videoFormats.isEmpty()) { - onProgress(Result.Failed(Error.NonFatalError(NO_VIDEO_STATUS_AVAILABLE))) + clientRequestError(NO_VIDEO_STATUS_AVAILABLE) } else finalize() } diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Likee.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Likee.kt index fd45cd2a..cff1b9c7 100644 --- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Likee.kt +++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Likee.kt @@ -54,7 +54,7 @@ class Likee internal constructor(url: String) : Extractor(url) { responseData.getJSONArray("videoList")?.let { extractVideoList(it) } ?: run { - onProgress(Result.Failed(Error.MethodMissingLogic)) + missingLogic() } } diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/LinkedIn.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/LinkedIn.kt index b9b0af09..b9613255 100644 --- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/LinkedIn.kt +++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/LinkedIn.kt @@ -106,10 +106,10 @@ class LinkedIn internal constructor(url: String) : Extractor(url) { } } if (page.contains("Please enter your email address", true)) { - onProgress(Result.Failed(Error.LoginRequired)) + loginRequired() return } - onProgress(Result.Failed(Error.MethodMissingLogic)) + missingLogic() } private suspend fun extractFromIncluded(included: JSONArray) { diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Periscope.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Periscope.kt index 887197fc..8a408aeb 100644 --- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Periscope.kt +++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/Periscope.kt @@ -18,9 +18,7 @@ package com.mugames.vidsnapkit.extractor import com.mugames.vidsnapkit.MimeType -import com.mugames.vidsnapkit.dataholders.Error import com.mugames.vidsnapkit.dataholders.Formats -import com.mugames.vidsnapkit.dataholders.Result import com.mugames.vidsnapkit.dataholders.VideoResource import com.mugames.vidsnapkit.getNullableString import com.mugames.vidsnapkit.toJSONObject @@ -80,7 +78,7 @@ class Periscope internal constructor(url: String) : Extractor(url) { if (videoUrl.isNullOrEmpty() || videUrls.contains(videoUrl)) continue localFormats.videoData.add(VideoResource(videoUrl, MimeType.VIDEO_MP4)) if (formatId != "rtmp") { - onProgress(Result.Failed(Error.MethodMissingLogic)) + missingLogic() break } } diff --git a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/ShareChat.kt b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/ShareChat.kt index c0dd67e6..c144a71d 100644 --- a/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/ShareChat.kt +++ b/src/commonMain/kotlin/com/mugames/vidsnapkit/extractor/ShareChat.kt @@ -52,7 +52,8 @@ class ShareChat internal constructor(url: String) : Extractor(url) { Pattern.compile("""