diff --git a/.gitignore b/.gitignore index 4c7ac11..4df547c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ log/ config/cookies.json config/downloadme.txt -config/galleries.json config/headers.json Dockerfile tests/test.py \ No newline at end of file diff --git a/config/settings.json b/config/settings.json index 6a8e742..fb85a3c 100644 --- a/config/settings.json +++ b/config/settings.json @@ -1,4 +1,5 @@ { - "dest_path": "./hentai/", - "nhentai_tag": "language:english" + "nhentai_tag": "language:english", + "library_path": "./hentai/", + "library_split": "100000" } \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index b835c06..8b0d81d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -20,13 +20,13 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte [[package]] name = "certifi" -version = "2023.7.22" +version = "2023.11.17" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, - {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, ] [[package]] @@ -158,17 +158,17 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] [[package]] name = "hypothesis" -version = "6.88.4" +version = "6.92.0" description = "A library for property-based testing" optional = false python-versions = ">=3.8" files = [ - {file = "hypothesis-6.88.4-py3-none-any.whl", hash = "sha256:3443eeb6aace5fb1567f6f23401152ca417310789b296df21633d8284be51510"}, - {file = "hypothesis-6.88.4.tar.gz", hash = "sha256:5c1126b8cb3cef06a6f4a3d2a721b6c0412090cc5fca15fca573f46d5f24e89e"}, + {file = "hypothesis-6.92.0-py3-none-any.whl", hash = "sha256:d4577f99b912acc725bea684899b7cb62591a0412e2446c618be0b4855995276"}, + {file = "hypothesis-6.92.0.tar.gz", hash = "sha256:65b72c7dc7da3e16144db54fe093c6b74a33631b933a8063eb754c5a61361ae6"}, ] [package.dependencies] -attrs = ">=19.2.0" +attrs = ">=22.2.0" sortedcontainers = ">=2.1.0,<3.0.0" [package.extras] @@ -189,23 +189,23 @@ zoneinfo = ["backports.zoneinfo (>=0.2.1)", "tzdata (>=2023.3)"] [[package]] name = "idna" -version = "3.4" +version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] [[package]] name = "img2pdf" -version = "0.5.0" +version = "0.5.1" description = "Convert images to PDF via direct JPEG inclusion." optional = false python-versions = "*" files = [ - {file = "img2pdf-0.5.0.tar.gz", hash = "sha256:ae6c19731bde2551356c178bf356ca118ac32a232c737a14b423f8039df3c24b"}, + {file = "img2pdf-0.5.1.tar.gz", hash = "sha256:73847e47242f4b5bd113c70049e03e03212936c2727cd2a8bf564229a67d0b95"}, ] [package.dependencies] @@ -228,13 +228,13 @@ files = [ [[package]] name = "kfsconfig" -version = "1.0.2" +version = "1.1.0" description = "" optional = false python-versions = ">=3.11.0,<4.0.0" files = [ - {file = "kfsconfig-1.0.2-py3-none-any.whl", hash = "sha256:37d9c556b4016575cffb9e45bb08f018bdc4bc1bee962808c17313eeec82f140"}, - {file = "kfsconfig-1.0.2.tar.gz", hash = "sha256:6c7e92f12bc1dddc4eb629ca0c2bef6c72832dfb1c4425baa5b4f336369ff6c2"}, + {file = "kfsconfig-1.1.0-py3-none-any.whl", hash = "sha256:e6f80028db4346cbf2e8303d8f8f6276f83f437c4702fd4f723292272023027b"}, + {file = "kfsconfig-1.1.0.tar.gz", hash = "sha256:f4db6a35813b81e84bcf56f1de42d6db111542aa0ee79caa6e54ccff9e825234"}, ] [package.dependencies] @@ -284,13 +284,13 @@ files = [ [[package]] name = "kfsmedia" -version = "2.3.3" +version = "2.3.4" description = "" optional = false python-versions = ">=3.11.0,<4.0.0" files = [ - {file = "kfsmedia-2.3.3-py3-none-any.whl", hash = "sha256:5f5d48b7fafa9b21b89f79ba8a1218f4e062d46209473b46cb6eb4319146bdd7"}, - {file = "kfsmedia-2.3.3.tar.gz", hash = "sha256:4bf8a1eb583fd19d7d1765bef9700473415f1cd978cd0ecae12de8c7ee9fc009"}, + {file = "kfsmedia-2.3.4-py3-none-any.whl", hash = "sha256:7ba914a5ce80372e132935a1e8e8f4e7f33ae86286dc460a0466356fa851b6f3"}, + {file = "kfsmedia-2.3.4.tar.gz", hash = "sha256:e6d61681448e26dee70517c6899575e6cfacb137a4661aa5917288282cf90961"}, ] [package.dependencies] @@ -431,54 +431,54 @@ files = [ [[package]] name = "pebble" -version = "5.0.3" +version = "5.0.5" description = "Threading and multiprocessing eye-candy." optional = false python-versions = ">=3.6" files = [ - {file = "Pebble-5.0.3-py3-none-any.whl", hash = "sha256:8274aa0959f387b368ede47666129cbe5d123f276a1bd9cafe77e020194b2141"}, - {file = "Pebble-5.0.3.tar.gz", hash = "sha256:bdcfd9ea7e0aedb895b204177c19e6d6543d9962f4e3402ebab2175004863da8"}, + {file = "Pebble-5.0.5-py3-none-any.whl", hash = "sha256:18e2d33aa5f128e0c2a813fb314605628b8bd2007e5f626464b1cd13345ae89f"}, + {file = "Pebble-5.0.5.tar.gz", hash = "sha256:3759c33256fa8d7bba044761f96225da411268cc4cf4a200f9e7b187763e16a5"}, ] [[package]] name = "pikepdf" -version = "8.7.1" +version = "8.9.0" description = "Read and write PDFs with Python, powered by qpdf" optional = false python-versions = ">=3.8" files = [ - {file = "pikepdf-8.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3956b2abda0ef86b37b5d49066c3adfd29d056ed96b1ef93bacbd269c78c5e13"}, - {file = "pikepdf-8.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7075c79fa5d85b0acf1536a1849bcbb4e661b33f476c7c6d81255df247b2ae1f"}, - {file = "pikepdf-8.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94737db4beb7391ca78627e461050c5dd3a7cb7526fd6bdd0a0db94780e8b103"}, - {file = "pikepdf-8.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e687681f26533fee69904403fdf4fc2a1f0e47480b5bfbba64139c07d72c9d2"}, - {file = "pikepdf-8.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3b719a40f1b08da1ff6fd87b3847d4ce033f8ffd81a13dca8c3c947a080c071d"}, - {file = "pikepdf-8.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:dd5ad4cb6244ac885512f5385f6fc055e2bfc174636189fc1408f138458380c8"}, - {file = "pikepdf-8.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8a1ac9e17d890b39f717b040eaebc79325ff23278756805220138f054599f9f0"}, - {file = "pikepdf-8.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e982dbf2d25df72975e2d336ca6a5a8565add156bfdba328beb67b324a1689e"}, - {file = "pikepdf-8.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94998ff1d15b0eaec76494266e2e61d80573c5ee4ac0860a9af4a02e5107fe17"}, - {file = "pikepdf-8.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2771cba79c62b3f9d93cffa684a92d60e6050f727d4f52220d8180eaf967c4bf"}, - {file = "pikepdf-8.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0f118446960213c78b21fe106890028dcaff739369bceda9008719b43670e290"}, - {file = "pikepdf-8.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:3ff178684ed43de28ca3e7dcf9da5082ebf49bcd042e9308ffaa0162275da1a4"}, - {file = "pikepdf-8.7.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:67276e1731dc8b2f58663ff62b4728f9e0046c8217768fe0649428315cfe8253"}, - {file = "pikepdf-8.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:42360df95f08164162af465c493d1e612d3c0ff6d23dc17be073380b05db715c"}, - {file = "pikepdf-8.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9da8d1db2d7e17411afef351ea9ccddbff36ce748b16003d4f065250c941436"}, - {file = "pikepdf-8.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7899f43a198d184c3d4787390a67c654a8c600c6c844f29b557b4f236d11619"}, - {file = "pikepdf-8.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ece366b8e68be47a7252c27f73dd9666df6871fb6e302a7bfd00a5cb293743f2"}, - {file = "pikepdf-8.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:d0678ed952a85f4b3b23d5ee94445e55a05aec0b47de37843625eba1f9b0fbb3"}, - {file = "pikepdf-8.7.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c11acef5c211af2559e1753ed96b3b4f30e7a9e26acad4b2cbd55ea3b25e3154"}, - {file = "pikepdf-8.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd88db0b68da1d04dd5a1ed0897e37ef1968317a887781b1e26c1649bbd11e3"}, - {file = "pikepdf-8.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3781b8dc868e3d452ce36348b26735d7a2fc0fe6a4503f72411e5bc9341a7fa"}, - {file = "pikepdf-8.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:e93d706b40cebc3b2fdaa9e814b4008ed4c659c6df03ee6a4ed46683aa21ab24"}, - {file = "pikepdf-8.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2a320bfa4caa73d00d90b04433305b3ff7a6c63bdf36f55af16725066fe46b49"}, - {file = "pikepdf-8.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3748c4d67b40c26fa777d1f3eefcab4bcca101395ad54502020653024d23c37d"}, - {file = "pikepdf-8.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08070185661db9dfbaec014094fec9e323d8ab882dc458a07383f55e08bfdd22"}, - {file = "pikepdf-8.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c631c6fd16b44a2636988d8be7202f1ffd3932f505e8b9044a6b88ef62411871"}, - {file = "pikepdf-8.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9430eb3439c0e9cb1b477115d5a34905f5c6be55422d66fcba55cedbf711ae89"}, - {file = "pikepdf-8.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:dc8f0116237d2f07fd8a8763f2c59f2fa50f445471f0c619914075ec411920e2"}, - {file = "pikepdf-8.7.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:80275466282495c672264801b50c7188f86590779f3ada8148336e3e84fd06c0"}, - {file = "pikepdf-8.7.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5075a2a741ba23212e963d9ea3fc7a4f9394535fab11fdc7ea4641d61cab0287"}, - {file = "pikepdf-8.7.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:897cac14f9e3db97d4effea50190e6c59e87d4147da98ff6108d254dcc1949d9"}, - {file = "pikepdf-8.7.1.tar.gz", hash = "sha256:69d69a93d07027e351996ef8232f26bba762d415206414e0ae3814be0aee0f7a"}, + {file = "pikepdf-8.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5ed450611299f63828abe2b07d2ebb1329076d330008f8cae17e3406338a6105"}, + {file = "pikepdf-8.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:78331c50d6552f49f0c8b18fcd7819c4ca28efea656b234a74b11a01c0625185"}, + {file = "pikepdf-8.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d49df3d4eefbe12bc9fc43e0edcb26af21c1c0f2ffc8943b91ea2d6c0f078357"}, + {file = "pikepdf-8.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84897208a93d395c96aeddbfd4f013c26c351a0be99cc162df7ede8b688268dc"}, + {file = "pikepdf-8.9.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:deb6c4b0fd93292e983284d0f9e4587cb3f200045ba051a82ded3f09535d1c02"}, + {file = "pikepdf-8.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ba11f40ba76124a6e420026fd6806dcc0a95a35b724ad32a74eb5c162622ae06"}, + {file = "pikepdf-8.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7ba0707d646eb0347aa2d90bc181fe557edeb445489a64b16b77d176e689950f"}, + {file = "pikepdf-8.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d45b14c750b276fbd22415b032d6d13ab3791e782eb197f890c332aa3350312b"}, + {file = "pikepdf-8.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3501e0497e30711c44f9258cefed6242eadd951ce54f5d8aac7ed6c4258c996"}, + {file = "pikepdf-8.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc13929f51df3d37d2add8d7dfba92fbe7605909661c865bfb9ee78d502454da"}, + {file = "pikepdf-8.9.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:806ac911b15b56c8c07c2d6b3eeb06ea258afe7e01190c0cced4c7f854358030"}, + {file = "pikepdf-8.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:3b9df18a3f549008d85b8146ef1eca98353466c22ab7a1cbd2634d64aaa65f95"}, + {file = "pikepdf-8.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f4b84e4cdeeca2a0ed04b751d3fdd6e5596ab76d10b9a8739e74a2e487fb1deb"}, + {file = "pikepdf-8.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:85f7116ccc043e3c2f3512dc7e4c6b005f3b47e1299ea8532b63f2ae143484d1"}, + {file = "pikepdf-8.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:def290fde46596a0da112fc349b2a5f578703d7a91367e9814a706129fcad7d4"}, + {file = "pikepdf-8.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7eb572e76b2a7c89d8975f30c7c7e234c219e50a35612c2cc7e23930352ead9"}, + {file = "pikepdf-8.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7f127f4840964a269868b8257466123e28314fbe170238ac68d1e99b0d48378d"}, + {file = "pikepdf-8.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:eee6e346220bc47aed633ff2bea29bf3a3929ec6dc47ffffa5b7a6de638dc253"}, + {file = "pikepdf-8.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a043bccc711594a44e3e6dfeab75ce0e333161a0f15b27ee1dc0a7452c0eb55"}, + {file = "pikepdf-8.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c41d67144d566ed55dd46e69d352054134fbc524f013b236dfb6aa42959167b2"}, + {file = "pikepdf-8.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ff69c360f30943a09106c324a256bcdbc19e443a6cad453c7298f44855e721"}, + {file = "pikepdf-8.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:807de4e920f34ab37dbc55a0a7f6516e200a9e700205a9e436aca4fe7060442e"}, + {file = "pikepdf-8.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:56bf07d424f5f062c9a802a2f39671976ac402ce059da3f2b128e34a7de86e72"}, + {file = "pikepdf-8.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:acf38720fe96547a1fd9414844f572164cc59c512d84e07c15254df01f0520a5"}, + {file = "pikepdf-8.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61a4bb07ef793fd377c59ddd81357b85d29df72fb810ac037cce151c5955e813"}, + {file = "pikepdf-8.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:194b1738b7f53fba041f01f1cfcd05c81d4f98d0093e28537004530563cb0af9"}, + {file = "pikepdf-8.9.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a66126e33cdbca3c0eb7028329645332b12837e4355d26d6327d6645c0f8eaa2"}, + {file = "pikepdf-8.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:3bb534d5e97221d7d5d732675be2074b98d37ba8b51f387ce0e9554ad33d8905"}, + {file = "pikepdf-8.9.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:eb76d011799c2c91b7b9ae5f864648ffb991283903a3b479574cfdfafffdf343"}, + {file = "pikepdf-8.9.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcc814648ae92933f3cb6bed39e9378d151e884783c386793d2910d30291728e"}, + {file = "pikepdf-8.9.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7aec1f19b616deb049f99048208f9e077e28403a45f6bfd03898babe9abf18a1"}, + {file = "pikepdf-8.9.0.tar.gz", hash = "sha256:65c34ed3d9834d01ec0622576af2636f162eaf1a2c1ddf9bc19daf8641c71389"}, ] [package.dependencies] @@ -489,7 +489,7 @@ Pillow = ">=10.0.1" [package.extras] dev = ["pre-commit", "typer[all]"] -docs = ["GitPython", "PyGithub", "Sphinx (>=3)", "ipython", "matplotlib", "pybind11", "requests", "sphinx-design", "sphinx-issues", "sphinx-rtd-theme", "tomli"] +docs = ["Sphinx (>=3)", "sphinx-autoapi", "sphinx-design", "sphinx-issues", "sphinx-rtd-theme", "tomli"] mypy = ["lxml-stubs", "types-Pillow", "types-requests", "types-setuptools"] test = ["attrs (>=20.2.0)", "coverage[toml]", "hypothesis (>=6.36)", "numpy (>=1.21.0)", "psutil (>=5.9)", "pybind11", "pytest (>=6.2.5)", "pytest-cov (>=3.0.0)", "pytest-timeout (>=2.1.0)", "pytest-xdist (>=2.5.0)", "python-dateutil (>=2.8.1)", "python-xmp-toolkit (>=2.0.1)", "tomli"] @@ -725,4 +725,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11.0" -content-hash = "85c35564d7e9ace807f866fbe39318c4b3944f4b55ac3e303339ee1b64351f64" +content-hash = "4f6ffb95706fc5dd24e3e2263c13b23439583650f5d4cb7c4b9a61e4cc91be60" diff --git a/pyproject.toml b/pyproject.toml index 955a96c..61ed050 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,13 +5,13 @@ license = "MIT" name = "x" # can't leave empty because of bug with `poetry install` from poetry.lock file readme = "readme.md" repository = "https://github.com/9-FS/2022-10-23-nHentai-to-PDF-Server" -version = "2.3.12" +version = "3.0.0" [tool.poetry.dependencies] kfsconfig = "^1.0.0" kfsfstr = "^1.0.0" kfslog = "^1.0.0" -kfsmedia = "^2.3.0" +kfsmedia = "^2.3.4" kfssleep = "^1.0.0" python = "^3.11.0" diff --git a/readme.md b/readme.md index 7e862a9..15b88cb 100644 --- a/readme.md +++ b/readme.md @@ -16,7 +16,7 @@ Author: "구FS" ## 1. General -This is the nHentai downloader I wrote to archive as much of the [english nHentai library](https://nhentai.net/language/english/popular) as I can. This server version builds upon the original [nhentai to PDF downloader](https://github.com/9-FS/2021-11-15-nHentai-to-PDF) and automates searching by `nhentai_tag`, creating a `downloadme.txt` from that, downloading the hentai, and finally repeating the process all over again. It basically syncs part of the nhentai library to the local directory at `dest_path`. +This is the nHentai downloader I wrote to archive as much of the [english nHentai library](https://nhentai.net/language/english/popular) as I can. This server version builds upon the original [nhentai to PDF downloader](https://github.com/9-FS/2021-11-15-nHentai-to-PDF) and automates searching by `nhentai_tag`, creating a `downloadme.txt` from that, downloading the hentai, and finally repeating the process all over again. It basically syncs part of the nhentai library to the local directory at `library_path`. Big thanks go out to [h3nTa1m4st3r_xx69](https://github.com/sam-k0), who helped me using nhentai's completely undocumented API. Without him this project could not have been reactivated. I'm happy about anyone who finds my software useful and feedback is also always welcome. Happy downloading~ @@ -32,7 +32,7 @@ I'm happy about anyone who finds my software useful and feedback is also always 1. Copy the cookie values into the `cookies.json`. 1. Execute the program again. This will create a default `./config/headers.json`. 1. Go to https://www.whatismybrowser.com/detect/what-is-my-user-agent/ and copy your user agent into `headers.json`. -1. In `./config/settings.json` set `dest_path` to the directory you want to download to. By default, it will download to `./hentai/`. +1. In `./config/settings.json` set `library_path` to the directory you want to download to. By default, it will download to `./hentai/`. 1. In `./config/settings.json` set `nhentai_tag` to the tag you want to download. By default, it will download "language:english"; all english hentai. You can set it to any tag you want with "tag:{tag}". You can find a list of all tags [here](https://nhentai.net/tags/). ### 2.2. Google Chrome @@ -44,7 +44,7 @@ I'm happy about anyone who finds my software useful and feedback is also always 1. Copy the cookie values into the `cookies.json`. 1. Execute the program again. This will create a default `./config/headers.json`. 1. Go to https://www.whatismybrowser.com/detect/what-is-my-user-agent/ and copy your user agent into `headers.json`. -1. In `./config/settings.json` set `dest_path` to the directory you want to download to. By default, it will download to `./hentai/`. +1. In `./config/settings.json` set `library_path` to the directory you want to download to. By default, it will download to `./hentai/`. 1. In `./config/settings.json` set `nhentai_tag` to the tag you want to download. By default, it will download "language:english"; all english hentai. You can set it to any tag you want with "tag:{tag}". You can find a list of all tags [here](https://nhentai.net/tags/). > :information_source: diff --git a/requirements.txt b/requirements.txt index c9bbeca..537d117 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,19 @@ -certifi==2023.7.22 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" +certifi==2023.11.17 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" charset-normalizer==3.3.2 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" colorama==0.4.6 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" deprecated==1.2.14 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" -idna==3.4 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" -img2pdf==0.5.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" -kfsconfig==1.0.2 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" +idna==3.6 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" +img2pdf==0.5.1 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" +kfsconfig==1.1.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" kfsfstr==1.1.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" kfslog==1.0.1 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" kfsmath==1.0.1 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" -kfsmedia==2.3.3 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" +kfsmedia==2.3.4 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" kfssleep==1.0.1 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" lxml==4.9.3 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" packaging==23.2 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" -pebble==5.0.3 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" -pikepdf==8.7.1 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" +pebble==5.0.5 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" +pikepdf==8.9.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" pillow==10.1.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" requests==2.31.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" urllib3==2.1.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0" diff --git a/src/Hentai.py b/src/Hentai.py index a26e60d..3a50372 100644 --- a/src/Hentai.py +++ b/src/Hentai.py @@ -16,17 +16,18 @@ @dataclasses.dataclass class Hentai: """ - represents an individual hentai + represents an individual hentai from nhentai.net """ - galleries: typing.ClassVar[list[dict]]=[] # list of already downloaded galleries - GALLERIES_FILEPATH: str="./config/galleries.json" # path to file containing already downloaded galleries - galleries_modified: bool=False # has galleries been modified since last save? + galleries: typing.ClassVar[dict[int, list[dict]]]={} # list of already downloaded galleries + galleries_modified: typing.ClassVar[dict[int, bool]]={} # has galleries been modified since last save? + GALLERIES_PATH: typing.ClassVar[str]="./config/" # path to save galleries to + GALLERIES_SPLIT: typing.ClassVar[int]=100000 # split galleries into separate files every 100000 hentai def __init__(self, nhentai_ID: int, cookies: dict[str, str], headers: dict[str, str]): """ - Constructs hentai object. Downloads data from the nhentai API. + Constructs a hentai object. Downloads data from the nhentai API. Arguments: - nhentai_ID: the hentai from nhentai.net found here: https://nhentai.net/g/{hentai_ID} @@ -65,7 +66,7 @@ def __str__(self) -> str: @classmethod def _get_gallery(cls, nhentai_ID: int, cookies: dict[str, str], headers: dict[str, str]) -> dict: """ - Tries to load nhentai API gallery from class variable first, if unsuccessful from file, if unsuccesful again downloads from nhentai API. + Tries to load nhentai API gallery from class variable first, if unsuccessful from files, if unsuccesful again downloads from nhentai API. Arguments: - nhentai_ID: the hentai from nhentai.net found here: https://nhentai.net/g/{hentai_ID} @@ -76,32 +77,33 @@ def _get_gallery(cls, nhentai_ID: int, cookies: dict[str, str], headers: dict[st - gallery: gallery from nhentai API Raises: - - requests.HTTPError: Downloading gallery from \"{NHENTAI_GALLERY_API_URL}/{self.ID}\" failed multiple times. - - ValueError: Hentai with ID \"{self.ID}\" does not exist. + - requests.HTTPError: Downloading gallery from \"{NHENTAI_GALLERY_API_URL}/{nhentai_ID}\" failed multiple times. + - ValueError: Hentai with ID \"{nhentai_ID}\" does not exist. """ - gallery: dict # gallery to return + gallery: dict # gallery to return + gallery_list_filepath: str=os.path.join(cls.GALLERIES_PATH, f"galleries{nhentai_ID//cls.GALLERIES_SPLIT}.json") # appropiate gallery filepath gallery_page: requests.Response - NHENTAI_GALLERY_API_URL: str="https://nhentai.net/api/gallery" # URL to nhentai API - - + NHENTAI_GALLERY_API_URL: str="https://nhentai.net/api/gallery" # URL to nhentai API + + logging.info(f"Loading gallery {nhentai_ID}...") - if 0 None: return - def download(self, dest_path: str) -> bytes: + def download(self, library_path: str, library_split: int) -> bytes: """ - Downloads the hentai, saves it at f"./{DEST_PATH}{self.ID} {self.title}.pdf", and also returns it in case needed. + Downloads the hentai and saves it as PDF at f"./{library_path}/", and also returns it in case needed. If library_split is set, library will be split into subdirectories of maximum library_split many hentai, set 0 to disable. + + Arguments: + - library_path: path to download hentai to + - library_split: split library into subdirectories of maximum this many hentai, 0 to disable Returns: - PDF: finished PDF @@ -203,13 +211,21 @@ def download(self, dest_path: str) -> bytes: raise KFSmedia.DownloadError(f"Error in {self.download.__name__}{inspect.signature(self.download)}: Can't generate page URL for {self} page {i+1}, because media type \"{page['t']}\" is unknown.") pages_URL.append(f"https://i{random.choice(['', '2', '3', '5', '7'])}.nhentai.net/galleries/{self._gallery['media_id']}/{i+1}{MEDIA_TYPES[page['t']]}") # URL, use random image server instance to distribute load - images_filepath.append(f"{dest_path}{self.ID}/{self.ID}-{i+1}{MEDIA_TYPES[page['t']]}") # media filepath, but usually image filepath + images_filepath.append(os.path.join(library_path, str(self.ID), f"{self.ID}-{i+1}{MEDIA_TYPES[page['t']]}")) # media filepath, but usually image filepath PDF_filepath=self.title - for c in TITLE_CHARACTERS_FORBIDDEN: # remove forbidden characters for filenames + for c in TITLE_CHARACTERS_FORBIDDEN: # remove forbidden characters for filenames PDF_filepath=PDF_filepath.replace(c, "") - PDF_filepath=PDF_filepath[:140] # limit title length to 140 characters - PDF_filepath=f"{dest_path}{self.ID} {PDF_filepath}.pdf" + PDF_filepath=PDF_filepath[:140] # limit title length to 140 characters + match library_split: + case 0: + PDF_filepath=os.path.join(library_path, f"{self.ID} {PDF_filepath}.pdf") # PDF filepath, splitting library into subdirectories disabled + case library_split if 0 bytes: raise KFSmedia.DownloadError(f"Error in {self.download.__name__}{inspect.signature(self.download)}: Tried to download and convert hentai \"{self}\" several times, but failed. Giving up.") - if os.path.isdir(f"{dest_path}{self.ID}") and len(os.listdir(f"{dest_path}{self.ID}"))==0: # if cache folder still exists and is empty: + if os.path.isdir(os.path.join(library_path, str(self.ID))) and len(os.listdir(os.path.join(library_path, str(self.ID))))==0: # if cache folder still exists and is empty: try: - os.rmdir(f"{dest_path}{self.ID}") # try to clean up - except PermissionError: # may fail if another process is still using directory like dropbox - pass # don't warn because will be retried in main + os.rmdir(os.path.join(library_path, str(self.ID))) # try to clean up + except PermissionError: # may fail if another process is still using directory like dropbox + pass # don't warn because will be retried in main return PDF @@ -254,15 +270,17 @@ def save_galleries(cls) -> None: Saves galleries to file. """ - if len(cls.galleries)==0 or cls.galleries_modified==False: # don't save if nothing to save, might prevent overwriting galleries file with uninitialised galleries - return - + for gallery_list_id, gallery_list in cls.galleries.items(): + if cls.galleries_modified[gallery_list_id]==False: # if gallery list not modified since last save: skip + continue - logging.info(f"Saving galleries in \"{cls.GALLERIES_FILEPATH}\"...") - with open(cls.GALLERIES_FILEPATH, "wt") as galleries_file: - galleries_file.write(json.dumps(cls.galleries, indent=4)) - logging.info(f"\rSaved galleries in \"{cls.GALLERIES_FILEPATH}\".") - - cls.galleries_modified=False # reset modified flag + gallery_list_filepath: str=os.path.join(cls.GALLERIES_PATH, f"galleries{gallery_list_id}.json") # appropiate gallery filepath + + logging.info(f"Saving galleries in \"{gallery_list_filepath}\"...") + with open(gallery_list_filepath, "wt") as galleries_file: + galleries_file.write(json.dumps(gallery_list, indent=4)) + logging.info(f"\rSaved galleries in \"{gallery_list_filepath}\".") + + cls.galleries_modified[gallery_list_id]=False # reset modified flag return \ No newline at end of file diff --git a/src/main.py b/src/main.py index ae6e9a5..d83864d 100644 --- a/src/main.py +++ b/src/main.py @@ -28,8 +28,9 @@ def main(DEBUG: bool): hentai_ID_list: list[int] # hentai ID to download settings: dict[str, str] # settings SETTINGS_DEFAULT: str=json.dumps({ # settings default - "dest_path": "./hentai/", # path to download hentai to "nhentai_tag": "language:english", # download hentai with this tag, normal tags are in format "tag:{tag}" for example "tag:ffm-threesome" + "library_path": "./hentai/", # path to download hentai to + "library_split": "100000", # split library into subdirectories of maximum this many hentai, 0 to disable }, indent=4) @@ -60,31 +61,31 @@ def main(DEBUG: bool): logging.info(hentai) try: - hentai.download(settings["dest_path"]) # download hentai - except FileExistsError: # if hentai already exists: - continue # skip to next hentai + hentai.download(settings["library_path"], int(settings["library_split"])) # download hentai + except FileExistsError: # if hentai already exists: + continue # skip to next hentai except KFSmedia.DownloadError: - with open("./log/FAILURES.txt", "at") as fails_file: # append in failure file + with open("./log/FAILURES.txt", "at") as fails_file: # append in failure file fails_file.write(f"{hentai.ID}\n") - continue # skip to next hentai + continue # skip to next hentai logging.info("--------------------------------------------------") Hentai.save_galleries() # save all galleries to file logging.info("Deleting leftover image directories...") - for hentai_ID in hentai_ID_list: # attempt final cleanup - if os.path.isdir(f"{settings['dest_path']}{hentai_ID}") and len(os.listdir(f"{settings['dest_path']}{hentai_ID}"))==0: # if cache folder still exists and is empty: + for hentai_ID in hentai_ID_list: # attempt final cleanup + if os.path.isdir(os.path.join(settings["library_path"], str(hentai_ID))) and len(os.listdir(os.path.join(settings["library_path"], str(hentai_ID))))==0: # if cache folder still exists and is empty: try: - os.rmdir(f"{settings['dest_path']}{hentai_ID}") # try to clean up - except PermissionError as e: # may fail if another process is still using directory like dropbox - logging.warning(f"Deleting \"{settings['dest_path']}{hentai_ID}/\" failed with {KFSfstr.full_class_name(e)}.") - cleanup_success=False # cleanup unsuccessful + os.rmdir(os.path.join(settings["library_path"], str(hentai_ID))) # try to clean up + except PermissionError as e: # may fail if another process is still using directory like dropbox + logging.warning(f"Deleting \"{os.path.join(settings['library_path'], str(hentai_ID))}/\" failed with {KFSfstr.full_class_name(e)}.") + cleanup_success=False # cleanup unsuccessful if cleanup_success==True: logging.info("\rDeleted leftover image directories.") try: - os.remove(DOWNLOADME_FILEPATH) # delete downloaded.txt + os.remove(DOWNLOADME_FILEPATH) # delete downloadme.txt except PermissionError as e: logging.error(f"Deleting \"{DOWNLOADME_FILEPATH}\" failed with {KFSfstr.full_class_name(e)}.")