merge(feature/windows-installer-with-playwright): add windows install…

…er and fix tests after Cloudflare challenge solver

.dockerignore

@@ -1,4 +1,5 @@
 dist
 node_modules
 venv
-out
+out
+MonitorrentInstaller

.gitignore

@@ -129,4 +129,5 @@ nssm.exe
 assets/style.css
 report.html
 /venv/
-/out/
+/out/
+/MonitorrentInstaller/browsers/

MonitorrentInstaller/MonitorrentInstaller/InstallService.bat

@@ -1,2 +1,3 @@
 @echo off
+SET PLAYWRIGHT_BROWSERS_PATH="%~dp0/browsers"
 "%~dp0/nssm.exe" install monitorrent "%~dp0/RunApp.bat"

MonitorrentInstaller/MonitorrentInstaller/MonitorrentInstaller.wixproj

@@ -17,17 +17,18 @@
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
     <OutputPath>bin\$(Configuration)\</OutputPath>
     <IntermediateOutputPath>obj\$(Configuration)\</IntermediateOutputPath>
-    <DefineConstants>Debug;MonitorrentSource=..\..\monitorrent;WebAppSource=..\..\webapp;PythonEnv=..\env</DefineConstants>
+    <DefineConstants>Debug;MonitorrentSource=..\..\dist\monitorrent;WebAppSource=..\..\dist\webapp;PythonEnv=..\env;Browsers=..\browsers</DefineConstants>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x86' ">
     <OutputPath>bin\$(Configuration)\</OutputPath>
     <IntermediateOutputPath>obj\$(Configuration)\</IntermediateOutputPath>
-    <DefineConstants>MonitorrentSource=..\..\monitorrent;WebAppSource=..\..\webapp;PythonEnv=..\env</DefineConstants>
+    <DefineConstants>MonitorrentSource=..\..\dist\monitorrent;WebAppSource=..\..\dist\webapp;PythonEnv=..\env;Browsers=..\browsers</DefineConstants>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="Monitorrent.wxs" />
     <Compile Include="Product.wxs" />
     <Compile Include="PythonEnvironment.wxs" />
+    <Compile Include="Browsers.wxs" />
     <Compile Include="WebApp.wxs" />
   </ItemGroup>
   <ItemGroup>
@@ -48,6 +49,7 @@
   </ItemGroup>
   <Import Project="$(WixTargetsPath)" />
   <Target Name="BeforeBuild">
+    <HeatDirectory Directory="..\browsers" OutputFile="Browsers.wxs" ToolPath="$(WixToolPath)" AutogenerateGuids="true" DirectoryRefId="INSTALLFOLDER" ComponentGroupName="BrowsersGroup" PreprocessorVariable="var.Browsers" SuppressAllWarnings="True" VerboseOutput="True" SuppressCom="True" SuppressRegistry="True" />
     <HeatDirectory Directory="..\env" OutputFile="PythonEnvironment.wxs" ToolPath="$(WixToolPath)" AutogenerateGuids="true" DirectoryRefId="INSTALLFOLDER" ComponentGroupName="PythonEnvironmentGroup" PreprocessorVariable="var.PythonEnv" SuppressAllWarnings="True" VerboseOutput="True" SuppressCom="True" SuppressRegistry="True" />
     <HeatDirectory Directory="..\..\dist\monitorrent" OutputFile="Monitorrent.wxs" ToolPath="$(WixToolPath)" AutogenerateGuids="true" DirectoryRefId="INSTALLFOLDER" ComponentGroupName="MonitorrentGroup" PreprocessorVariable="var.MonitorrentSource" />
     <HeatDirectory Directory="..\..\dist\webapp" OutputFile="WebApp.wxs" ToolPath="$(WixToolPath)" AutogenerateGuids="true" DirectoryRefId="INSTALLFOLDER" ComponentGroupName="WebAppGroup" PreprocessorVariable="var.WebAppSource" />

MonitorrentInstaller/MonitorrentInstaller/Product.wxs

@@ -37,6 +37,7 @@
           <Directory Id="MONITORRENT" Name="monitorrent"/>
           <Directory Id="WEBAPP" Name="webapp"/>
           <Directory Id="PYTHONENV" Name="env"/>
+          <Directory Id="BROWSERS" Name="browsers"/>
         </Directory>
 			</Directory>
 		</Directory>
@@ -47,6 +48,7 @@
       <ComponentGroupRef Id="MonitorrentGroup"/>
       <ComponentGroupRef Id="WebAppGroup"/>
       <ComponentGroupRef Id="PythonEnvironmentGroup"/>
+      <ComponentGroupRef Id="BrowsersGroup"/>
       <Component Id="Server" Guid="F2F9CA91-E5AB-427E-BE7C-506C127A0C55">
         <File Id="server.py" Source="..\..\dist\server.py" />
         <File Id="nssm.exe" Source="..\..\nssm.exe"/>

MonitorrentInstaller/MonitorrentInstaller/RestartService.bat

@@ -1,2 +1,3 @@
 @echo off
+SET PLAYWRIGHT_BROWSERS_PATH="%~dp0/browsers"
 "%~dp0/nssm.exe" restart monitorrent

MonitorrentInstaller/MonitorrentInstaller/RunApp.bat

@@ -1,2 +1,3 @@
 @echo off
+SET PLAYWRIGHT_BROWSERS_PATH="%~dp0/browsers"
 "%~dp0/env/python" "%~dp0/server.py"

MonitorrentInstaller/MonitorrentInstaller/StartService.bat

@@ -1,2 +1,3 @@
 @echo off
-"%~dp0/nssm.exe" start monitorrent
+SET PLAYWRIGHT_BROWSERS_PATH="%~dp0/browsers"
+"%~dp0/nssm.exe" start monitorrent

MonitorrentInstaller/README.md

@@ -1,4 +1,11 @@
 Updating installer:
-1) Copy your python 2.7.11 environment to env folder in this directory.
+1) Copy your python 3.7-3.8 environment to env folder in this directory.
 2) Manually update new packages in PythonEnvironment.wsx.
+3) install playwright browsers over powershell in this folder:
+```powershell
+$env:PLAYWRIGHT_BROWSERS_PATH="$(Get-Location)\browsers"
+pip install playwright==1.20.0
+playwright install firefox
+```
+
 All other files required by installation are collected dynamiacally during the build process.

appveyor/prepare_for_installer.ps1

@@ -28,4 +28,8 @@ Copy-Item $nssmFodlerPath\nssm-2.24\win32\nssm.exe .\nssm.exe
 Remove-Item -Recurse -Force "$targetDir\MonitorrentInstaller\env\Doc"
 
 <#  Install web client#>
-gulp dist
+gulp dist
+
+<# Install playwright browser #>
+$env:PLAYWRIGHT_BROWSERS_PATH="$(Get-Location)\MonitorrentInstaller\browsers"
+playwright install firefox

monitorrent/plugins/trackers/__init__.py

@@ -354,7 +354,7 @@ def _execute_login(self, engine):
         return True
 
 
-def extract_cloudflare_credentials_and_headers(url: str, headers: dict, cookies: dict, timeout: int=30000):
+def extract_cloudflare_credentials_and_headers(url: str, headers: dict, cookies: dict, timeout: int = 30000):
     scrapper = cloudscraper.create_scraper()
 
     try:

monitorrent/plugins/trackers/lostfilm.py

@@ -505,10 +505,9 @@ def verify(self):
         return r1.url == my_settings_url and '<meta http-equiv="refresh" content="0; url=/">' not in r1.text
 
     def get_cookies(self):
-        if not self.session:
-            return False
         new_cookies = dict(**self.cookies)
-        new_cookies.update({'lf_session': self.session})
+        if self.session:
+            new_cookies.update({'lf_session': self.session})
         return new_cookies
 
     def can_parse_url(self, url):
@@ -696,8 +695,9 @@ class LostFilmPlugin(WithCredentialsMixin, TrackerPluginBase):
         }]
     }]
 
-    def __init__(self):
-        self.tracker = LostFilmTVTracker(headers_cookies_updater=self._update_headers_and_cookies)
+    def __init__(self, headers=None, cookies=None):
+        self.tracker = LostFilmTVTracker(headers_cookies_updater=self._update_headers_and_cookies,
+                                         headers=headers, cookies=cookies)
 
     def configure(self, config):
         self.tracker.playwright_timeout = config.playwright_timeout
@@ -743,7 +743,9 @@ def login(self):
             if not username or not password:
                 return LoginResult.CredentialsNotSpecified
         try:
-            self.tracker.login(username, password, headers, cookies)
+            self.tracker.login(username, password,
+                               headers or self.tracker.headers,
+                               cookies or self.tracker.cookies)
             with DBSession() as db:
                 cred = db.query(self.credentials_class).first()
                 cred.session = self.tracker.session

package.json

@@ -7,7 +7,7 @@
     "test": "tests"
   },
   "scripts": {
-    "preinstall": "npm install -g gulp"
+    "preinstall": "npm install -g gulp@3.9.0"
   },
   "repository": {
     "type": "git",

requirements.txt

@@ -17,6 +17,7 @@ dateutils==0.6.6
 phpserialize==1.3
 semver==2.8.1
 playwright==1.20.0
+brotli==1.0.9
 
 #python 2/3 compatibility and backports
 six==1.12.0

tests/__init__.py

@@ -137,7 +137,7 @@ def assertTable(self, expected_table):
         """
         m = MetaData(self.engine)
         table = Table(expected_table.name, m, autoload=True)
-        self.assertEqual(len(expected_table.columns), len(table.columns))
+        assert len(expected_table.columns) == len(table.columns)
 
         for column_name in list(expected_table.columns.keys()):
             self.assertTrue(column_name in table.columns, 'Can\'t find column {0} in table {1}'