Allow manual cookie/usersession for Javlibrary (#127)

src/Javinizer/Private/Scraper.Javlibrary.ps1

@@ -172,7 +172,10 @@ function Get-JavlibraryActress {
         [Object]$Webrequest,
 
         [Parameter()]
-        [String]$JavlibraryBaseUrl = 'https://www.javlibrary.com'
+        [String]$JavlibraryBaseUrl = 'http://www.javlibrary.com',
+
+        [Parameter()]
+        [PSObject]$Session
     )
 
     process {
@@ -192,7 +195,7 @@ function Get-JavlibraryActress {
             $actressName = $actress.Groups[2].Value
             if ($actress -match '[\u3040-\u309f]|[\u30a0-\u30ff]|[\uff66-\uff9f]|[\u4e00-\u9faf]') {
                 try {
-                    $engActressName = ((Invoke-WebRequest -Uri $engActressUrl).Content | Select-String -Pattern '<div class="boxtitle">Videos starring (.*)<\/div>').Matches.Groups[1].Value
+                    $engActressName = ((Invoke-WebRequest -Uri $engActressUrl -WebSession:$Session -UserAgent:$Session.UserAgent -Verbose:$false).Content | Select-String -Pattern '<div class="boxtitle">Videos starring (.*)<\/div>').Matches.Groups[1].Value
                 } catch {
                     $engActressName = $null
                 }
@@ -214,7 +217,7 @@ function Get-JavlibraryActress {
                 }
             } else {
                 try {
-                    $jaActressName = ((Invoke-WebRequest -Uri $jaActressUrl).Content | Select-String -Pattern '<div class="boxtitle">(.*)のビデオ<\/div>').Matches.Groups[1].Value
+                    $jaActressName = ((Invoke-WebRequest -Uri $jaActressUrl -WebSession:$Session -UserAgent:$Session.UserAgent -Verbose:$false).Content | Select-String -Pattern '<div class="boxtitle">(.*)のビデオ<\/div>').Matches.Groups[1].Value
                 } catch {
                     $jaActressName = $null
                 }

src/Javinizer/Private/Test-JVSettings.ps1

@@ -118,6 +118,9 @@ function Test-JVSettings {
             'emby.apikey',
             'emby.url',
             'javlibrary.baseurl',
+            'javlibrary.browser.useragent',
+            'javlibrary.cookie.cfduid',
+            'javlibrary.cookie.cfclearance',
             'javlibrary.cookie.session',
             'javlibrary.cookie.userid',
             'match.regex.string',

src/Javinizer/Public/Get-CfSession.ps1

@@ -0,0 +1,23 @@
+function Get-CfSession {
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory = $true, Position = 0)]
+        [String]$Cfduid,
+
+        [Parameter(Mandatory = $true, Position = 1)]
+        [String]$Cfclearance,
+
+        [Parameter(Mandatory = $true, Position = 2)]
+        [String]$UserAgent
+    )
+
+    process {
+        $session = New-Object Microsoft.PowerShell.Commands.WebRequestSession
+        $cookie = New-Object System.Net.Cookie('__cfduid', "$Cfduid", '/', 'javlibrary.com')
+        $session.Cookies.Add($cookie)
+        $cookie = New-Object System.Net.Cookie('cf_clearance', "$Cfclearance", '/', 'javlibrary.com')
+        $session.Cookies.Add($cookie)
+        $session.UserAgent = $UserAgent
+        Write-Output $session
+    }
+}

src/Javinizer/Public/Get-JVData.ps1

@@ -75,7 +75,11 @@ function Get-JVData {
         [PSObject]$Url,
 
         [Parameter(ParameterSetName = 'Id')]
-        [Switch]$Strict
+        [Switch]$Strict,
+
+        [Parameter(ParameterSetName = 'Url')]
+        [Parameter(ParameterSetName = 'Id')]
+        [PSObject]$Session
     )
 
     process {
@@ -163,18 +167,18 @@ function Get-JVData {
 
             if ($Javlibrary -or $JavlibraryJa -or $JavlibraryZh) {
                 if (!($JavlibraryUrl -or $JavlibraryJaUrl -or $JavlibraryZhUrl)) {
-                    $jvJavlibraryUrl = Get-JavlibraryUrl -Id $Id -BaseUrl $JavlibraryBaseUrl
+                    $jvJavlibraryUrl = Get-JavlibraryUrl -Id $Id -BaseUrl $JavlibraryBaseUrl -Session:$Session
                 }
                 if ($Javlibrary) {
                     Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - Javlibrary] [Url - $JavlibraryUrl]"
                     Start-ThreadJob -Name "jvdata-Javlibrary" -ThrottleLimit $throttleLimit -ScriptBlock {
                         Import-Module $using:jvModulePath
                         if ($using:JavlibraryUrl) {
-                            $using:JavlibraryUrl | Get-JavlibraryData -JavlibraryBaseUrl $using:JavlibraryBaseUrl
+                            $using:JavlibraryUrl | Get-JavlibraryData -JavlibraryBaseUrl $using:JavlibraryBaseUrl -Session:$using:Session
                         } elseif ($using:jvJavlibraryUrl) {
                             $jvJavlibraryUrl = $using:jvJavlibraryUrl
                             if ($jvJavlibraryUrl) {
-                                $jvJavlibraryUrl.En | Get-JavlibraryData -JavlibraryBaseUrl $using:JavlibraryBaseUrl
+                                $jvJavlibraryUrl.En | Get-JavlibraryData -JavlibraryBaseUrl $using:JavlibraryBaseUrl -Session:$using:Session
                             }
                         }
                     } | Out-Null

src/Javinizer/Public/Get-JavlibraryData.ps1

@@ -7,7 +7,10 @@ function Get-JavlibraryData {
         [String]$Url,
 
         [Parameter()]
-        [String]$JavlibraryBaseUrl
+        [String]$JavlibraryBaseUrl,
+
+        [Parameter()]
+        [PSObject]$Session
     )
 
     process {
@@ -33,7 +36,7 @@ function Get-JavlibraryData {
             Maker         = Get-JavlibraryMaker -WebRequest $webRequest
             Label         = Get-JavlibraryLabel -WebRequest $webRequest
             Rating        = Get-JavlibraryRating -WebRequest $webRequest
-            Actress       = Get-JavlibraryActress -WebRequest $webRequest -JavlibraryBaseUrl $JavlibraryBaseUrl
+            Actress       = Get-JavlibraryActress -WebRequest $webRequest -JavlibraryBaseUrl $JavlibraryBaseUrl -Session:$Session
             Genre         = Get-JavlibraryGenre -WebRequest $webRequest
             CoverUrl      = Get-JavlibraryCoverUrl -WebRequest $webRequest
             ScreenshotUrl = Get-JavlibraryScreenshotUrl -WebRequest $webRequest

src/Javinizer/Public/Get-JavlibraryUrl.ps1

@@ -7,7 +7,10 @@ function Get-JavlibraryUrl {
         [String]$Id,
 
         [Parameter(Position = 1)]
-        [String]$BaseUrl = 'https://www.javlibrary.com'
+        [String]$BaseUrl = 'http://www.javlibrary.com',
+
+        [Parameter(Position = 2)]
+        [PSObject]$Session
     )
 
     process {
@@ -20,12 +23,12 @@ function Get-JavlibraryUrl {
 
         try {
             Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] Performing [GET] on URL [$searchUrl]"
-            $webRequest = Invoke-WebRequest -Uri $searchUrl -Method Get -Verbose:$false
+            $webRequest = Invoke-WebRequest -Uri $searchUrl -WebSession:$Session -UserAgent:$Session.UserAgent -Method Get -Verbose:$false
         } catch {
             try {
                 # Add a retry to the URL search due to 500 errors occurring randomly when scraping Javlibrary
                 Start-Sleep -Seconds 3
-                $webRequest = Invoke-WebRequest -Uri $searchUrl -Method Get -Verbose:$false
+                $webRequest = Invoke-WebRequest -Uri $searchUrl -WebSession:$Session -UserAgent:$Session.UserAgent -Method Get -Verbose:$false
             } catch {
                 Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Error -Message "[$Id] [$($MyInvocation.MyCommand.Name)] Error occured on [GET] on URL [$searchUrl]: $PSItem" -Action 'Continue'
             }
@@ -37,7 +40,7 @@ function Get-JavlibraryUrl {
         if ($searchResultUrl -match "$BaseUrl?v=") {
             try {
                 Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] Performing [GET] on URL [$searchResultUrl]"
-                $webRequest = Invoke-WebRequest -Uri $searchResultUrl -Method Get -Verbose:$false
+                $webRequest = Invoke-WebRequest -Uri $searchResultUrl -WebSession:$Session -UserAgent:$Session.UserAgent -Method Get -Verbose:$false
             } catch {
                 Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Error -Message "[$Id] [$($MyInvocation.MyCommand.Name)] Error occured on [GET] on URL [$searchResultUrl]: $PSItem" -Action 'Continue'
             }
@@ -66,7 +69,7 @@ function Get-JavlibraryUrl {
 
                     try {
                         Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] Performing [GET] on URL [$directUrl]"
-                        $webRequest = Invoke-WebRequest -Uri $directUrl -Method Get -Verbose:$false
+                        $webRequest = Invoke-WebRequest -Uri $directUrl -WebSession:$Session -UserAgent:$Session.UserAgent -Method Get -Verbose:$false
                     } catch {
                         Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Error -Message "[$Id] [$($MyInvocation.MyCommand.Name)] Error occured on [GET] on URL [$directUrl]: $PSItem" -Action 'Continue'
                     }

src/Javinizer/Public/Javinizer.ps1

@@ -350,6 +350,11 @@ function Javinizer {
         [Parameter(ParameterSetName = 'Info')]
         [Switch]$Jav321Ja,
 
+        [Parameter(ParameterSetName = 'Info')]
+        [Parameter(ParameterSetName = 'Path')]
+        [Parameter(ParameterSetName = 'Javlibrary')]
+        [PSObject]$CfSession,
+
         [Parameter(ParameterSetName = 'Emby')]
         [Switch]$SetEmbyThumbs,
 
@@ -499,6 +504,37 @@ function Javinizer {
         # Validate the values in the settings file following all command-line transformations
         $Settings = $Settings | Test-JVSettings
 
+        if (($Settings.'scraper.movie.javlibrary' -or $Settings.'scraper.movie.javlibraryja' -or $Settings.'scraper.movie.javlibraryzh' -and $Javlibrary) -or $SetOwned) {
+            if ($Settings.'javlibrary.baseurl' -match 'javlibrary.com') {
+                if (!($CfSession)) {
+                    try {
+                        $CfSession = Get-CfSession -Cfduid:$Settings.'javlibrary.cookie.cfduid' -Cfclearance:$Settings.'javlibrary.cookie.cfclearance' -UserAgent:$Settings.'javlibrary.browser.useragent'
+                        $test = Invoke-WebRequest 'https://www.javlibrary.com' -WebSession $CfSession -UserAgent $CfSession.UserAgent -Verbose:$false
+                    } catch {
+                        Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Warning -Message "[$($MyInvocation.MyCommand.Name)] Unable reach Javlibrary, enter websession/cookies to use scraper"
+                        $CfSession = Get-CfSession
+                        try {
+                            Start-Sleep -Seconds 2
+                            $test = Invoke-WebRequest 'https://www.javlibrary.com' -WebSession $CfSession -UserAgent $CfSession.UserAgent -Verbose:$false
+                        } catch {
+                            Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Error -Message "[$($MyInvocation.MyCommand.Name)] Unable reach Javlibrary, invalid websession values"
+                        }
+
+                        $settingsContent = Get-Content -Path $SettingsPath
+                        $cookies = $CfSession.Cookies.GetCookies('https://javlibrary.com')
+                        $cfduid = ($cookies | Where-Object { $_.Name -eq '__cfduid' }).Value
+                        $cfclearance = ($cookies | Where-Object { $_.Name -eq 'cf_clearance' }).Value
+                        $userAgent = $CfSession.UserAgent
+                        $settingsContent = $settingsContent -replace '"javlibrary\.cookie\.cfduid": ".*"', "`"javlibrary.cookie.cfduid`": `"$cfduid`""
+                        $settingsContent = $settingsContent -replace '"javlibrary\.cookie\.cfclearance": ".*"', "`"javlibrary.cookie.cfclearance`": `"$cfclearance`""
+                        $settingsContent = $settingsContent -replace '"javlibrary\.browser\.useragent": ".*"', "`"javlibrary.browser.useragent`": `"$userAgent`""
+                        $settingsContent | Out-File -FilePath $SettingsPath
+                        Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Warning -Message "[$($MyInvocation.MyCommand.Name)] Replaced Javlibrary websession with updated values in [$SettingsPath]"
+                    }
+                }
+            }
+        }
+
         switch ($PsCmdlet.ParameterSetName) {
             'Info' {
                 if ($Find -match 'https?:\/\/') {
@@ -535,7 +571,7 @@ function Javinizer {
                 } else {
                     $data = Get-JVData -Id $Find -R18:$R18 -R18Zh:$R18Zh -Javlibrary:$Javlibrary -JavlibraryJa:$JavlibraryJa -JavlibraryZh:$JavlibraryZh -Dmm:$Dmm `
                         -DmmJa:$DmmJa -Javbus:$Javbus -JavbusJa:$JavbusJa -JavbusZh:$JavbusZh -Jav321Ja:$Jav321Ja -JavlibraryBaseUrl $Settings.'javlibrary.baseurl' `
-                        -UncensorCsvPath $uncensorCsvPath -Strict:$Strict
+                        -UncensorCsvPath $uncensorCsvPath -Strict:$Strict -Session:$CfSession
                 }
 
                 if ($Aggregated) {
@@ -641,7 +677,7 @@ function Javinizer {
 
             'Javlibrary' {
                 try {
-                    $request = Invoke-WebRequest -Uri "https://www.javlibrary.com/en/mv_owned_print.php" -Verbose:$false -Headers @{
+                    $request = Invoke-WebRequest -Uri "https://www.javlibrary.com/en/mv_owned_print.php" -WebSession $CfSession -UserAgent $CfSession.UserAgent -Verbose:$false -Headers @{
                         "method"                    = "GET"
                         "authority"                 = "www.javlibrary.com"
                         "scheme"                    = "https"
@@ -689,7 +725,7 @@ function Javinizer {
                     $index = 1
                     foreach ($movieId in $unowned) {
                         Write-Progress -Id 1 -Activity "Javinizer" -Status "Remaining Jobs: $($unowned.Count-$index)" -PercentComplete ($index/$unowned.Count*100) -CurrentOperation "Setting owned: $movieId"
-                        Set-JavlibraryOwned -Id $movieId -UserId $Settings.'javlibrary.cookie.userid' -Session $Settings.'javlibrary.cookie.session'
+                        Set-JavlibraryOwned -Id $movieId -UserId $Settings.'javlibrary.cookie.userid' -LoginSession $Settings.'javlibrary.cookie.session' -Session:$CfSession
                         $index++
                     }
                 } else {
@@ -754,7 +790,7 @@ function Javinizer {
                         $mediaInfo = Get-JVMediaInfo -Path $movie.FullName
                     }
 
-                    $javData = Get-JVData -Url $Url -Settings $Settings -UncensorCsvPath $uncensorCsvPath
+                    $javData = Get-JVData -Url $Url -Settings $Settings -UncensorCsvPath $uncensorCsvPath -Session:$CfSession
                     if ($null -ne $javData) {
                         $javAggregatedData = $javData | Get-JVAggregatedData -Settings $Settings -MediaInfo $mediaInfo | Test-JVData -RequiredFields $Settings.'sort.metadata.requiredfield'
                         if ($null -ne $javAggregatedData) {
@@ -772,7 +808,7 @@ function Javinizer {
                             Import-Module $using:jvModulePath
                             $jvMovie = $_
                             $Settings = $using:Settings
-                            Javinizer -IsThread -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
+                            Javinizer -IsThread -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -MoveToFolder:$Settings.'sort.movetofolder' -RenameFile:$Settings.'sort.renamefile' -CfSession:$using:CfSession -Update:$using:Update -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
                         }
                     }
 
@@ -782,7 +818,7 @@ function Javinizer {
                                 $mediaInfo = Get-JVMediaInfo -Path $movie.FullName
                             }
 
-                            $javData = Get-JVData -Id $movie.Id -Settings $Settings -UncensorCsvPath $uncensorCsvPath -Strict:$Strict
+                            $javData = Get-JVData -Id $movie.Id -Settings $Settings -UncensorCsvPath $uncensorCsvPath -Strict:$Strict -Session:$CfSession
                             if ($null -ne $javData) {
                                 $javAggregatedData = $javData | Get-JVAggregatedData -Settings $Settings -MediaInfo $mediaInfo | Test-JVData -RequiredFields $Settings.'sort.metadata.requiredfield'
                                 if ($javAggregatedData.NullFields -eq '') {

src/Javinizer/Public/Set-JavlibraryOwned.ps1

@@ -8,16 +8,20 @@ function Set-JavlibraryOwned {
         [String]$UserId,
 
         [Parameter(Mandatory = $true)]
-        [String]$Session
+        [String]$LoginSession,
+
+        [Parameter(Mandatory = $true)]
+        [PSObject]$Session
     )
 
     process {
         $ProgressPreference = 'SilentlyContinue'
         Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Info "[$Id] [$($MyInvocation.MyCommand.Name)] Setting owned on JavLibrary"
+
         try {
-            $url = (Get-JavlibraryUrl -Id $Id).En
+            $url = (Get-JavlibraryUrl -Id $Id -BaseUrl 'http://www.javlibrary.com' -Session:$Session).En
             if ($null -ne $url) {
-                $request = Invoke-WebRequest -Uri $url -Method Get -Verbose:$false
+                $request = Invoke-WebRequest -Uri $url -WebSession $Session -UserAgent $Session.UserAgent -Method Get -Verbose:$false
                 $ajaxId = Get-JavlibraryAjaxId -Webrequest $request
 
             } else {
@@ -47,10 +51,12 @@ function Set-JavlibraryOwned {
                     "referer"          = $url
                     "accept-encoding"  = "gzip, deflate, br"
                     "accept-language"  = "en-US, en; q=0.9"
-                    "cookie"           = "timezone=420; over18=18; userid=$UserId; session=$Session"
+                    "cookie"           = "timezone=420; over18=18; userid=$UserId; session=$LoginSession"
                 } `
                     -ContentType "application/x-www-form-urlencoded; charset=UTF-8" `
                     -Body "type=2&targetid=$ajaxId" `
+                    -WebSession $Session `
+                    -UserAgent $Session.UserAgent `
                     -Verbose:$false
             }
             if ($stopwatch.elapsed -gt $timeout) {

src/Javinizer/jvSettings.json

@@ -95,6 +95,9 @@
     "emby.url": "http:\/\/192.168.0.1:8096",
     "emby.apikey": "",
     "javlibrary.baseurl": "http:\/\/www.javlibrary.com",
+    "javlibrary.browser.useragent": "",
+    "javlibrary.cookie.cfduid": "",
+    "javlibrary.cookie.cfclearance": "",
     "javlibrary.cookie.session": "",
     "javlibrary.cookie.userid": "",
     "admin.log": 1,