Merge pull request #132 from jvlflame/dev

2.1.3
javinizer · Oct 12, 2020 · 3c48a08 · 3c48a08
2 parents cfb080b + f7de215
commit 3c48a08
Show file tree

Hide file tree

Showing 14 changed files with 335 additions and 73 deletions.
diff --git a/.github/CHANGELOG.md b/.github/CHANGELOG.md
@@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.1.3]
+
+### Added
+- Function `Get-CfSession` to create a websession object to access the Javlibrary scraper
+- Settings `javlibrary.browser.useragent`, `javlibrary.cookie.cfduid`, and `javlibrary.cookie.cfclearance` to persist Javlibrary websession
+
+### Changed
+- Javinizer out-of-box-experience (OOBE) updated to remove Javlibrary as a default scraper due to Cloudflare IAUM protection
+    - Javbus is now enabled as a default scraper
+
+### Fixed
+- Sort now properly fails when there are missing required metadata fields when using `-Url`
+- Javbus scraper now properly finds uncensored videos when similar search results are found in the censored section
+- Maxtitlelength behavior for Japanese titles now do not cut off at the last whitespace character
+
+
 ## [2.1.2]
 
 ### Added

diff --git a/README.md b/README.md
@@ -426,6 +426,38 @@ ID-###-cd\d       - ID-069-cd1, ID-069-cd2
 | Emby | Set-up a `Movie` library with all metadata/image downloaders disabled. Input your server url and API key in the `jvSettings.json` file and use `Javinizer -SetEmbyThumbs` to set actress thumbnails |
 | Jellyfin | Set-up a `Movie` library with all metadata/image downloaders disabled. Input your server url and API key in the `jvSettings.json` file and use `Javinizer -SetEmbyThumbs` to set actress thumbnails |
 
+
+### Javlibrary Scraper
+
+The Javlibrary scraper periodically has Cloudflare IUAM (I'm Under Attack Mode) activated on its site, meaning that Javinizer is unable to automatically bypass the bot protection. In this case, we need to manually specify the browser cookies and useragent when running Javinizer.
+
+This can be done in multiple ways when the Javlibrary scraper is enabled in the settings.
+Updated websession settings are automatically populated in your settings files after a successful connection to Javlibrary.
+```powershell
+# 1. Input websession values during Javinizer runtime
+Javinizer -Path 'C:\JAV\Unsorted' -DestinationPath 'C:\JAV\Sorted'
+
+cmdlet Get-CfSession at command pipeline position 1
+Supply values for the following parameters:
+Cfduid: dec857987e88528669d3bda527c8119731602366718
+Cfclearance: 812db5df4cfffb1ec11b2b5cff38d6323b842a9d-1602366718-0-1zc9220a8fz43c94e72z7b83181-150
+UserAgent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36
+
+# 2. Create the websession object first and specify it as a parameter
+$session = Get-CfSession -Cduid 'dec857987e88528669d3bda527c8119731602366718' `
+-Cfclearance '812db5df4cfffb1ec11b2b5cff38d6323b842a9d-1602366718-0-1zc9220a8fz43c94e72z7b83181-150' `
+-UserAgent 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36'
+
+Javinizer -Path 'C:\JAV\Unsorted' -DestinationPath 'C:\JAV\Sorted' -CfSession $session
+
+# 3. Manually populate the settings file with websession values
+Javinizer -OpenSettings
+
+"javlibrary.browser.useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36",
+"javlibrary.cookie.cfduid": "dec857987e88528669d3bda527c8119731602366718",
+"javlibrary.cookie.cfclearance": "812db5df4cfffb1ec11b2b5cff38d6323b842a9d-1602366718-0-1zc9220a8fz43c94e72z7b83181-150",
+```
+
 ### Metadata Priorities
 
 Scrapers by default are english. Scrapers appended by Ja are Japanese, Zh are Chinese.
@@ -614,5 +646,10 @@ For example, if your jvGenres.csv file looks like this:
 | `emby.url` | Specifies the base URL of your Emby/Jellyfin instance to add actress images. | http:\\/\\/192.168.0.1:8096
 | `emby.apikey` | Specifies the API key of your Emby/Jellyfin instance. | API Key string
 | `javlibrary.baseurl` | Specifies the base URL of the Javlibrary instance you want to scrape. This is useful if you are running into CloudFlare errors on the main site and want to use a mirror such as g46e.com or m45e.com | http:\\/\\/javlibrary.com
+| `javlibrary.browser.useragent` | Specifies your browser's user agent when accessing Javlibrary. This can be found by googling your user agent | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36
+| `javlibrary.cookie.cfduid` | Specifies the cookie value of the __cfduid cookie when accessing Javlibrary | dec857987e88528669d3bda527c8119731602366718
+| `javlibrary.cookie.cfclearance` | Specifies the cookie value of the cf_clearance cookie when accessing Javlibrary | 812db5df4cfffb1ec11b2b5cff38d6323b842a9d-1602222718-0-1zc9220a8fz16c94e72z7b83181-150
+| `javlibrary.cookie.session` | Specifies the cookie value of the session cookie when logged into Javlibrary, used to set owned movies | 0HDNScw5zuKGOFMoRZJxQI9GGGIGhffZ49W9tz
+| `javlibrary.cookie.userid` | Specifies the cookie value of the userid cookie when logged into Javlibrary, used to set owned movies | jvlflame
 | `admin.log` | Specifies to write debug, warning, error, and verbose messages to the log file. | 0, 1
 | `admin.log.level` | Specifies the level of logs that will be written to the log file. | Debug, Info, Warning, Error
diff --git a/src/Javinizer/Javinizer.psd1 b/src/Javinizer/Javinizer.psd1
@@ -13,7 +13,7 @@
 
     # Version number of this module.
 
-    ModuleVersion     = '2.1.2'
+    ModuleVersion     = '2.1.3'
 
     # Supported PSEditions
     # CompatiblePSEditions = @('Core')

diff --git a/src/Javinizer/Private/Convert-JVString.ps1 b/src/Javinizer/Private/Convert-JVString.ps1
@@ -44,17 +44,41 @@ function Convert-JVString {
         if ($maxTitleLength) {
             if ($Data.Title.Length -ge $MaxTitleLength) {
                 $shortTitle = $Data.Title.Substring(0, $MaxTitleLength)
-                $splitTitle = $shortTitle -split ' '
-                if ($splitTitle.Count -gt 1) {
-                    # Remove the last word of the title just in case it is cut off
-                    $title = ($splitTitle[0..($splitTitle.Length - 2)] -join ' ')
-                    if ($title[-1] -match '\W') {
-                        $Data.Title = ($title.Substring(0, $title.Length - 2)) + '...'
+                if ($shortTitle -match '[\u3040-\u309f]|[\u30a0-\u30ff]|[\uff66-\uff9f]|[\u4e00-\u9faf]') {
+                    $Data.Title = $shortTitle + '...'
+                } else {
+                    $splitTitle = $shortTitle -split ' '
+                    if ($splitTitle.Count -gt 1) {
+                        # Remove the last word of the title just in case it is cut off
+                        $title = ($splitTitle[0..($splitTitle.Length - 2)] -join ' ')
+                        if ($title[-1] -match '\W') {
+                            $Data.Title = ($title.Substring(0, $title.Length - 2)) + '...'
+                        } else {
+                            $Data.Title = $title + '...'
+                        }
                     } else {
-                        $Data.Title = $title + '...'
+                        $Data.Title = $shortTitle + '...'
                     }
+                }
+            }
+
+            if ($Data.AlternateTitle.Length -ge $MaxTitleLength) {
+                $shortTitle = $Data.AlternateTitle.Substring(0, $MaxTitleLength)
+                if ($shortTitle -match '[\u3040-\u309f]|[\u30a0-\u30ff]|[\uff66-\uff9f]|[\u4e00-\u9faf]') {
+                    $Data.AlternateTitle = $shortTitle + '...'
                 } else {
-                    $Data.Title = $shortTitle + '...'
+                    $splitTitle = $shortTitle -split ' '
+                    if ($splitTitle.Count -gt 1) {
+                        # Remove the last word of the title just in case it is cut off
+                        $title = ($splitTitle[0..($splitTitle.Length - 2)] -join ' ')
+                        if ($title[-1] -match '\W') {
+                            $Data.AlternateTitle = ($title.Substring(0, $title.Length - 2)) + '...'
+                        } else {
+                            $Data.AlternateTitle = $title + '...'
+                        }
+                    } else {
+                        $Data.AlternateTitle = $shortTitle + '...'
+                    }
                 }
             }
         }

diff --git a/src/Javinizer/Private/Scraper.Javlibrary.ps1 b/src/Javinizer/Private/Scraper.Javlibrary.ps1
@@ -172,7 +172,10 @@ function Get-JavlibraryActress {
         [Object]$Webrequest,
 
         [Parameter()]
-        [String]$JavlibraryBaseUrl = 'https://www.javlibrary.com'
+        [String]$JavlibraryBaseUrl = 'http://www.javlibrary.com',
+
+        [Parameter()]
+        [PSObject]$Session
     )
 
     process {
@@ -192,7 +195,7 @@ function Get-JavlibraryActress {
             $actressName = $actress.Groups[2].Value
             if ($actress -match '[\u3040-\u309f]|[\u30a0-\u30ff]|[\uff66-\uff9f]|[\u4e00-\u9faf]') {
                 try {
-                    $engActressName = ((Invoke-WebRequest -Uri $engActressUrl).Content | Select-String -Pattern '<div class="boxtitle">Videos starring (.*)<\/div>').Matches.Groups[1].Value
+                    $engActressName = ((Invoke-WebRequest -Uri $engActressUrl -WebSession:$Session -UserAgent:$Session.UserAgent -Verbose:$false).Content | Select-String -Pattern '<div class="boxtitle">Videos starring (.*)<\/div>').Matches.Groups[1].Value
                 } catch {
                     $engActressName = $null
                 }
@@ -214,7 +217,7 @@ function Get-JavlibraryActress {
                 }
             } else {
                 try {
-                    $jaActressName = ((Invoke-WebRequest -Uri $jaActressUrl).Content | Select-String -Pattern '<div class="boxtitle">(.*)のビデオ<\/div>').Matches.Groups[1].Value
+                    $jaActressName = ((Invoke-WebRequest -Uri $jaActressUrl -WebSession:$Session -UserAgent:$Session.UserAgent -Verbose:$false).Content | Select-String -Pattern '<div class="boxtitle">(.*)のビデオ<\/div>').Matches.Groups[1].Value
                 } catch {
                     $jaActressName = $null
                 }

diff --git a/src/Javinizer/Private/Test-JVSettings.ps1 b/src/Javinizer/Private/Test-JVSettings.ps1
@@ -118,6 +118,9 @@ function Test-JVSettings {
             'emby.apikey',
             'emby.url',
             'javlibrary.baseurl',
+            'javlibrary.browser.useragent',
+            'javlibrary.cookie.cfduid',
+            'javlibrary.cookie.cfclearance',
             'javlibrary.cookie.session',
             'javlibrary.cookie.userid',
             'match.regex.string',

diff --git a/src/Javinizer/Public/Get-CfSession.ps1 b/src/Javinizer/Public/Get-CfSession.ps1
@@ -0,0 +1,23 @@
+function Get-CfSession {
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory = $true, Position = 0)]
+        [String]$Cfduid,
+
+        [Parameter(Mandatory = $true, Position = 1)]
+        [String]$Cfclearance,
+
+        [Parameter(Mandatory = $true, Position = 2)]
+        [String]$UserAgent
+    )
+
+    process {
+        $session = New-Object Microsoft.PowerShell.Commands.WebRequestSession
+        $cookie = New-Object System.Net.Cookie('__cfduid', "$Cfduid", '/', 'javlibrary.com')
+        $session.Cookies.Add($cookie)
+        $cookie = New-Object System.Net.Cookie('cf_clearance', "$Cfclearance", '/', 'javlibrary.com')
+        $session.Cookies.Add($cookie)
+        $session.UserAgent = $UserAgent
+        Write-Output $session
+    }
+}
diff --git a/src/Javinizer/Public/Get-JVData.ps1 b/src/Javinizer/Public/Get-JVData.ps1
@@ -75,7 +75,11 @@ function Get-JVData {
         [PSObject]$Url,
 
         [Parameter(ParameterSetName = 'Id')]
-        [Switch]$Strict
+        [Switch]$Strict,
+
+        [Parameter(ParameterSetName = 'Url')]
+        [Parameter(ParameterSetName = 'Id')]
+        [PSObject]$Session
     )
 
     process {
@@ -163,18 +167,18 @@ function Get-JVData {
 
             if ($Javlibrary -or $JavlibraryJa -or $JavlibraryZh) {
                 if (!($JavlibraryUrl -or $JavlibraryJaUrl -or $JavlibraryZhUrl)) {
-                    $jvJavlibraryUrl = Get-JavlibraryUrl -Id $Id -BaseUrl $JavlibraryBaseUrl
+                    $jvJavlibraryUrl = Get-JavlibraryUrl -Id $Id -BaseUrl $JavlibraryBaseUrl -Session:$Session
                 }
                 if ($Javlibrary) {
                     Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - Javlibrary] [Url - $JavlibraryUrl]"
                     Start-ThreadJob -Name "jvdata-Javlibrary" -ThrottleLimit $throttleLimit -ScriptBlock {
                         Import-Module $using:jvModulePath
                         if ($using:JavlibraryUrl) {
-                            $using:JavlibraryUrl | Get-JavlibraryData -JavlibraryBaseUrl $using:JavlibraryBaseUrl
+                            $using:JavlibraryUrl | Get-JavlibraryData -JavlibraryBaseUrl $using:JavlibraryBaseUrl -Session:$using:Session
                         } elseif ($using:jvJavlibraryUrl) {
                             $jvJavlibraryUrl = $using:jvJavlibraryUrl
                             if ($jvJavlibraryUrl) {
-                                $jvJavlibraryUrl.En | Get-JavlibraryData -JavlibraryBaseUrl $using:JavlibraryBaseUrl
+                                $jvJavlibraryUrl.En | Get-JavlibraryData -JavlibraryBaseUrl $using:JavlibraryBaseUrl -Session:$using:Session
                             }
                         }
                     } | Out-Null