Merge pull request #95 from jvlflame/dev

2.0.1
javinizer · Sep 21, 2020 · 6a2fcc7 · 6a2fcc7
2 parents 918fe8d + 54ef250
commit 6a2fcc7
Show file tree

Hide file tree

Showing 12 changed files with 116 additions and 78 deletions.
diff --git a/.github/CHANGELOG.md b/.github/CHANGELOG.md
@@ -5,7 +5,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
-## [2.0.0-alpha10]
+## [2.0.1]
+
+### Added
+- Setting `location.uncensorcsv` to allow user-defined uncensor csv path
+
+### Changed
+- Dmm scraper now includes TrailerUrl
+- `-Update` parameter now downloads metadata items in addition to updating the nfo file
+- Actress matcher logic changed for more accurate results
+- Changed default file matching logic to better clean and match filenames downloaded from torrent sites
+
+### Fixed
+- Javlibrary ratings are now properly assigned to the nfo
+- Dmm scraper now correctly gets actresses
+- Regex file match no longer incorrectly changes the movie ID during sort
+- `sort.metadata.nfo.firstnameorder` now correctly applies to `<ACTORS>` in sort formats
+
+## [2.0.0]
 
 ### Added
 
@@ -21,6 +38,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 
+- JapaneseName added to nfo file as altname
 - Setting `throttlelimit` limited to 5
 - R18 uncensored words and replacements are now moved to a file at module root `jvUncensor.csv`
 - Path will default to your current commandline location if `-Path` is not specified and setting `location.input` is blank

diff --git a/README.md b/README.md
@@ -144,7 +144,7 @@ PARAMETERS
     Specifies to not automatically try to match filenames to the movie ID. Can be useful for movies like T28- and R18-.
 
 -Update [<SwitchParameter>]
-    Specifies to only create/update the nfo file when sorting a file.
+    Specifies to only create/update metadata files without moving any existing files.
 
 -MoveToFolder <Boolean>
     Specifies whether or not to move sorted files to its own folder. Defaults to 'sort.movetofolder' in the settings file.
@@ -292,11 +292,11 @@ Sorts a path of JAV files without attemping automatic filename cleaning.
 
 -------------------------- EXAMPLE 5 --------------------------
 
-PS > Javinizer -Path 'C:\JAV\Sorted' -DestinationPath 'C:\JAV\Sorted' -Update
+PS > Javinizer -Path 'C:\JAV\Sorted' -Recurse -Update
 
 Description
 -----------
-Sorts a path of JAV files while only updating/creating the nfo file without moving/renaming any existing files.
+Sorts a path of JAV files while only updating/creating metadata without moving any existing files.
 
 -------------------------- EXAMPLE 6 --------------------------
 
@@ -446,7 +446,7 @@ It is enabled by the following settings:
 
 After your scraper data is aggregated using your metadata priority settings, Javinizer will automatically attempt to match the actress by `JapaneseName` -> `FirstName` -> `LastName FirstName`.
 
-If the actress is matched, then the full details (FirstName, LastName, JapaneseName, ThumbUrl) are written to that actress. The FullName column is not used in any way except for reference.
+If a unique actress is matched from the csv, then the full details (FirstName, LastName, JapaneseName, ThumbUrl) are written to that actress. The FullName column is not used in any way except for reference.
 
 If `sort.metadata.thumbcsv.convertalias` is enabled in addition to `sort.metadata.thumbcsv`, then Javinizer will automatically convert any of the listed aliases (separated by `|` and entered in `LastName FirstName` format) to the actress that the alias corresponds to. Using the JapaneseName as the alias will yield the most accurate results.
 
@@ -491,11 +491,12 @@ For example, if your jvGenres.csv file looks like this:
 
 | Setting | Description | Accepted or Example Value |
 | ------------- | ------------- | ------------- |
-| `throttlelimit` | Specifies the limit Javinizer will run video sorting threads. | 1-10
+| `throttlelimit` | Specifies the limit Javinizer will run video sorting threads. | 1-5
 | `location.input` | Specifies the default -Path that Javinizer will use to sort videos. | C:\\\JAV\\\Unsorted
 | `location.output` | Specifies the default -DestinationPath that Javinizer will use to sort videos. | C:\\\JAV\\\Unsorted
 | `location.thumbcsv` | Specifies the location of the actress thumbnail csv that is used to better match actresses. This will point to the file within the Javinizer module folder by default. | C:\\\JAV\\\jvThumbs.csv
 | `location.genrecsv` | Specifies the location of the genre replacement csv that is used to do a string replacement of genres of your choice. This will point to the file within your Javinizer module folder by default. | C:\\\JAV\\\jvGenres.csv
+| `location.uncensorcsv` | Specifies the location of the uncensor csv that is used to do string replacements of censored words in R18 metadata. This will point to the file within your Javinizer module folder by default. | C:\\\JAV\\\Uncensor.csv
 | `location.log` | Specifies the location of the log file. This will point to the file within the Javinizer module folder by default. | C:\\\JAV\\\jvLogs.log
 | `scraper.movie.dmm` | Specifies whether the dmm.co.jp scraper is on/off. | 0, 1
 | `scraper.movie.dmmja` | Specifies whether the dmm.co.jp japanese scraper is on/off. | 0, 1

diff --git a/src/Javinizer/Javinizer.psd1 b/src/Javinizer/Javinizer.psd1
@@ -13,7 +13,7 @@
 
     # Version number of this module.
 
-    ModuleVersion     = '2.0.0'
+    ModuleVersion     = '2.0.1'
 
     # Supported PSEditions
     # CompatiblePSEditions = @('Core')

diff --git a/src/Javinizer/Private/Convert-JVTitle.ps1 b/src/Javinizer/Private/Convert-JVTitle.ps1
@@ -31,15 +31,11 @@ function Convert-JVTitle {
         # Unwanted strings in files to remove
         $RemoveStrings = @(
             # Prefixes
-            'hjd2048.com-',
-            '1080fhd.com_',
+            '[\u3040-\u309f]|[\u30a0-\u30ff]|[\uff66-\uff9f]|[\u4e00-\u9faf]',
+            '[@|-|_]?[a-zA-Z0-9]+(\.com|\.net|\.tk)[_|-]?',
+            '^_'
             '^[0-9]{4}',
-            'xhd1080.com',
-            'ShareSex.net',
-            'jav365.com_',
             '069-3XPLANET-',
-            'fun2048.com@',
-            'javl.in_',
             'Watch18plus-',
             '\[(.*?)\]',
             'FHD-',
@@ -106,7 +102,6 @@ function Convert-JVTitle {
                         $fileBaseNameUpper[$index] = $file
                     }
                 }
-
                 $index++
             }
         } else {
@@ -288,12 +283,14 @@ function Convert-JVTitle {
                 }
                 $contentId = $splitId[0] + $splitId[1].PadLeft(5, '0') + $appendChar
                 $contentId = $contentId.Trim()
+            } elseif ($RegexEnabled) {
+                $movieId = $fileBaseNameUpperCleaned[$x]
+                $contentId = $fileBaseNameUpperCleaned[$x]
             } else {
                 $movieId = ($fileBaseNameUpperCleaned[$x] -split '\d', 3 | Where-Object { $_ -ne '' }) -join '-'
                 $contentId = $fileBaseNameUpperCleaned[$x]
             }
 
-
             if ($Files.Count -eq '1') {
                 $originalFileName = $Files.Name
                 $originalBaseName = $Files.BaseName
@@ -309,7 +306,7 @@ function Convert-JVTitle {
             }
 
             if ($Strict.IsPresent) {
-                $dataObject += [pscustomobject]@{
+                $dataObject += [PSCustomObject]@{
                     Id         = $originalBaseName
                     ContentId  = $contentId
                     FileName   = $originalFileName
@@ -321,7 +318,7 @@ function Convert-JVTitle {
                     PartNumber = $filePartNumber
                 }
             } else {
-                $dataObject += [pscustomobject]@{
+                $dataObject += [PSCustomObject]@{
                     Id         = $movieId
                     ContentId  = $contentId
                     FileName   = $originalFileName

diff --git a/src/Javinizer/Private/Scraper.Dmm.ps1 b/src/Javinizer/Private/Scraper.Dmm.ps1
@@ -231,7 +231,8 @@ function Get-DmmActress {
     process {
         $movieActressObject = @()
         try {
-            $movieActress = ($Webrequest.Content | Select-String -Pattern '\/article=actress\/id=(\d*)\/">(.*)<\/a>' -AllMatches).Matches
+            $actressBlock = (($Webrequest.Content -split '<td align="right" valign="top" class="nw">(出演者：|Performers:)<\/td>')[2] -split '<\/td>')[0]
+            $movieActress = ($actressBlock | Select-String -Pattern '\/article=actress\/id=(\d*)\/">(.*)<\/a>' -AllMatches).Matches
         } catch {
             return
         }
@@ -293,7 +294,7 @@ function Get-DmmActress {
                     $cookie.Domain = 'dmm.co.jp'
                     $session.Cookies.Add($cookie)
                     try {
-                        $jaActressName = ((((Invoke-WebRequest -Uri $jaActressUrl -WebSession $session -Verbose:$false).Content | Select-String -Pattern '<title>(.*)</title>').Matches.Groups[1].Value -split '-')[0] -replace '\(.*\)', '').Trim()
+                        $jaActressName = ((((Invoke-WebRequest -Uri $jaActressUrl -WebSession $session -Verbose:$false).Content | Select-String -Pattern '<title>(.*)</title>').Matches.Groups[1].Value -split '-')[0] -replace '\(.*\)', '' -replace '（.*）').Trim()
                     } catch {
                         $jaActressName = $null
                     }
@@ -346,7 +347,7 @@ function Get-DmmGenre {
             return
         }
 
-        if ($null -ne $genre -or $genre -ne '') {
+        if ($null -ne $genre -and $genre -ne '') {
             $genre = $genre | ForEach-Object { $genreArray += $_.Groups[1].Value -replace '<.*>', '' }
         }
 
@@ -400,8 +401,8 @@ function Get-DmmTrailerUrl {
 
     process {
         $trailerUrl = @()
-        $iFrameUrl = 'https://www.dmm.co.jp' + ($Webrequest.Content | Select-String -Pattern "onclick.+sampleplay\('([^']+)'\)").Matches.Groups[1].Value
         try {
+            $iFrameUrl = 'https://www.dmm.co.jp' + ($Webrequest.Content | Select-String -Pattern "onclick.+sampleplay\('([^']+)'\)").Matches.Groups[1].Value
             $trailerPageUrl = ((Invoke-WebRequest -Uri $iFrameUrl -WebSession $session -Verbose:$false).Content | Select-String -Pattern 'src="([^"]+)"').Matches.Groups[1].Value -replace '/en', ''
             $trailerUrl = ((Invoke-WebRequest -Uri $trailerPageUrl -Verbose:$false).Content | Select-String -Pattern '\\/\\/cc3001\.dmm\.co\.jp\\/litevideo\\/freepv[^"]+').Matches.Groups[0].Value -replace '\\', ''
             if ($trailerUrl -match '^//') {
@@ -414,4 +415,3 @@ function Get-DmmTrailerUrl {
         Write-Output "https:$trailerUrl"
     }
 }
-
diff --git a/src/Javinizer/Private/Scraper.Javlibrary.ps1 b/src/Javinizer/Private/Scraper.Javlibrary.ps1
@@ -132,7 +132,11 @@ function Get-JavlibraryRating {
     process {
         $rating = (((($Webrequest.Content -split '<div id="video_review" class="item">')[1]) -split '<\/span>')[0] -split '<span class="score">')[1]
         $rating = ($rating -replace '\(', '') -replace '\)', ''
-        Write-Output $rating
+        $ratingObject = [PSCustomObject]@{
+            Rating = $rating
+            Votes  = $null
+        }
+        Write-Output $ratingObject
     }
 }
 

diff --git a/src/Javinizer/Private/Test-JVSettings.ps1 b/src/Javinizer/Private/Test-JVSettings.ps1
@@ -63,7 +63,8 @@ function Test-JVSettings {
             'location.input',
             'location.log',
             'location.output',
-            'location.thumbcsv'
+            'location.thumbcsv',
+            'location.uncensorcsv'
         ) | Test-JVSettingsGroup -Settings $Settings -Type Path
 
         $booleanSettings = @(

diff --git a/src/Javinizer/Public/Get-JVAggregatedData.ps1 b/src/Javinizer/Public/Get-JVAggregatedData.ps1
@@ -158,6 +158,7 @@ function Get-JVAggregatedData {
             $DelimiterFormat = $Settings.'sort.format.delimiter'
             $ActressLanguageJa = $Settings.'sort.metadata.nfo.actresslanguageja'
             $ThumbCsvAutoAdd = $Settings.'sort.metadata.thumbcsv.autoadd'
+            $FirstNameOrder = $Settings.'sort.metadata.nfo.firstnameorder'
             if ($Settings.'location.genrecsv' -ne '') {
                 $GenreCsvPath = $Settings.'location.genrecsv'
             }
@@ -386,53 +387,54 @@ function Get-JVAggregatedData {
                 for ($x = 0; $x -lt $aggregatedDataObject.Actress.Count; $x++) {
                     $matched = @()
                     $matchedActress = @()
-
                     if (($aggregatedDataObject.Actress[$x].JapaneseName -ne '') -and ($matched = Compare-Object -ReferenceObject $actressCsv -DifferenceObject $aggregatedDataObject.Actress[$x] -IncludeEqual -ExcludeDifferent -PassThru -Property @('JapaneseName'))) {
+                        $originalActressString = $aggregatedDataObject.Actress[$x] | ConvertTo-Json -Compress
                         if ($matched.Count -eq 1) {
                             $matchedActress = $matched
-                        } elseif ($matched.Count -gt 1) {
-                            $matchedActress = $matched[0]
-                        }
-
-                        if ($null -ne $matchedActress) {
-                            $originalActressString = $aggregatedDataObject.Actress[$x] | ConvertTo-Json -Compress
                             $aggregatedDataObject.Actress[$x].FirstName = $matchedActress.FirstName
                             $aggregatedDataObject.Actress[$x].LastName = $matchedActress.LastName
-                            if ($null -eq $aggregatedDataObject.Actress[$x].ThumbUrl) {
+                            $aggregatedDataObject.Actress[$x].JapaneseName = $matchedActress.JapaneseName
+                            if ($null -eq $aggregatedDataObject.ThumbUrl -or $aggregatedDataObject.ThumbUrl -eq '') {
                                 $aggregatedDataObject.Actress[$x].ThumbUrl = $matchedActress.ThumbUrl
                             }
                             $actressString = $aggregatedDataObject.Actress[$x] | ConvertTo-Json -Compress
                             Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$($Data[0].Id)] [$($MyInvocation.MyCommand.Name)] [Actress - $originalActressString] matched to [$actressString]"
+                        } elseif ($matched.Count -gt 1) {
+                            $matchedActress = $matched | Where-Object { $_.FirstName -like $aggregatedDataObject.Actress[$x].FirstName -and $_.LastName -like $aggregatedDataObject.Actress[$x].LastName }
+                            if ($matchedActress.Count -eq 1) {
+                                $aggregatedDataObject.Actress[$x].FirstName = $matchedActress.FirstName
+                                $aggregatedDataObject.Actress[$x].LastName = $matchedActress.LastName
+                                $aggregatedDataObject.Actress[$x].JapaneseName = $matchedActress.JapaneseName
+                                if ($null -eq $aggregatedDataObject.ThumbUrl -or $aggregatedDataObject.ThumbUrl -eq '') {
+                                    $aggregatedDataObject.Actress[$x].ThumbUrl = $matchedActress.ThumbUrl
+                                }
+                                $actressString = $aggregatedDataObject.Actress[$x] | ConvertTo-Json -Compress
+                                Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$($Data[0].Id)] [$($MyInvocation.MyCommand.Name)] [Actress - $originalActressString] matched to [$actressString]"
+                            }
                         }
                     } elseif (($aggregatedDataObject.Actress[$x].LastName -eq '' -and $aggregatedDataObject.Actress[$x].FirstName -ne '') -and ($matched = Compare-Object -ReferenceObject ($actressCsv | Where-Object { $_.LastName -eq '' }) -DifferenceObject $aggregatedDataObject.Actress[$x] -IncludeEqual -ExcludeDifferent -PassThru -Property @('FirstName'))) {
+                        $originalActressString = $aggregatedDataObject.Actress[$x] | ConvertTo-Json -Compress
                         if ($matched.Count -eq 1) {
                             $matchedActress = $matched
-                        } elseif ($matched.Count -gt 1) {
-                            $matchedActress = $matched[0]
-                        }
-
-                        if ($null -ne $matchedActress) {
-                            $originalActressString = $aggregatedDataObject.Actress[$x] | ConvertTo-Json -Compress
-                            if ($null -eq $aggregatedDataObject.Actress[$x].ThumbUrl) {
+                            $aggregatedDataObject.Actress[$x].FirstName = $matchedActress.FirstName
+                            $aggregatedDataObject.Actress[$x].LastName = $matchedActress.LastName
+                            $aggregatedDataObject.Actress[$x].JapaneseName = $matchedActress.JapaneseName
+                            if ($null -eq $aggregatedDataObject.ThumbUrl -or $aggregatedDataObject.ThumbUrl -eq '') {
                                 $aggregatedDataObject.Actress[$x].ThumbUrl = $matchedActress.ThumbUrl
                             }
-                            $aggregatedDataObject.Actress[$x].JapaneseName = $matchedActress.JapaneseName
                             $actressString = $aggregatedDataObject.Actress[$x] | ConvertTo-Json -Compress
                             Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$($Data[0].Id)] [$($MyInvocation.MyCommand.Name)] [Actress - $originalActressString] matched to [$actressString]"
                         }
                     } elseif ($matched = Compare-Object -ReferenceObject $actressCsv -DifferenceObject $aggregatedDataObject.Actress[$x] -IncludeEqual -ExcludeDifferent -PassThru -Property @('FirstName', 'LastName')) {
+                        $originalActressString = $aggregatedDataObject.Actress[$x] | ConvertTo-Json -Compress
                         if ($matched.Count -eq 1) {
                             $matchedActress = $matched
-                        } elseif ($matched.Count -gt 1) {
-                            $matchedActress = $matched[0]
-                        }
-
-                        if ($null -ne $matchedActress) {
-                            $originalActressString = $aggregatedDataObject.Actress[$x] | ConvertTo-Json -Compress
-                            if ($null -eq $aggregatedDataObject[$x].ThumbUrl) {
+                            $aggregatedDataObject.Actress[$x].FirstName = $matchedActress.FirstName
+                            $aggregatedDataObject.Actress[$x].LastName = $matchedActress.LastName
+                            $aggregatedDataObject.Actress[$x].JapaneseName = $matchedActress.JapaneseName
+                            if ($null -eq $aggregatedDataObject.ThumbUrl -or $aggregatedDataObject.ThumbUrl -eq '') {
                                 $aggregatedDataObject.Actress[$x].ThumbUrl = $matchedActress.ThumbUrl
                             }
-                            $aggregatedDataObject.Actress[$x].JapaneseName = $matchedActress.JapaneseName
                             $actressString = $aggregatedDataObject.Actress[$x] | ConvertTo-Json -Compress
                             Write-JVLog -Write:$script:JVLogWrite -LogPath $script:JVLogPath -WriteLevel $script:JVLogWriteLevel -Level Debug -Message "[$($Data[0].Id)] [$($MyInvocation.MyCommand.Name)] [Actress - $originalActressString] matched to [$actressString]"
                         }