Updated - feat: UI - Performer scraper result list with image and basic data

graphql/documents/data/config.graphql

@@ -13,6 +13,7 @@ fragment ConfigGeneralData on ConfigGeneralResult {
   calculateMD5
   videoFileNamingAlgorithm
   parallelTasks
+  concurrentGetImages
   previewAudio
   previewSegments
   previewSegmentDuration

graphql/schema/types/config.graphql

@@ -53,6 +53,8 @@ input ConfigGeneralInput {
   videoFileNamingAlgorithm: HashAlgorithm
   """Number of parallel tasks to start during scan/generate"""
   parallelTasks: Int
+  """Number of concurrent workers to download and convert images url to base64"""
+  concurrentGetImages: Int
   """Include audio stream in previews"""
   previewAudio: Boolean
   """Number of segments in a preview file"""
@@ -156,6 +158,8 @@ type ConfigGeneralResult {
   videoFileNamingAlgorithm: HashAlgorithm!
   """Number of parallel tasks to start during scan/generate"""
   parallelTasks: Int!
+  """Number of concurrent workers to download and convert images url to base64"""
+  concurrentGetImages: Int!
   """Include audio stream in previews"""
   previewAudio: Boolean!
   """Number of segments in a preview file"""

internal/api/resolver_mutation_configure.go

@@ -161,6 +161,10 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen
 		c.Set(config.ParallelTasks, *input.ParallelTasks)
 	}
 
+	if input.ConcurrentGetImages != nil {
+		c.Set(config.ConcurrentGetImages, *input.ConcurrentGetImages)
+	}
+
 	if input.PreviewAudio != nil {
 		c.Set(config.PreviewAudio, *input.PreviewAudio)
 	}

internal/api/resolver_query_configuration.go

@@ -94,6 +94,7 @@ func makeConfigGeneralResult() *ConfigGeneralResult {
 		CalculateMd5:                 config.IsCalculateMD5(),
 		VideoFileNamingAlgorithm:     config.GetVideoFileNamingAlgorithm(),
 		ParallelTasks:                config.GetParallelTasks(),
+		ConcurrentGetImages:          config.GetConcurrentGetImages(),
 		PreviewAudio:                 config.GetPreviewAudio(),
 		PreviewSegments:              config.GetPreviewSegments(),
 		PreviewSegmentDuration:       config.GetPreviewSegmentDuration(),

internal/manager/config/config.go

@@ -72,6 +72,9 @@ const (
 	SequentialScanning        = "sequential_scanning"
 	SequentialScanningDefault = false
 
+	ConcurrentGetImages        = "concurrent_get_images"
+	concurrentGetImagesDefault = 10
+
 	PreviewPreset = "preview_preset"
 
 	PreviewAudio        = "preview_audio"
@@ -762,6 +765,23 @@ func (i *Instance) GetParallelTasksWithAutoDetection() int {
 	return parallelTasks
 }
 
+// GetConcurrentGetImages returns the number of Concurrent workers that should be started
+// to download and convert images url to base64.
+func (i *Instance) GetConcurrentGetImages() int {
+	return i.getInt(ConcurrentGetImages)
+}
+
+// GetConcurrentGetImagesOrDefault returns the number of Concurrent workers that should be started
+// to download and convert images url to base64. returns the default value if is below 1
+func (i *Instance) GetConcurrentGetImagesOrDefault() int {
+	result := i.GetConcurrentGetImages()
+	if result < 1 {
+		return concurrentGetImagesDefault
+	}
+
+	return result
+}
+
 func (i *Instance) GetPreviewAudio() bool {
 	return i.getBool(PreviewAudio)
 }
@@ -1472,6 +1492,7 @@ func (i *Instance) setDefaultValues(write bool) error {
 
 	i.main.SetDefault(ParallelTasks, parallelTasksDefault)
 	i.main.SetDefault(SequentialScanning, SequentialScanningDefault)
+	i.main.SetDefault(ConcurrentGetImages, concurrentGetImagesDefault)
 	i.main.SetDefault(PreviewSegmentDuration, previewSegmentDurationDefault)
 	i.main.SetDefault(PreviewSegments, previewSegmentsDefault)
 	i.main.SetDefault(PreviewExcludeStart, previewExcludeStartDefault)

internal/manager/config/config_concurrency_test.go

@@ -48,6 +48,7 @@ func TestConcurrentConfigAccess(t *testing.T) {
 				i.Set(VideoFileNamingAlgorithm, i.GetVideoFileNamingAlgorithm())
 				i.Set(ScrapersPath, i.GetScrapersPath())
 				i.Set(ScraperUserAgent, i.GetScraperUserAgent())
+				i.Set(ConcurrentGetImages, i.GetConcurrentGetImages())
 				i.Set(ScraperCDPPath, i.GetScraperCDPPath())
 				i.Set(ScraperCertCheck, i.GetScraperCertCheck())
 				i.Set(ScraperExcludeTagPatterns, i.GetScraperExcludeTagPatterns())

pkg/scraper/cache.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/stashapp/stash/pkg/fsutil"
@@ -42,6 +43,7 @@ type GlobalConfig interface {
 	GetScraperCertCheck() bool
 	GetPythonPath() string
 	GetProxy() string
+	GetConcurrentGetImages() int
 }
 
 func isCDPPathHTTP(c GlobalConfig) bool {
@@ -227,6 +229,12 @@ func (c Cache) findScraper(scraperID string) scraper {
 	return nil
 }
 
+type PostScrapedItem struct {
+	index int
+	item  ScrapedContent
+	err   error
+}
+
 func (c Cache) ScrapeName(ctx context.Context, id, query string, ty ScrapeContentType) ([]ScrapedContent, error) {
 	// find scraper with the provided id
 	s := c.findScraper(id)
@@ -242,7 +250,62 @@ func (c Cache) ScrapeName(ctx context.Context, id, query string, ty ScrapeConten
 		return nil, fmt.Errorf("%w: cannot use scraper %s to scrape by name", ErrNotSupported, id)
 	}
 
-	return ns.viaName(ctx, c.client, query, ty)
+	content, err := ns.viaName(ctx, c.client, query, ty)
+	if err != nil {
+		return nil, fmt.Errorf("error while name scraping with scraper %s: %w", id, err)
+	}
+
+	// prevent extra code executions and locks if there's no result.
+	if content == nil || len(content) < 1 {
+		return content, nil
+	}
+
+	// post-scraping items concurrently
+	// post scraping may download images then it would be slow if done single threaded on a big list.
+
+	concurrentGetImages := c.globalConfig.GetConcurrentGetImages()
+	threads := concurrentGetImages
+	// Check if the Threads is more than total results then set threads to the result's count.
+	if threads > len(content) {
+		threads = len(content)
+	}
+	channelQueue := make(chan PostScrapedItem, len(content))
+	channelResult := make(chan PostScrapedItem, len(content))
+	wg := sync.WaitGroup{}
+	wg.Add(threads)
+	for i := 0; i < threads; i++ {
+		go func(queue chan PostScrapedItem) {
+			for {
+				item, ok := <-queue // Get the queue item and remove it.
+				if !ok {            // if there is nothing to do and the channel has been closed then end the goroutine
+					wg.Done()
+					return
+				}
+				postScrape, err := c.postScrape(ctx, item.item)
+				result := PostScrapedItem{index: item.index, item: postScrape, err: err}
+				channelResult <- result // Add the result to result channel
+			}
+		}(channelQueue)
+	}
+
+	// Now the jobs can be added to the channel, which is used as a queue
+	for i := 0; i < len(content); i++ {
+		channelQueue <- PostScrapedItem{index: i, item: content[i], err: nil} // add to queue
+	}
+
+	close(channelQueue) // This tells the goroutines there's nothing else to do
+	wg.Wait()           // Wait for the threads to finish
+	close(channelResult)
+
+	// Set post-scraped results
+	for postScrapedItem := range channelResult {
+		if postScrapedItem.err != nil {
+			return nil, fmt.Errorf("error while post scraping with scraper %s: %w", id, err)
+		}
+		content[postScrapedItem.index] = postScrapedItem.item
+	}
+
+	return content, err
 }
 
 // ScrapeFragment uses the given fragment input to scrape

pkg/scraper/image.go

@@ -3,28 +3,42 @@ package scraper
 import (
 	"context"
 	"fmt"
+	"github.com/stashapp/stash/pkg/logger"
 	"io"
 	"net/http"
+	pkg_neturl "net/url"
 	"strings"
 
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/utils"
 )
 
 func setPerformerImage(ctx context.Context, client *http.Client, p *models.ScrapedPerformer, globalConfig GlobalConfig) error {
-	if p.Image == nil || !strings.HasPrefix(*p.Image, "http") {
-		// nothing to do
-		return nil
+	if p.Images != nil && len(p.Images) > 0 {
+		for i := 0; i < len(p.Images); i++ {
+			if strings.HasPrefix(p.Images[i], "http") {
+				img, err := getImage(ctx, p.Images[i], client, globalConfig)
+				if err != nil {
+					logger.Warnf("Could not set image using URL %s: %s", p.Images[i], err.Error())
+					return err
+				}
+
+				p.Images[i] = *img
+				// Image is deprecated. Use images instead
+			}
+		}
 	}
 
-	img, err := getImage(ctx, *p.Image, client, globalConfig)
-	if err != nil {
-		return err
-	}
+	if p.Image != nil && strings.HasPrefix(*p.Image, "http") {
+		img, err := getImage(ctx, *p.Image, client, globalConfig)
+		if err != nil {
+			return err
+		}
 
-	p.Image = img
-	// Image is deprecated. Use images instead
-	p.Images = []string{*img}
+		p.Image = img
+		// Image is deprecated. Use images instead
+		p.Images = append([]string{*img}, p.Images...)
+	}
 
 	return nil
 }
@@ -81,6 +95,13 @@ func setMovieBackImage(ctx context.Context, client *http.Client, m *models.Scrap
 }
 
 func getImage(ctx context.Context, url string, client *http.Client, globalConfig GlobalConfig) (*string, error) {
+	if strings.HasPrefix(url, "https%3A") || strings.HasPrefix(url, "http%3A") {
+		urlDecoded, err := pkg_neturl.PathUnescape(url)
+		if err != nil {
+			return nil, err
+		}
+		url = urlDecoded
+	}
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
 		return nil, err

pkg/scraper/mapped.go

@@ -47,9 +47,9 @@ func (s mappedConfig) process(ctx context.Context, q mappedQuery, common commonM
 	for k, attrConfig := range s {
 
 		if attrConfig.Fixed != "" {
-			// TODO - not sure if this needs to set _all_ indexes for the key
-			const i = 0
-			ret = ret.setKey(i, k, attrConfig.Fixed)
+			for i := range ret {
+				ret = ret.setKey(i, k, attrConfig.Fixed)
+			}
 		} else {
 			selector := attrConfig.Selector
 			selector = s.applyCommon(common, selector)
@@ -747,12 +747,15 @@ func (r mappedResult) apply(dest interface{}) {
 
 		if field.IsValid() {
 			var reflectValue reflect.Value
-			if field.Kind() == reflect.Ptr {
+			switch field.Kind() {
+			case reflect.Ptr:
 				// need to copy the value, otherwise everything is set to the
 				// same pointer
 				localValue := value
 				reflectValue = reflect.ValueOf(&localValue)
-			} else {
+			case reflect.Slice:
+				reflectValue = reflect.ValueOf([]string{value})
+			default:
 				reflectValue = reflect.ValueOf(value)
 			}
 

pkg/scraper/postprocessing.go

@@ -63,7 +63,9 @@ func (c Cache) postScrapePerformer(ctx context.Context, p models.ScrapedPerforme
 
 	// post-process - set the image if applicable
 	if err := setPerformerImage(ctx, c.client, &p, c.globalConfig); err != nil {
-		logger.Warnf("Could not set image using URL %s: %s", *p.Image, err.Error())
+		if p.Image != nil {
+			logger.Warnf("Could not set image using URL %s: %s", *p.Image, err.Error())
+		}
 	}
 
 	p.Country = resolveCountryName(p.Country)

pkg/scraper/xpath_test.go

@@ -838,6 +838,10 @@ func (mockGlobalConfig) GetProxy() string {
 	return ""
 }
 
+func (mockGlobalConfig) GetConcurrentGetImages() int {
+	return 4
+}
+
 func TestSubScrape(t *testing.T) {
 	retHTML := `
 	<div>

ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx

@@ -1,4 +1,4 @@
-import React, { useEffect, useMemo, useState } from "react";
+import React, { useEffect, useMemo, useState, lazy } from "react";
 import { Button, Tabs, Tab, Col, Row } from "react-bootstrap";
 import { FormattedMessage, useIntl } from "react-intl";
 import { useParams, useHistory } from "react-router-dom";
@@ -28,17 +28,19 @@ import { PerformerScenesPanel } from "./PerformerScenesPanel";
 import { PerformerGalleriesPanel } from "./PerformerGalleriesPanel";
 import { PerformerMoviesPanel } from "./PerformerMoviesPanel";
 import { PerformerImagesPanel } from "./PerformerImagesPanel";
-import { PerformerEditPanel } from "./PerformerEditPanel";
 import { PerformerSubmitButton } from "./PerformerSubmitButton";
 import GenderIcon from "../GenderIcon";
 import { faHeart, faLink } from "@fortawesome/free-solid-svg-icons";
 import { faInstagram, faTwitter } from "@fortawesome/free-brands-svg-icons";
 import { IUIConfig } from "src/core/config";
 import { useRatingKeybinds } from "src/hooks/keybinds";
 
+const PerformerEditPanel = lazy(() => import("./PerformerEditPanel"));
+
 interface IProps {
   performer: GQL.PerformerDataFragment;
 }
+
 interface IPerformerParams {
   tab?: string;
 }
@@ -429,6 +431,19 @@ const PerformerPage: React.FC<IProps> = ({ performer }) => {
 const PerformerLoader: React.FC = () => {
   const { id } = useParams<{ id?: string }>();
   const { data, loading, error } = useFindPerformer(id ?? "");
+  const { configuration } = React.useContext(ConfigurationContext);
+  const [showScrubber, setShowScrubber] = useState(
+    configuration?.interface.showScrubber ?? true
+  );
+
+  // set up hotkeys
+  useEffect(() => {
+    Mousetrap.bind(".", () => setShowScrubber(!showScrubber));
+
+    return () => {
+      Mousetrap.unbind(".");
+    };
+  });
 
   if (loading) return <LoadingIndicator />;
   if (error) return <ErrorMessage error={error.message} />;