diff --git a/graphql/documents/data/config.graphql b/graphql/documents/data/config.graphql index f102e217173..80c85d64f74 100644 --- a/graphql/documents/data/config.graphql +++ b/graphql/documents/data/config.graphql @@ -13,6 +13,7 @@ fragment ConfigGeneralData on ConfigGeneralResult { calculateMD5 videoFileNamingAlgorithm parallelTasks + concurrentGetImages previewAudio previewSegments previewSegmentDuration diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index 359229a4a19..67f5628dcf4 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -53,6 +53,8 @@ input ConfigGeneralInput { videoFileNamingAlgorithm: HashAlgorithm """Number of parallel tasks to start during scan/generate""" parallelTasks: Int + """Number of concurrent workers to download and convert images url to base64""" + concurrentGetImages: Int """Include audio stream in previews""" previewAudio: Boolean """Number of segments in a preview file""" @@ -156,6 +158,8 @@ type ConfigGeneralResult { videoFileNamingAlgorithm: HashAlgorithm! """Number of parallel tasks to start during scan/generate""" parallelTasks: Int! + """Number of concurrent workers to download and convert images url to base64""" + concurrentGetImages: Int! """Include audio stream in previews""" previewAudio: Boolean! """Number of segments in a preview file""" diff --git a/internal/api/resolver_mutation_configure.go b/internal/api/resolver_mutation_configure.go index 56c14867bd9..934f19f2a78 100644 --- a/internal/api/resolver_mutation_configure.go +++ b/internal/api/resolver_mutation_configure.go @@ -161,6 +161,10 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen c.Set(config.ParallelTasks, *input.ParallelTasks) } + if input.ConcurrentGetImages != nil { + c.Set(config.ConcurrentGetImages, *input.ConcurrentGetImages) + } + if input.PreviewAudio != nil { c.Set(config.PreviewAudio, *input.PreviewAudio) } diff --git a/internal/api/resolver_query_configuration.go b/internal/api/resolver_query_configuration.go index b9aae65de0a..2ea34e35302 100644 --- a/internal/api/resolver_query_configuration.go +++ b/internal/api/resolver_query_configuration.go @@ -94,6 +94,7 @@ func makeConfigGeneralResult() *ConfigGeneralResult { CalculateMd5: config.IsCalculateMD5(), VideoFileNamingAlgorithm: config.GetVideoFileNamingAlgorithm(), ParallelTasks: config.GetParallelTasks(), + ConcurrentGetImages: config.GetConcurrentGetImages(), PreviewAudio: config.GetPreviewAudio(), PreviewSegments: config.GetPreviewSegments(), PreviewSegmentDuration: config.GetPreviewSegmentDuration(), diff --git a/internal/manager/config/config.go b/internal/manager/config/config.go index bea1381e46c..1a6bd710507 100644 --- a/internal/manager/config/config.go +++ b/internal/manager/config/config.go @@ -72,6 +72,9 @@ const ( SequentialScanning = "sequential_scanning" SequentialScanningDefault = false + ConcurrentGetImages = "concurrent_get_images" + concurrentGetImagesDefault = 10 + PreviewPreset = "preview_preset" PreviewAudio = "preview_audio" @@ -762,6 +765,23 @@ func (i *Instance) GetParallelTasksWithAutoDetection() int { return parallelTasks } +// GetConcurrentGetImages returns the number of Concurrent workers that should be started +// to download and convert images url to base64. +func (i *Instance) GetConcurrentGetImages() int { + return i.getInt(ConcurrentGetImages) +} + +// GetConcurrentGetImagesOrDefault returns the number of Concurrent workers that should be started +// to download and convert images url to base64. returns the default value if is below 1 +func (i *Instance) GetConcurrentGetImagesOrDefault() int { + result := i.GetConcurrentGetImages() + if result < 1 { + return concurrentGetImagesDefault + } + + return result +} + func (i *Instance) GetPreviewAudio() bool { return i.getBool(PreviewAudio) } @@ -1472,6 +1492,7 @@ func (i *Instance) setDefaultValues(write bool) error { i.main.SetDefault(ParallelTasks, parallelTasksDefault) i.main.SetDefault(SequentialScanning, SequentialScanningDefault) + i.main.SetDefault(ConcurrentGetImages, concurrentGetImagesDefault) i.main.SetDefault(PreviewSegmentDuration, previewSegmentDurationDefault) i.main.SetDefault(PreviewSegments, previewSegmentsDefault) i.main.SetDefault(PreviewExcludeStart, previewExcludeStartDefault) diff --git a/internal/manager/config/config_concurrency_test.go b/internal/manager/config/config_concurrency_test.go index 81bb7e81687..6bb625161a4 100644 --- a/internal/manager/config/config_concurrency_test.go +++ b/internal/manager/config/config_concurrency_test.go @@ -48,6 +48,7 @@ func TestConcurrentConfigAccess(t *testing.T) { i.Set(VideoFileNamingAlgorithm, i.GetVideoFileNamingAlgorithm()) i.Set(ScrapersPath, i.GetScrapersPath()) i.Set(ScraperUserAgent, i.GetScraperUserAgent()) + i.Set(ConcurrentGetImages, i.GetConcurrentGetImages()) i.Set(ScraperCDPPath, i.GetScraperCDPPath()) i.Set(ScraperCertCheck, i.GetScraperCertCheck()) i.Set(ScraperExcludeTagPatterns, i.GetScraperExcludeTagPatterns()) diff --git a/pkg/scraper/cache.go b/pkg/scraper/cache.go index 3b53919947f..c6a50319f5d 100644 --- a/pkg/scraper/cache.go +++ b/pkg/scraper/cache.go @@ -9,6 +9,7 @@ import ( "path/filepath" "sort" "strings" + "sync" "time" "github.com/stashapp/stash/pkg/fsutil" @@ -42,6 +43,7 @@ type GlobalConfig interface { GetScraperCertCheck() bool GetPythonPath() string GetProxy() string + GetConcurrentGetImages() int } func isCDPPathHTTP(c GlobalConfig) bool { @@ -227,6 +229,12 @@ func (c Cache) findScraper(scraperID string) scraper { return nil } +type PostScrapedItem struct { + index int + item ScrapedContent + err error +} + func (c Cache) ScrapeName(ctx context.Context, id, query string, ty ScrapeContentType) ([]ScrapedContent, error) { // find scraper with the provided id s := c.findScraper(id) @@ -242,7 +250,62 @@ func (c Cache) ScrapeName(ctx context.Context, id, query string, ty ScrapeConten return nil, fmt.Errorf("%w: cannot use scraper %s to scrape by name", ErrNotSupported, id) } - return ns.viaName(ctx, c.client, query, ty) + content, err := ns.viaName(ctx, c.client, query, ty) + if err != nil { + return nil, fmt.Errorf("error while name scraping with scraper %s: %w", id, err) + } + + // prevent extra code executions and locks if there's no result. + if content == nil || len(content) < 1 { + return content, nil + } + + // post-scraping items concurrently + // post scraping may download images then it would be slow if done single threaded on a big list. + + concurrentGetImages := c.globalConfig.GetConcurrentGetImages() + threads := concurrentGetImages + // Check if the Threads is more than total results then set threads to the result's count. + if threads > len(content) { + threads = len(content) + } + channelQueue := make(chan PostScrapedItem, len(content)) + channelResult := make(chan PostScrapedItem, len(content)) + wg := sync.WaitGroup{} + wg.Add(threads) + for i := 0; i < threads; i++ { + go func(queue chan PostScrapedItem) { + for { + item, ok := <-queue // Get the queue item and remove it. + if !ok { // if there is nothing to do and the channel has been closed then end the goroutine + wg.Done() + return + } + postScrape, err := c.postScrape(ctx, item.item) + result := PostScrapedItem{index: item.index, item: postScrape, err: err} + channelResult <- result // Add the result to result channel + } + }(channelQueue) + } + + // Now the jobs can be added to the channel, which is used as a queue + for i := 0; i < len(content); i++ { + channelQueue <- PostScrapedItem{index: i, item: content[i], err: nil} // add to queue + } + + close(channelQueue) // This tells the goroutines there's nothing else to do + wg.Wait() // Wait for the threads to finish + close(channelResult) + + // Set post-scraped results + for postScrapedItem := range channelResult { + if postScrapedItem.err != nil { + return nil, fmt.Errorf("error while post scraping with scraper %s: %w", id, err) + } + content[postScrapedItem.index] = postScrapedItem.item + } + + return content, err } // ScrapeFragment uses the given fragment input to scrape diff --git a/pkg/scraper/image.go b/pkg/scraper/image.go index 5757bc9b383..1ea3ddc2c31 100644 --- a/pkg/scraper/image.go +++ b/pkg/scraper/image.go @@ -3,8 +3,10 @@ package scraper import ( "context" "fmt" + "github.com/stashapp/stash/pkg/logger" "io" "net/http" + pkg_neturl "net/url" "strings" "github.com/stashapp/stash/pkg/models" @@ -12,19 +14,31 @@ import ( ) func setPerformerImage(ctx context.Context, client *http.Client, p *models.ScrapedPerformer, globalConfig GlobalConfig) error { - if p.Image == nil || !strings.HasPrefix(*p.Image, "http") { - // nothing to do - return nil + if p.Images != nil && len(p.Images) > 0 { + for i := 0; i < len(p.Images); i++ { + if strings.HasPrefix(p.Images[i], "http") { + img, err := getImage(ctx, p.Images[i], client, globalConfig) + if err != nil { + logger.Warnf("Could not set image using URL %s: %s", p.Images[i], err.Error()) + return err + } + + p.Images[i] = *img + // Image is deprecated. Use images instead + } + } } - img, err := getImage(ctx, *p.Image, client, globalConfig) - if err != nil { - return err - } + if p.Image != nil && strings.HasPrefix(*p.Image, "http") { + img, err := getImage(ctx, *p.Image, client, globalConfig) + if err != nil { + return err + } - p.Image = img - // Image is deprecated. Use images instead - p.Images = []string{*img} + p.Image = img + // Image is deprecated. Use images instead + p.Images = append([]string{*img}, p.Images...) + } return nil } @@ -81,6 +95,13 @@ func setMovieBackImage(ctx context.Context, client *http.Client, m *models.Scrap } func getImage(ctx context.Context, url string, client *http.Client, globalConfig GlobalConfig) (*string, error) { + if strings.HasPrefix(url, "https%3A") || strings.HasPrefix(url, "http%3A") { + urlDecoded, err := pkg_neturl.PathUnescape(url) + if err != nil { + return nil, err + } + url = urlDecoded + } req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err diff --git a/pkg/scraper/mapped.go b/pkg/scraper/mapped.go index 77c4911ae92..4ea60e7dcee 100644 --- a/pkg/scraper/mapped.go +++ b/pkg/scraper/mapped.go @@ -47,9 +47,9 @@ func (s mappedConfig) process(ctx context.Context, q mappedQuery, common commonM for k, attrConfig := range s { if attrConfig.Fixed != "" { - // TODO - not sure if this needs to set _all_ indexes for the key - const i = 0 - ret = ret.setKey(i, k, attrConfig.Fixed) + for i := range ret { + ret = ret.setKey(i, k, attrConfig.Fixed) + } } else { selector := attrConfig.Selector selector = s.applyCommon(common, selector) @@ -747,12 +747,15 @@ func (r mappedResult) apply(dest interface{}) { if field.IsValid() { var reflectValue reflect.Value - if field.Kind() == reflect.Ptr { + switch field.Kind() { + case reflect.Ptr: // need to copy the value, otherwise everything is set to the // same pointer localValue := value reflectValue = reflect.ValueOf(&localValue) - } else { + case reflect.Slice: + reflectValue = reflect.ValueOf([]string{value}) + default: reflectValue = reflect.ValueOf(value) } diff --git a/pkg/scraper/postprocessing.go b/pkg/scraper/postprocessing.go index cf8cac1eb34..e90fc538398 100644 --- a/pkg/scraper/postprocessing.go +++ b/pkg/scraper/postprocessing.go @@ -63,7 +63,9 @@ func (c Cache) postScrapePerformer(ctx context.Context, p models.ScrapedPerforme // post-process - set the image if applicable if err := setPerformerImage(ctx, c.client, &p, c.globalConfig); err != nil { - logger.Warnf("Could not set image using URL %s: %s", *p.Image, err.Error()) + if p.Image != nil { + logger.Warnf("Could not set image using URL %s: %s", *p.Image, err.Error()) + } } p.Country = resolveCountryName(p.Country) diff --git a/pkg/scraper/xpath_test.go b/pkg/scraper/xpath_test.go index 06b6ad5b686..2f6182a1693 100644 --- a/pkg/scraper/xpath_test.go +++ b/pkg/scraper/xpath_test.go @@ -838,6 +838,10 @@ func (mockGlobalConfig) GetProxy() string { return "" } +func (mockGlobalConfig) GetConcurrentGetImages() int { + return 4 +} + func TestSubScrape(t *testing.T) { retHTML := `
diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx index 357886ffb08..ea944665f42 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useMemo, useState } from "react"; +import React, { useEffect, useMemo, useState, lazy } from "react"; import { Button, Tabs, Tab, Col, Row } from "react-bootstrap"; import { FormattedMessage, useIntl } from "react-intl"; import { useParams, useHistory } from "react-router-dom"; @@ -28,7 +28,6 @@ import { PerformerScenesPanel } from "./PerformerScenesPanel"; import { PerformerGalleriesPanel } from "./PerformerGalleriesPanel"; import { PerformerMoviesPanel } from "./PerformerMoviesPanel"; import { PerformerImagesPanel } from "./PerformerImagesPanel"; -import { PerformerEditPanel } from "./PerformerEditPanel"; import { PerformerSubmitButton } from "./PerformerSubmitButton"; import GenderIcon from "../GenderIcon"; import { faHeart, faLink } from "@fortawesome/free-solid-svg-icons"; @@ -36,9 +35,12 @@ import { faInstagram, faTwitter } from "@fortawesome/free-brands-svg-icons"; import { IUIConfig } from "src/core/config"; import { useRatingKeybinds } from "src/hooks/keybinds"; +const PerformerEditPanel = lazy(() => import("./PerformerEditPanel")); + interface IProps { performer: GQL.PerformerDataFragment; } + interface IPerformerParams { tab?: string; } @@ -429,6 +431,19 @@ const PerformerPage: React.FC = ({ performer }) => { const PerformerLoader: React.FC = () => { const { id } = useParams<{ id?: string }>(); const { data, loading, error } = useFindPerformer(id ?? ""); + const { configuration } = React.useContext(ConfigurationContext); + const [showScrubber, setShowScrubber] = useState( + configuration?.interface.showScrubber ?? true + ); + + // set up hotkeys + useEffect(() => { + Mousetrap.bind(".", () => setShowScrubber(!showScrubber)); + + return () => { + Mousetrap.unbind("."); + }; + }); if (loading) return ; if (error) return ; diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerEditPanel.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerEditPanel.tsx index cc77ccda322..20c1bcfd5ab 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerEditPanel.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerEditPanel.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useState } from "react"; +import React, { useEffect, useState, lazy } from "react"; import { Button, Form, Col, Row, Badge, Dropdown } from "react-bootstrap"; import { FormattedMessage, useIntl } from "react-intl"; import Mousetrap from "mousetrap"; @@ -6,12 +6,12 @@ import * as GQL from "src/core/generated-graphql"; import * as yup from "yup"; import { useListPerformerScrapers, - queryScrapePerformer, mutateReloadScrapers, usePerformerUpdate, usePerformerCreate, useTagCreate, queryScrapePerformerURL, + queryScrapePerformerQueryFragment, } from "src/core/StashService"; import { Icon } from "src/components/Shared/Icon"; import { ImageInput } from "src/components/Shared/ImageInput"; @@ -33,8 +33,7 @@ import { } from "src/utils/gender"; import { ConfigurationContext } from "src/hooks/Config"; import { PerformerScrapeDialog } from "./PerformerScrapeDialog"; -import PerformerScrapeModal from "./PerformerScrapeModal"; -import PerformerStashBoxModal, { IStashBox } from "./PerformerStashBoxModal"; +import { IStashBox } from "./PerformerStashBoxModal"; import cx from "classnames"; import { faPlus, @@ -43,11 +42,9 @@ import { } from "@fortawesome/free-solid-svg-icons"; import { StringListInput } from "src/components/Shared/StringListInput"; -const isScraper = ( - scraper: GQL.Scraper | GQL.StashBox -): scraper is GQL.Scraper => (scraper as GQL.Scraper).id !== undefined; +const PerformerScrapeModal = lazy(() => import("./PerformerScrapeModal")); -interface IPerformerDetails { +interface IProps { performer: Partial; isVisible: boolean; onImageChange?: (image?: string | null) => void; @@ -55,7 +52,7 @@ interface IPerformerDetails { onCancelEditing?: () => void; } -export const PerformerEditPanel: React.FC = ({ +export const PerformerEditPanel: React.FC = ({ performer, isVisible, onImageChange, @@ -67,8 +64,8 @@ export const PerformerEditPanel: React.FC = ({ const isNew = performer.id === undefined; - // Editing stat - const [scraper, setScraper] = useState(); + // Editing state + const [scraper, setScraper] = useState(); const [newTags, setNewTags] = useState(); const [isScraperModalOpen, setIsScraperModalOpen] = useState(false); @@ -490,6 +487,11 @@ export const PerformerEditPanel: React.FC = ({ formik.setFieldValue("image", url); } + function onScrapeQueryClicked(s: GQL.ScraperSourceInput) { + setScraper(s); + setIsScraperModalOpen(true); + } + async function onReloadScrapers() { setIsLoading(true); try { @@ -504,42 +506,6 @@ export const PerformerEditPanel: React.FC = ({ } } - async function onScrapePerformer( - selectedPerformer: GQL.ScrapedPerformerDataFragment, - selectedScraper: GQL.Scraper - ) { - setIsScraperModalOpen(false); - try { - if (!scraper) return; - setIsLoading(true); - - const { - __typename, - images: _image, - tags: _tags, - ...ret - } = selectedPerformer; - - const result = await queryScrapePerformer(selectedScraper.id, ret); - if (!result?.data?.scrapeSinglePerformer?.length) return; - - // assume one result - // if this is a new performer, just dump the data - if (isNew) { - updatePerformerEditStateFromScraper( - result.data.scrapeSinglePerformer[0] - ); - setScraper(undefined); - } else { - setScrapedPerformer(result.data.scrapeSinglePerformer[0]); - } - } catch (e) { - Toast.error(e); - } finally { - setIsLoading(false); - } - } - async function onScrapePerformerURL() { const { url } = formik.values; if (!url) return; @@ -563,29 +529,6 @@ export const PerformerEditPanel: React.FC = ({ } } - async function onScrapeStashBox(performerResult: GQL.ScrapedPerformer) { - setIsScraperModalOpen(false); - - const result: GQL.ScrapedPerformerDataFragment = { - ...performerResult, - images: performerResult.images ?? undefined, - __typename: "ScrapedPerformer", - }; - - // if this is a new performer, just dump the data - if (isNew) { - updatePerformerEditStateFromScraper(result); - setScraper(undefined); - } else { - setScrapedPerformer(result); - } - } - - function onScraperSelected(s: GQL.Scraper | IStashBox | undefined) { - setScraper(s); - setIsScraperModalOpen(true); - } - function renderScraperMenu() { if (!performer) { return; @@ -599,7 +542,12 @@ export const PerformerEditPanel: React.FC = ({ as={Button} key={s.endpoint} className="minimal" - onClick={() => onScraperSelected({ ...s, index })} + onClick={() => + onScrapeQueryClicked({ + stash_box_index: index, + stash_box_endpoint: s.endpoint, + }) + } > {stashboxDisplayName(s.name, index)} @@ -610,7 +558,7 @@ export const PerformerEditPanel: React.FC = ({ as={Button} key={s.name} className="minimal" - onClick={() => onScraperSelected(s)} + onClick={() => onScrapeQueryClicked({ scraper_id: s.id })} > {s.name} @@ -721,24 +669,61 @@ export const PerformerEditPanel: React.FC = ({ ); } + async function scrapeFromQuery( + s: GQL.ScraperSourceInput, + fragment: GQL.ScrapedPerformerDataFragment + ) { + setIsLoading(true); + try { + const input: GQL.ScrapedPerformerInput = { + name: fragment.name, + gender: fragment.gender, + birthdate: fragment.birthdate, + url: fragment.url, + }; + + const result = await queryScrapePerformerQueryFragment(s, input); + if (!result.data || !result.data?.scrapeSinglePerformer?.length) { + Toast.success({ + content: "No performer found", + }); + return; + } + // assume one returned scene + setScrapedPerformer(result.data?.scrapeSinglePerformer[0]); + } catch (e) { + Toast.error(e); + } finally { + setIsLoading(false); + } + } + + function onPerformerSelected(s: GQL.ScrapedPerformerDataFragment) { + if (!scraper) return; + + if (scraper?.stash_box_index !== undefined) { + // must be stash-box - assume full scene + setScrapedPerformer(s); + } else { + // must be scraper + scrapeFromQuery(scraper, s); + } + } + const renderScrapeModal = () => { - if (!isScraperModalOpen) return; + if (!isScraperModalOpen || !scraper) return; - return scraper !== undefined && isScraper(scraper) ? ( + return ( setScraper(undefined)} - onSelectPerformer={onScrapePerformer} - name={formik.values.name || ""} - /> - ) : scraper !== undefined && !isScraper(scraper) ? ( - setScraper(undefined)} - onSelectPerformer={onScrapeStashBox} + onSelectPerformer={(s) => { + setIsScraperModalOpen(false); + onPerformerSelected(s); + }} name={formik.values.name || ""} /> - ) : undefined; + ); }; function renderTagsField() { @@ -1037,3 +1022,5 @@ export const PerformerEditPanel: React.FC = ({ ); }; + +export default PerformerEditPanel; diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx index 0f62285f5ab..13808942b5e 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx @@ -5,7 +5,7 @@ import { ScrapeDialog, ScrapeResult, ScrapedInputGroupRow, - ScrapedImageRow, + ScrapedImagesRow, ScrapeDialogRow, ScrapedTextAreaRow, ScrapedCountryRow, @@ -123,7 +123,7 @@ function renderScrapedTagsRow( interface IPerformerScrapeDialogProps { performer: Partial; scraped: GQL.ScrapedPerformer; - scraper?: GQL.Scraper | IStashBox; + scraper?: GQL.ScraperSourceInput; onClose: (scrapedPerformer?: GQL.ScrapedPerformer) => void; } @@ -320,13 +320,22 @@ export const PerformerScrapeDialog: React.FC = ( const [image, setImage] = useState>( new ScrapeResult( - props.performer.image, + props.performer.image as string, props.scraped.images && props.scraped.images.length > 0 ? props.scraped.images[0] - : undefined + : props.scraped.image + ? (props.scraped.image as string) + : "" ) ); + const images = + props.scraped.images && props.scraped.images.length > 0 + ? props.scraped.images + : props.scraped.image + ? [props.scraped.image as string] + : []; + const allFields = [ name, disambiguation, @@ -545,10 +554,11 @@ export const PerformerScrapeDialog: React.FC = ( newTags, createNewTag )} - setImage(value)} /> = ({ performer }) => { + function renderImage() { + if (performer.images && performer.images.length > 0) { + return ( +
+ +
+ ); + } + } + + function calculateAge() { + if (performer?.birthdate) { + // calculate the age from birthdate. In future, this should probably be + // provided by the server + return TextUtils.age(performer.birthdate, performer.death_date); + } + } + + function renderTags() { + if (performer.tags) { + return ( + + + {performer.tags?.map((tag) => ( + + {tag.name} + + ))} + + + ); + } + } + + let calculated_age = calculateAge(); + + return ( +
+ + {renderImage()} +
+

+ {performer.name} + {performer.disambiguation && ( + + {` (${performer.disambiguation})`} + + )} +

+
+ {performer.gender && ( + + )} + {performer.gender && calculated_age && ` • `} + {calculated_age} + {calculated_age && " "} + {calculated_age && } +
+
+ + + + + + + {renderTags()} +
+ ); +}; + +export interface IPerformerSearchResult { + performer: GQL.ScrapedPerformerDataFragment; +} + +export const PerformerSearchResult: React.FC = ({ + performer, +}) => { + return ( +
+ +
+ ); +}; interface IProps { - scraper: GQL.Scraper; + scraper: GQL.ScraperSourceInput; onHide: () => void; - onSelectPerformer: ( - performer: GQL.ScrapedPerformerDataFragment, - scraper: GQL.Scraper - ) => void; + onSelectPerformer: (performer: GQL.ScrapedPerformerDataFragment) => void; name?: string; } + const PerformerScrapeModal: React.FC = ({ scraper, name, onHide, onSelectPerformer, }) => { + const CLASSNAME = "PerformerScrapeModal"; + const CLASSNAME_LIST = `${CLASSNAME}-list`; + const CLASSNAME_LIST_CONTAINER = `${CLASSNAME_LIST}-container`; + const intl = useIntl(); + const Toast = useToast(); + const inputRef = useRef(null); - const [query, setQuery] = useState(name ?? ""); - const { data, loading } = useScrapePerformerList(scraper.id, query); + const [loading, setLoading] = useState(false); + const [performers, setPerformers] = useState< + GQL.ScrapedPerformer[] | undefined + >(); + const [error, setError] = useState(); - const performers = data?.scrapeSinglePerformer ?? []; + const doQuery = useCallback( + async (input: string) => { + if (!input) return; - const onInputChange = debounce((input: string) => { - setQuery(input); - }, 500); + setLoading(true); + try { + const r = await queryScrapePerformerQuery(scraper, input); + setPerformers(r.data?.scrapeSinglePerformer); + } catch (err) { + if (err instanceof Error) setError(err); + } finally { + setLoading(false); + } + }, + [scraper] + ); useEffect(() => inputRef.current?.focus(), []); + useEffect(() => { + doQuery(name ?? ""); + }, [doQuery, name]); + useEffect(() => { + if (error) { + Toast.error(error); + setError(undefined); + } + }, [error, Toast]); + + function renderResults() { + if (!performers) { + return; + } + + return ( +
+
+ +
+
    + {performers.map((p, i) => ( + // eslint-disable-next-line jsx-a11y/click-events-have-key-events, jsx-a11y/no-noninteractive-element-interactions, react/no-array-index-key +
  • onSelectPerformer(p)}> + +
    • + ))} +
+
+ ); + } return ( = ({ }} >
- onInputChange(e.currentTarget.value)} - defaultValue={name ?? ""} - placeholder="Performer name..." - className="text-input mb-4" - ref={inputRef} - /> + + ) => + e.key === "Enter" && doQuery(inputRef.current?.value ?? "") + } + /> + + + + {loading ? (
) : ( -
    - {performers.map((p) => ( -
  • - -
    • - ))} -
+ renderResults() )}
diff --git a/ui/v2.5/src/components/Performers/Performers.tsx b/ui/v2.5/src/components/Performers/Performers.tsx index f919fdba51f..5ee01720af0 100644 --- a/ui/v2.5/src/components/Performers/Performers.tsx +++ b/ui/v2.5/src/components/Performers/Performers.tsx @@ -4,10 +4,11 @@ import { useIntl } from "react-intl"; import { Helmet } from "react-helmet"; import { TITLE_SUFFIX } from "src/components/Shared/constants"; import { PersistanceLevel } from "src/hooks/ListHook"; -import Performer from "./PerformerDetails/Performer"; import PerformerCreate from "./PerformerDetails/PerformerCreate"; import { PerformerList } from "./PerformerList"; +const Performer = React.lazy(() => import("./PerformerDetails/Performer")); + const Performers: React.FC = () => { const intl = useIntl(); diff --git a/ui/v2.5/src/components/Performers/styles.scss b/ui/v2.5/src/components/Performers/styles.scss index 2ec6433d485..d587dd139d6 100644 --- a/ui/v2.5/src/components/Performers/styles.scss +++ b/ui/v2.5/src/components/Performers/styles.scss @@ -210,3 +210,15 @@ font-size: 0.875em; /* stylelint-enable */ } + +.PerformerScrapeModal-list { + list-style: none; + max-height: 50vh; + overflow-x: hidden; + overflow-y: auto; + padding-inline-start: 0; + + li { + cursor: pointer; + } +} diff --git a/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx b/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx index 108f9652bbb..360ca61bd43 100644 --- a/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx @@ -259,7 +259,7 @@ export const SettingsConfigurationPanel: React.FC = () => { /> - + { value={general.parallelTasks ?? undefined} onChange={(v) => saveGeneral({ parallelTasks: v })} /> + + saveGeneral({ concurrentGetImages: v })} + /> diff --git a/ui/v2.5/src/components/Shared/ScrapeDialog.tsx b/ui/v2.5/src/components/Shared/ScrapeDialog.tsx index 410534e4123..863625b92e6 100644 --- a/ui/v2.5/src/components/Shared/ScrapeDialog.tsx +++ b/ui/v2.5/src/components/Shared/ScrapeDialog.tsx @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useState } from "react"; import { Form, Col, @@ -15,6 +15,8 @@ import isEqual from "lodash-es/isEqual"; import clone from "lodash-es/clone"; import { FormattedMessage, useIntl } from "react-intl"; import { + faArrowLeft, + faArrowRight, faCheck, faPencilAlt, faPlus, @@ -354,6 +356,169 @@ export const ScrapedImageRow: React.FC = (props) => { ); }; +interface IScrapedImageDialogRowProps< + T extends ScrapeResult, + V extends IHasName +> extends IScrapedFieldProps { + title: string; + renderOriginalField: () => JSX.Element | undefined; + renderNewField: () => JSX.Element | undefined; + onChange: (value: T) => void; + newValues?: V[]; + images: string[]; + onCreateNew?: (index: number) => void; +} + +export const ScrapeImageDialogRow = < + T extends ScrapeResult, + V extends IHasName +>( + props: IScrapedImageDialogRowProps +) => { + const [imageIndex, setImageIndex] = useState(0); + + function hasNewValues() { + return props.newValues && props.newValues.length > 0 && props.onCreateNew; + } + + function setPrev() { + let newIdx = imageIndex - 1; + if (newIdx < 0) { + newIdx = props.images.length - 1; + } + const ret = props.result.cloneWithValue(props.images[newIdx]); + props.onChange(ret as T); + setImageIndex(newIdx); + } + + function setNext() { + let newIdx = imageIndex + 1; + if (newIdx >= props.images.length) { + newIdx = 0; + } + const ret = props.result.cloneWithValue(props.images[newIdx]); + props.onChange(ret as T); + setImageIndex(newIdx); + } + + if (!props.result.scraped && !hasNewValues()) { + return <>; + } + + function renderSelector() { + return ( + props.images.length > 0 && ( +
+ +
+ Select performer image +
+ {imageIndex + 1} of {props.images.length} +
+ +
+ ) + ); + } + + function renderNewValues() { + if (!hasNewValues()) { + return; + } + + const ret = ( + <> + {props.newValues!.map((t, i) => ( + props.onCreateNew!(i)} + > + {t.name} + + + ))} + + ); + + const minCollapseLength = 10; + + if (props.newValues!.length >= minCollapseLength) { + return ( + + {ret} + + ); + } + + return ret; + } + + return ( + + + {props.title} + + + + + + {props.renderOriginalField()} + + + + {props.renderNewField()} + {renderSelector()} + + {renderNewValues()} + + + + + ); +}; + +interface IScrapedImagesRowProps { + title: string; + className?: string; + result: ScrapeResult; + images: string[]; + onChange: (value: ScrapeResult) => void; +} + +export const ScrapedImagesRow: React.FC = (props) => { + return ( + ( + + )} + renderNewField={() => ( + + )} + onChange={props.onChange} + /> + ); +}; + interface IScrapeDialogProps { title: string; existingLabel?: string; diff --git a/ui/v2.5/src/core/StashService.ts b/ui/v2.5/src/core/StashService.ts index 5578c8c33de..88265a13bf1 100644 --- a/ui/v2.5/src/core/StashService.ts +++ b/ui/v2.5/src/core/StashService.ts @@ -1025,6 +1025,37 @@ export const queryScrapePerformerURL = (url: string) => fetchPolicy: "network-only", }); +export const queryScrapePerformerQuery = ( + source: GQL.ScraperSourceInput, + q: string +) => + client.query({ + query: GQL.ScrapeSinglePerformerDocument, + variables: { + source, + input: { + query: q, + }, + }, + // skip: q === "", + fetchPolicy: "network-only", + }); + +export const queryScrapePerformerQueryFragment = ( + source: GQL.ScraperSourceInput, + input: GQL.ScrapedPerformerInput +) => + client.query({ + query: GQL.ScrapeSinglePerformerDocument, + variables: { + source, + input: { + performer_input: input, + }, + }, + fetchPolicy: "network-only", + }); + export const queryScrapeSceneQuery = ( source: GQL.ScraperSourceInput, q: string diff --git a/ui/v2.5/src/locales/en-GB.json b/ui/v2.5/src/locales/en-GB.json index c4d766e749f..0105863b440 100644 --- a/ui/v2.5/src/locales/en-GB.json +++ b/ui/v2.5/src/locales/en-GB.json @@ -322,6 +322,9 @@ "number_of_parallel_task_for_scan_generation_desc": "Set to 0 for auto-detection. Warning running more tasks than is required to achieve 100% cpu utilisation will decrease performance and potentially cause other issues.", "number_of_parallel_task_for_scan_generation_head": "Number of parallel task for scan/generation", "parallel_scan_head": "Parallel Scan/Generation", + "parallel_head": "Parallel", + "number_of_parallel_task_for_image_download_head": "Number of concurrent workers for image download", + "number_of_parallel_task_for_image_download_desc": "Number of workers to download and convert performer/scene scraped images url to base64 concurrently", "preview_generation": "Preview Generation", "python_path": { "description": "Location of python executable. Used for script scrapers and plugins. If blank, python will be resolved from the environment", @@ -786,6 +789,7 @@ "video_previews_tooltip": "Video previews which play when hovering over a scene" }, "scenes_found": "{count} scenes found", + "performers_found": "{count} performers found", "scrape_entity_query": "{entity_type} Scrape Query", "scrape_entity_title": "{entity_type} Scrape Results", "scrape_results_existing": "Existing",