From 5923917e6cf53e43d5eaf85ec3954c165600c15e Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Sat, 25 Apr 2020 09:32:55 +1000 Subject: [PATCH] Clean missing galleries (#489) * Clean missing galleries * Refactor matchFile --- pkg/manager/exclude_files.go | 13 +--- pkg/manager/manager_tasks.go | 39 +++++++++-- pkg/manager/task_clean.go | 65 ++++++++++++++----- pkg/models/querybuilder_gallery.go | 5 ++ .../SettingsTasksPanel/SettingsTasksPanel.tsx | 4 +- 5 files changed, 92 insertions(+), 34 deletions(-) diff --git a/pkg/manager/exclude_files.go b/pkg/manager/exclude_files.go index 818cb25f14e..6d5a28f9f56 100644 --- a/pkg/manager/exclude_files.go +++ b/pkg/manager/exclude_files.go @@ -1,9 +1,10 @@ package manager import ( - "github.com/stashapp/stash/pkg/logger" "regexp" "strings" + + "github.com/stashapp/stash/pkg/logger" ) func excludeFiles(files []string, patterns []string) ([]string, int) { @@ -37,21 +38,13 @@ func excludeFiles(files []string, patterns []string) ([]string, int) { } func matchFile(file string, patterns []string) bool { - if patterns == nil { - logger.Infof("No exclude patterns in config.") - - } else { + if patterns != nil { fileRegexps := generateRegexps(patterns) - if len(fileRegexps) == 0 { - return false - } - for _, regPattern := range fileRegexps { if regPattern.MatchString(strings.ToLower(file)) { return true } - } } diff --git a/pkg/manager/manager_tasks.go b/pkg/manager/manager_tasks.go index 34286eda022..6c77b360fee 100644 --- a/pkg/manager/manager_tasks.go +++ b/pkg/manager/manager_tasks.go @@ -1,15 +1,16 @@ package manager import ( + "path/filepath" + "strconv" + "sync" + "time" + "github.com/bmatcuk/doublestar" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" - "path/filepath" - "strconv" - "sync" - "time" ) var extensionsToScan = []string{"zip", "m4v", "mp4", "mov", "wmv", "avi", "mpg", "mpeg", "rmvb", "rm", "flv", "asf", "mkv", "webm"} @@ -474,6 +475,7 @@ func (s *singleton) Clean() { s.Status.indefiniteProgress() qb := models.NewSceneQueryBuilder() + gqb := models.NewGalleryQueryBuilder() go func() { defer s.returnToIdleState() @@ -484,6 +486,12 @@ func (s *singleton) Clean() { return } + galleries, err := gqb.All() + if err != nil { + logger.Errorf("failed to fetch list of galleries for cleaning") + return + } + if s.Status.stopping { logger.Info("Stopping due to user request") return @@ -491,7 +499,7 @@ func (s *singleton) Clean() { var wg sync.WaitGroup s.Status.Progress = 0 - total := len(scenes) + total := len(scenes) + len(galleries) for i, scene := range scenes { s.Status.setProgress(i, total) if s.Status.stopping { @@ -506,7 +514,26 @@ func (s *singleton) Clean() { wg.Add(1) - task := CleanTask{Scene: *scene} + task := CleanTask{Scene: scene} + go task.Start(&wg) + wg.Wait() + } + + for i, gallery := range galleries { + s.Status.setProgress(len(scenes)+i, total) + if s.Status.stopping { + logger.Info("Stopping due to user request") + return + } + + if gallery == nil { + logger.Errorf("nil gallery, skipping Clean") + continue + } + + wg.Add(1) + + task := CleanTask{Gallery: gallery} go task.Start(&wg) wg.Wait() } diff --git a/pkg/manager/task_clean.go b/pkg/manager/task_clean.go index 580d3631a86..8b05c0e7734 100644 --- a/pkg/manager/task_clean.go +++ b/pkg/manager/task_clean.go @@ -2,33 +2,47 @@ package manager import ( "context" - "github.com/stashapp/stash/pkg/database" - "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" - "github.com/stashapp/stash/pkg/models" "os" "path/filepath" "strings" "sync" + + "github.com/stashapp/stash/pkg/database" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/manager/config" + "github.com/stashapp/stash/pkg/models" ) type CleanTask struct { - Scene models.Scene + Scene *models.Scene + Gallery *models.Gallery } func (t *CleanTask) Start(wg *sync.WaitGroup) { defer wg.Done() - if t.fileExists(t.Scene.Path) && t.pathInStash() { - logger.Debugf("File Found: %s", t.Scene.Path) - if matchFile(t.Scene.Path, config.GetExcludes()) { - logger.Infof("File matched regex. Cleaning: \"%s\"", t.Scene.Path) - t.deleteScene(t.Scene.ID) + if t.Scene != nil && t.shouldClean(t.Scene.Path) { + t.deleteScene(t.Scene.ID) + } + + if t.Gallery != nil && t.shouldClean(t.Gallery.Path) { + t.deleteGallery(t.Gallery.ID) + } +} + +func (t *CleanTask) shouldClean(path string) bool { + if t.fileExists(path) && t.pathInStash(path) { + logger.Debugf("File Found: %s", path) + if matchFile(path, config.GetExcludes()) { + logger.Infof("File matched regex. Cleaning: \"%s\"", path) + return true } } else { - logger.Infof("File not found. Cleaning: \"%s\"", t.Scene.Path) - t.deleteScene(t.Scene.ID) + logger.Infof("File not found. Cleaning: \"%s\"", path) + return true } + + return false } func (t *CleanTask) deleteScene(sceneID int) { @@ -53,6 +67,25 @@ func (t *CleanTask) deleteScene(sceneID int) { DeleteGeneratedSceneFiles(scene) } +func (t *CleanTask) deleteGallery(galleryID int) { + ctx := context.TODO() + qb := models.NewGalleryQueryBuilder() + tx := database.DB.MustBeginTx(ctx, nil) + + err := qb.Destroy(galleryID, tx) + + if err != nil { + logger.Infof("Error deleting gallery from database: %s", err.Error()) + tx.Rollback() + return + } + + if err := tx.Commit(); err != nil { + logger.Infof("Error deleting gallery from database: %s", err.Error()) + return + } +} + func (t *CleanTask) fileExists(filename string) bool { info, err := os.Stat(filename) if os.IsNotExist(err) { @@ -61,19 +94,19 @@ func (t *CleanTask) fileExists(filename string) bool { return !info.IsDir() } -func (t *CleanTask) pathInStash() bool { +func (t *CleanTask) pathInStash(pathToCheck string) bool { for _, path := range config.GetStashPaths() { - rel, error := filepath.Rel(path, filepath.Dir(t.Scene.Path)) + rel, error := filepath.Rel(path, filepath.Dir(pathToCheck)) if error == nil { if !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - logger.Debugf("File %s belongs to stash path %s", t.Scene.Path, path) + logger.Debugf("File %s belongs to stash path %s", pathToCheck, path) return true } } } - logger.Debugf("File %s is out from stash path", t.Scene.Path) + logger.Debugf("File %s is out from stash path", pathToCheck) return false } diff --git a/pkg/models/querybuilder_gallery.go b/pkg/models/querybuilder_gallery.go index 73c4ff854a9..b27af9b5486 100644 --- a/pkg/models/querybuilder_gallery.go +++ b/pkg/models/querybuilder_gallery.go @@ -3,6 +3,7 @@ package models import ( "database/sql" "path/filepath" + "strconv" "github.com/jmoiron/sqlx" "github.com/stashapp/stash/pkg/database" @@ -51,6 +52,10 @@ func (qb *GalleryQueryBuilder) Update(updatedGallery Gallery, tx *sqlx.Tx) (*Gal return &updatedGallery, nil } +func (qb *GalleryQueryBuilder) Destroy(id int, tx *sqlx.Tx) error { + return executeDeleteQuery("galleries", strconv.Itoa(id), tx) +} + type GalleryNullSceneID struct { SceneID sql.NullInt64 } diff --git a/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx b/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx index 5b8a3992e19..7302e225f5d 100644 --- a/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx @@ -105,8 +105,8 @@ export const SettingsTasksPanel: React.FC = () => { cancel={{ onClick: () => setIsCleanAlertOpen(false) }} >

- Are you sure you want to Clean? This will delete db information and - generated content for all scenes that are no longer found in the + Are you sure you want to Clean? This will delete database information and + generated content for all scenes and galleries that are no longer found in the filesystem.