Bug fixing added moving torrents after download, getting ready for al…

…pha release

.gitignore

@@ -1,4 +1,6 @@
 downloads/
+downloading/
+uploadedTorrents/
 storage.db.lock
 storage.db
 .torrent.bolt.db.lock
@@ -10,4 +12,5 @@ output.json
 .idea/workspace.xml
 .idea/vcs.xml
 *.torrent
-boltbrowser.win64.exe
+boltbrowser.win64.exe
+logs/server.log

config.toml

@@ -3,16 +3,16 @@
 
     ServerPort = ":8000" #leave format as is it expects a string with colon
     ServerAddr = "" #blank will bind to default IP address, usually fine to leave be
-    LogLevel = "Info" # Options = Debug, Info, Warn, Error, Fatal, Panic
-    LogOutput = "stdout" #Options = file, stdout #file will print it to logs/server.log
+    LogLevel = "Debug" # Options = Debug, Info, Warn, Error, Fatal, Panic
+    LogOutput = "file" #Options = file, stdout #file will print it to logs/server.log
 
     SeedRatioStop = 1.50 #automatically stops the torrent after it reaches this seeding ratio
-    DefaultMoveFolder = "downloads" #default path that a finished torrent is symlinked to after completion.  Torrents added via RSS will default here
+    DefaultMoveFolder = "downloaded" #default path that a finished torrent is symlinked to after completion.  Torrents added via RSS will default here
 
 
-    [notifications]
+[notifications]
 
-        PushBulletToken = "o.QW6G7F6FUOKXCUKmw948fBceCUn0msFi" #add your pushbullet api token here to notify of torrent completion to pushbullet
+    PushBulletToken = "o.QW6G7F6FUOKXCUKmw948fBceCUn0msFi" #add your pushbullet api token here to notify of torrent completion to pushbullet
 
 
 [EncryptionPolicy]
@@ -23,9 +23,12 @@
 
 
 [torrentClientConfig]
-    DownloadDir = "downloads"    #the full OR relative path where the torrent server stores in-progress torrents
+    DownloadDir = "downloading"    #the full OR relative path where the torrent server stores in-progress torrents
 
     Seed = true #boolean #seed after download
+
+    # Never send chunks to peers.
+    NoUpload = false #boolean
 
     #The address to listen for new uTP and TCP bittorrent protocolconnections. DHT shares a UDP socket with uTP unless configured otherwise.
     ListenAddr = "" #Leave Blank for default, syntax "HOST:PORT"
@@ -38,9 +41,6 @@
     # Don't create a DHT.
     NoDHT = false #boolean
 
-    # Never send chunks to peers.
-    NoUpload = false #boolean
-
     # Events are data bytes sent in pieces. The burst must be large enough to fit a whole chunk.
     UploadRateLimiter = "" #*rate.Limiter
 

engine/cronJobs.go

@@ -22,6 +22,7 @@ func InitializeCronEngine() *cron.Cron {
 //RefreshRSSCron refreshes all of the RSS feeds on an hourly basis
 func RefreshRSSCron(c *cron.Cron, db *storm.DB, tclient *torrent.Client, torrentLocalStorage Storage.TorrentLocal, dataDir string) {
 	c.AddFunc("@hourly", func() {
+		torrentHashHistory := Storage.FetchHashHistory(db)
 		RSSFeedStore := Storage.FetchRSSFeeds(db)
 		singleRSSTorrent := Storage.SingleRSSTorrent{}
 		newFeedStore := Storage.RSSFeedStore{ID: RSSFeedStore.ID} //creating a new feed store just using old one to parse for new torrents
@@ -36,6 +37,12 @@ func RefreshRSSCron(c *cron.Cron, db *storm.DB, tclient *torrent.Client, torrent
 				singleRSSTorrent.Link = RSSTorrent.Link
 				singleRSSTorrent.Title = RSSTorrent.Title
 				singleRSSTorrent.PubDate = RSSTorrent.Published
+				for _, hash := range torrentHashHistory.HashList {
+					linkHash := singleRSSTorrent.Link[20:60] //cutting the infohash out of the link
+					if linkHash == hash {
+						Logger.WithFields(logrus.Fields{"Torrent": RSSTorrent.Title}).Warn("Torrent already added for this RSS item, skipping torrent")
+					}
+				}
 				clientTorrent, err := tclient.AddMagnet(RSSTorrent.Link)
 				if err != nil {
 					Logger.WithFields(logrus.Fields{"err": err, "Torrent": RSSTorrent.Title}).Warn("Unable to add torrent to torrent client!")

engine/doneTorrentActions.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -11,44 +12,77 @@ import (
 	Storage "github.com/deranjer/goTorrent/storage"
 	pushbullet "github.com/mitsuse/pushbullet-go"
 	"github.com/mitsuse/pushbullet-go/requests"
+	folderCopy "github.com/otiai10/copy"
 	"github.com/sirupsen/logrus"
 )
 
 //MoveAndLeaveSymlink takes the file from the default download dir and moves it to the user specified directory and then leaves a symlink behind.
 func MoveAndLeaveSymlink(config FullClientSettings, singleTorrent *torrent.Torrent, db *storm.DB) {
+	Logger.WithFields(logrus.Fields{"Torrent Name": singleTorrent.Name()}).Error("Move and Create symlink started for torrent")
 	tStorage := Storage.FetchTorrentFromStorage(db, singleTorrent.InfoHash().String())
 	oldFilePath := filepath.Join(config.TorrentConfig.DataDir, singleTorrent.Name())
 	newFilePath := filepath.Join(tStorage.StoragePath, singleTorrent.Name())
+	_, err := os.Stat(tStorage.StoragePath)
+	if os.IsNotExist(err) {
+		err := os.MkdirAll(tStorage.StoragePath, 0644)
+		if err != nil {
+			Logger.WithFields(logrus.Fields{"New File Path": newFilePath, "error": err}).Error("Cannot create new directory")
+		}
+	}
+	oldFileInfo, err := os.Stat(oldFilePath)
+	if err != nil {
+		Logger.WithFields(logrus.Fields{"Old File info": oldFileInfo, "error": err}).Error("Cannot find the old file to copy/symlink!")
+		return
+	}
+
 	if oldFilePath != newFilePath {
 		if runtime.GOOS == "windows" { //TODO the windows symlink is broken on windows 10 creator edition, so doing a copy for now until Go 1.11
+			if oldFileInfo.IsDir() {
+				os.Mkdir(newFilePath, 0644)
+				folderCopy.Copy(oldFilePath, newFilePath) //copy the folder to the new location
+				notifyUser(tStorage, config, singleTorrent)
+				return
+			}
+
 			srcFile, err := os.Open(oldFilePath)
 			if err != nil {
 				Logger.WithFields(logrus.Fields{"Old File Path": oldFilePath, "error": err}).Error("Windows: Cannot open old file for copy")
 				return
 			}
-			destFile, err := os.Create(newFilePath) // creating new file to copy old one to
+			defer srcFile.Close()
+			destFile, err := os.Create(newFilePath)
 			if err != nil {
 				Logger.WithFields(logrus.Fields{"New File Path": newFilePath, "error": err}).Error("Windows: Cannot open new file for copying into")
 				return
 			}
-			_, err = io.Copy(srcFile, destFile)
+			defer destFile.Close()
+			bytesWritten, err := io.Copy(destFile, srcFile)
 			if err != nil {
 				Logger.WithFields(logrus.Fields{"Old File Path": oldFilePath, "New File Path": newFilePath, "error": err}).Error("Windows: Cannot copy old file into new")
 				return
 			}
-			Logger.WithFields(logrus.Fields{"Old File Path": oldFilePath, "New File Path": newFilePath}).Info("Windows Torrent Copy Completed")
+			err = destFile.Sync()
+			if err != nil {
+				Logger.WithFields(logrus.Fields{"Old File Path": oldFilePath, "New File Path": newFilePath, "error": err}).Error("Windows: Error syncing new file to disk")
+			}
+			Logger.WithFields(logrus.Fields{"Old File Path": oldFilePath, "New File Path": newFilePath, "bytesWritten": bytesWritten}).Info("Windows Torrent Copy Completed")
+			notifyUser(tStorage, config, singleTorrent)
 		} else {
 			err := os.Symlink(oldFilePath, newFilePath) //For all other OS's create a symlink
 			if err != nil {
 				Logger.WithFields(logrus.Fields{"Old File Path": oldFilePath, "New File Path": newFilePath, "error": err}).Error("Error creating symlink")
 				return
 			}
+			notifyUser(tStorage, config, singleTorrent)
 			Logger.WithFields(logrus.Fields{"Old File Path": oldFilePath, "New File Path": newFilePath}).Info("Moving completed torrent")
 		}
 	}
 	tStorage.TorrentMoved = true
 	Storage.AddTorrentLocalStorage(db, tStorage) //Updating the fact that we moved the torrent
+}
 
+func notifyUser(tStorage Storage.TorrentLocal, config FullClientSettings, singleTorrent *torrent.Torrent) {
+	fmt.Println("Pushbullet token", config.PushBulletToken)
 	if config.PushBulletToken != "" {
 		pb := pushbullet.New(config.PushBulletToken)
 		n := requests.NewNote()
@@ -60,5 +94,4 @@ func MoveAndLeaveSymlink(config FullClientSettings, singleTorrent *torrent.Torre
 		}
 		Logger.WithFields(logrus.Fields{"Torrent": singleTorrent.Name(), "New File Path": tStorage.StoragePath}).Info("Pushbullet note sent")
 	}
-
 }

engine/engine.go

@@ -80,7 +80,7 @@ func timeOutInfo(clientTorrent *torrent.Torrent, seconds time.Duration) (deleted
 
 }
 
-func readTorrentFileFromDB(element *Storage.TorrentLocal, singleTorrent *torrent.Torrent, tclient *torrent.Client, db *storm.DB) {
+func readTorrentFileFromDB(element *Storage.TorrentLocal, tclient *torrent.Client, db *storm.DB) (singleTorrent *torrent.Torrent) {
 	tempFile, err := ioutil.TempFile("", "TorrentFileTemp")
 	if err != nil {
 		Logger.WithFields(logrus.Fields{"tempfile": tempFile, "err": err}).Error("Unable to create tempfile")
@@ -92,13 +92,16 @@ func readTorrentFileFromDB(element *Storage.TorrentLocal, singleTorrent *torrent
 	if err := tempFile.Close(); err != nil { //close the tempfile so that we can add it back into the torrent client
 		Logger.WithFields(logrus.Fields{"tempfile": tempFile, "err": err}).Error("Unable to close tempfile")
 	}
-	singleTorrent, _ = tclient.AddTorrentFromFile(tempFile.Name())
-	if _, err := os.Stat(element.TorrentFileName); err == nil { //if we CAN find the torrent, add it
-		singleTorrent, _ = tclient.AddTorrentFromFile(element.TorrentFileName)
-	} else { //if we cant find the torrent delete it
-		Storage.DelTorrentLocalStorage(db, element.Hash)
-		Logger.WithFields(logrus.Fields{"tempfile": tempFile, "err": err}).Error("Unable to find Torrent, deleting..")
+	_, err = os.Stat(element.TorrentFileName) //if we CAN find the torrent, add it
+	if err != nil {
+		Logger.WithFields(logrus.Fields{"tempfile": tempFile, "err": err}).Error("Unable to find file")
 	}
+	singleTorrent, err = tclient.AddTorrentFromFile(element.TorrentFileName)
+	if err != nil {
+		Logger.WithFields(logrus.Fields{"tempfile": element.TorrentFileName, "err": err}).Error("Unable to add Torrent from file!")
+
+	}
+	return singleTorrent
 }
 
 //StartTorrent creates the storage.db entry and starts A NEW TORRENT and adds to the running torrent array
@@ -147,23 +150,20 @@ func StartTorrent(clientTorrent *torrent.Torrent, torrentLocalStorage Storage.To
 func CreateRunningTorrentArray(tclient *torrent.Client, TorrentLocalArray []*Storage.TorrentLocal, PreviousTorrentArray []ClientDB, config FullClientSettings, db *storm.DB) (RunningTorrentArray []ClientDB) {
 
 	for _, singleTorrentFromStorage := range TorrentLocalArray {
-
 		var singleTorrent *torrent.Torrent
 		var TempHash metainfo.Hash
 
 		fullClientDB := new(ClientDB)
 		//singleTorrentStorageInfo := Storage.FetchTorrentFromStorage(db, TempHash.String())  //pulling the single torrent info from storage ()
 
 		if singleTorrentFromStorage.TorrentType == "file" { //if it is a file pull it from the uploaded torrent folder
-			readTorrentFileFromDB(singleTorrentFromStorage, singleTorrent, tclient, db)
+			singleTorrent = readTorrentFileFromDB(singleTorrentFromStorage, tclient, db)
 			fullClientDB.SourceType = "Torrent File"
-			continue
 		} else {
 			singleTorrentFromStorageMagnet := "magnet:?xt=urn:btih:" + singleTorrentFromStorage.Hash //For magnet links just need to prepend the magnet part to the hash to readd
 			singleTorrent, _ = tclient.AddMagnet(singleTorrentFromStorageMagnet)
 			fullClientDB.SourceType = "Magnet Link"
 		}
-
 		if len(singleTorrentFromStorage.InfoBytes) == 0 { //TODO.. kind of a fringe scenario.. not sure if needed since the db should always have the infobytes
 			timeOut := timeOutInfo(singleTorrent, 45)
 			if timeOut == true { // if we did timeout then drop the torrent from the boltdb database
@@ -173,8 +173,12 @@ func CreateRunningTorrentArray(tclient *torrent.Client, TorrentLocalArray []*Sto
 			singleTorrentFromStorage.InfoBytes = singleTorrent.Metainfo().InfoBytes
 		}
 
+		err := singleTorrent.SetInfoBytes(singleTorrentFromStorage.InfoBytes) //setting the infobytes back into the torrent
+		if err != nil {
+			Logger.WithFields(logrus.Fields{"torrentFile": singleTorrent.Name(), "error": err}).Error("Unable to add infobytes to the torrent!")
+		}
+		//Logger.WithFields(logrus.Fields{"singleTorrent": singleTorrentFromStorage.TorrentName}).Info("Generating infohash")
 		TempHash = singleTorrent.InfoHash()
-		singleTorrent.SetInfoBytes(singleTorrentFromStorage.InfoBytes) //setting the infobytes back into the torrent
 
 		if (singleTorrent.BytesCompleted() == singleTorrent.Length()) && (singleTorrentFromStorage.TorrentMoved == false) { //if we are done downloading and havent moved torrent yet
 			MoveAndLeaveSymlink(config, singleTorrent, db)
@@ -190,7 +194,7 @@ func CreateRunningTorrentArray(tclient *torrent.Client, TorrentLocalArray []*Sto
 
 		downloadedSizeHumanized := HumanizeBytes(float32(singleTorrent.BytesCompleted())) //convert size to GB if needed
 		totalSizeHumanized := HumanizeBytes(float32(singleTorrent.Length()))
-
+		//Logger.WithFields(logrus.Fields{"singleTorrent": singleTorrentFromStorage.TorrentName}).Info("Generated infohash")
 		//grabbed from torrent client
 		fullClientDB.DownloadedSize = downloadedSizeHumanized
 		fullClientDB.Size = totalSizeHumanized
@@ -206,7 +210,7 @@ func CreateRunningTorrentArray(tclient *torrent.Client, TorrentLocalArray []*Sto
 		fullClientDB.DateAdded = singleTorrentFromStorage.DateAdded
 		fullClientDB.BytesCompleted = singleTorrent.BytesCompleted()
 		fullClientDB.NumberofFiles = len(singleTorrent.Files())
-		CalculateTorrentETA(singleTorrent, fullClientDB)
+
 		//ranging over the previous torrent array to calculate the speed for each torrent
 		if len(PreviousTorrentArray) > 0 { //if we actually have  a previous array
 			for _, previousElement := range PreviousTorrentArray {
@@ -217,6 +221,7 @@ func CreateRunningTorrentArray(tclient *torrent.Client, TorrentLocalArray []*Sto
 				}
 			}
 		}
+		CalculateTorrentETA(singleTorrent, fullClientDB) //needs to be here since we need the speed calcuated before we can estimate the eta.
 
 		fullClientDB.TotalUploadedSize = HumanizeBytes(float32(fullClientDB.TotalUploadedBytes))
 		fullClientDB.UploadRatio = CalculateUploadRatio(singleTorrent, fullClientDB) //calculate the upload ratio

engine/settings.go

@@ -67,7 +67,7 @@ func FullClientSettingsNew() FullClientSettings {
 	httpAddrPort := viper.GetString("serverConfig.ServerPort")
 	seedRatioStop := viper.GetFloat64("serverConfig.SeedRatioStop")
 	httpAddr = httpAddrIP + httpAddrPort
-	pushBulletToken := viper.GetString("serverConfig.notifications.PushBulletToken")
+	pushBulletToken := viper.GetString("notifications.PushBulletToken")
 	defaultMoveFolder := viper.GetString("serverConfig.DefaultMoveFolder")
 
 	dataDir := viper.GetString("torrentClientConfig.DownloadDir")
@@ -117,9 +117,6 @@ func FullClientSettingsNew() FullClientSettings {
 	downloadRateLimiter := new(rate.Limiter)
 	viper.UnmarshalKey("DownloadRateLimiter", &downloadRateLimiter)
 
-	rreferNoEncryption := viper.GetBool("EncryptionPolicy.PreferNoEncryption")
-	fmt.Println("Encryption", rreferNoEncryption)
-
 	encryptionPolicy := torrent.EncryptionPolicy{
 		DisableEncryption:  viper.GetBool("EncryptionPolicy.DisableEncryption"),
 		ForceEncryption:    viper.GetBool("EncryptionPolicy.ForceEncryption"),