Allow reading FileInfo from a dummy file instead of the file itself

Author: Myself5

README.md

@@ -102,6 +102,7 @@ DisallowRedirects | Disable any mirror trying to do an HTTP redirect
 WeightDistributionRange | Multiplier of the distance to the first mirror to find other possible mirrors in order to distribute the load
 DisableOnMissingFile | Disable a mirror if an advertised file on rsync/ftp appears to be missing on HTTP
 MaxLinkHeaders | Amount of backup mirror locations returned in HTTP headers
+DummyFiles | Allows reading file information from a dummy json file. This Allows saving storage on the host.
 Fallbacks | A list of possible mirrors to use as fallback if a request fails or if the database is unreachable. **These mirrors are not tracked by mirrorbits.** It is assumed they have all the files available in the local repository.
 
 ## Running

config/config.go

@@ -35,6 +35,7 @@ var (
 		CheckInterval:          1,
 		RepositoryScanInterval: 5,
 		MaxLinkHeaders:         10,
+		DummyFiles:             false,
 		Hashes: hashing{
 			SHA1:   true,
 			SHA256: false,
@@ -70,6 +71,7 @@ type Configuration struct {
 	CheckInterval           int        `yaml:"CheckInterval"`
 	RepositoryScanInterval  int        `yaml:"RepositoryScanInterval"`
 	MaxLinkHeaders          int        `yaml:"MaxLinkHeaders"`
+	DummyFiles              bool       `yaml:"DummyFiles"`
 	Hashes                  hashing    `yaml:"Hashes"`
 	DisallowRedirects       bool       `yaml:"DisallowRedirects"`
 	WeightDistributionRange float32    `yaml:"WeightDistributionRange"`

mirrorbits.conf

@@ -21,6 +21,7 @@ ConcurrentSync: 5
 ScanInterval: 30
 CheckInterval: 1
 RepositoryScanInterval: 5
+DummyFiles: false
 Hashes:
     SHA256: On
     SHA1: Off

scan/scan.go

@@ -4,8 +4,10 @@
 package scan
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -62,6 +64,14 @@ type scan struct {
 	count       uint
 }
 
+type DummyFile struct {
+	Size    int64  `json:"Size"`
+	ModTime string `json:"ModTime"`
+	Sha1    string `json:"Sha1"`
+	Sha256  string `json:"Sha256"`
+	Md5     string `json:"Md5"`
+}
+
 // IsScanning returns true is a scan is already in progress for the given mirror
 func IsScanning(conn redis.Conn, identifier string) (bool, error) {
 	return redis.Bool(conn.Do("EXISTS", fmt.Sprintf("SCANNING_%s", identifier)))
@@ -237,11 +247,51 @@ func (s *sourcescanner) walkSource(conn redis.Conn, path string, f os.FileInfo,
 		return nil, nil
 	}
 
+	var dfData DummyFile
+	dummyFile := GetConfig().DummyFiles
+
 	d := new(filedata)
 	d.path = path[len(GetConfig().Repository):]
-	d.size = f.Size()
-	d.modTime = f.ModTime()
 
+	if dummyFile {
+		fi, _ := os.Stat(d.path)
+		if fi.Size() > 1048576 {
+			log.Error("File is bigger than 1MB, falling back to normal")
+			goto skip
+		}
+		raw, err := ioutil.ReadFile(path)
+		if err != nil {
+			log.error(err.Error())
+			log.error("Failed to read file, falling back to normal")
+			dummyFile = false
+			d.size = f.Size()
+			d.modTime = f.ModTime()
+			goto skip
+		}
+
+		var c []DummyFile
+		err = json.Unmarshal(raw, &c)
+		if err != nil {
+			log.Errorf("error decoding json: %v", err)
+			if e, ok := err.(*json.SyntaxError); ok {
+				log.Errorf("syntax error at byte offset %d", e.Offset)
+			}
+			log.Errorf("json response: %q", c)
+			return nil, err
+		}
+
+		dfData = c[0]
+		d.size = dfData.Size
+		d.modTime, err = time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", dfData.ModTime)
+		if err != nil {
+			log.error(err)
+		}
+	} else {
+		d.size = f.Size()
+		d.modTime = f.ModTime()
+	}
+
+skip:
 	// Get the previous file properties
 	properties, err := redis.Strings(conn.Do("HMGET", fmt.Sprintf("FILE_%s", d.path), "size", "modTime", "sha1", "sha256", "md5"))
 	if err != nil && err != redis.ErrNil {
@@ -263,21 +313,27 @@ func (s *sourcescanner) walkSource(conn redis.Conn, path string, f os.FileInfo,
 		(GetConfig().Hashes.MD5 && len(md5) == 0)
 
 	if rehash || size != d.size || !modTime.Equal(d.modTime) {
-		h, err := filesystem.HashFile(GetConfig().Repository + d.path)
-		if err != nil {
-			log.Warningf("%s: hashing failed: %s", d.path, err.Error())
+		if dummyFile {
+			d.sha1 = dfData.Sha1
+			d.sha256 = dfData.Sha256
+			d.md5 = dfData.Md5
 		} else {
-			d.sha1 = h.Sha1
-			d.sha256 = h.Sha256
-			d.md5 = h.Md5
-			if len(d.sha1) > 0 {
-				log.Infof("%s: SHA1 %s", d.path, d.sha1)
-			}
-			if len(d.sha256) > 0 {
-				log.Infof("%s: SHA256 %s", d.path, d.sha256)
-			}
-			if len(d.md5) > 0 {
-				log.Infof("%s: MD5 %s", d.path, d.md5)
+			h, err := filesystem.HashFile(GetConfig().Repository + d.path)
+			if err != nil {
+				log.Warningf("%s: hashing failed: %s", d.path, err.Error())
+			} else {
+				d.sha1 = h.Sha1
+				d.sha256 = h.Sha256
+				d.md5 = h.Md5
+				if len(d.sha1) > 0 {
+					log.Infof("%s: SHA1 %s", d.path, d.sha1)
+				}
+				if len(d.sha256) > 0 {
+					log.Infof("%s: SHA256 %s", d.path, d.sha256)
+				}
+				if len(d.md5) > 0 {
+					log.Infof("%s: MD5 %s", d.path, d.md5)
+				}
 			}
 		}
 	} else {