Compact Index Format

diskindexstore.go

@@ -5,213 +5,153 @@ import (
 	"encoding/hex"
 	"fmt"
 	"os"
-	"path/filepath"
-	"strings"
+	"path"
+
+	"github.com/mandykoh/keva"
 )
 
 const nodeFingerprintFile = "fingerprint"
 const nodeEntriesDir = "entries"
+const thumbnailsDir = "thumbnails"
 
 type DiskIndexStore struct {
-	RootPath string
+	rootPath string
+	nodes    *keva.Store
 }
 
-func (s *DiskIndexStore) AddEntry(entry *IndexEntry, node *IndexNode) error {
-	entriesDir := filepath.Join(node.path, nodeEntriesDir)
-	os.Mkdir(entriesDir, os.ModePerm)
-
-	err := entry.saveToDir(entriesDir)
+func (s *DiskIndexStore) AddEntry(entry *IndexEntry, node *IndexNode, nodeFingerprint Fingerprint) error {
+	err := entry.saveThumbnail(s.pathForThumbnail(entry))
 	if err != nil {
 		return err
 	}
 
 	node.registerEntry(entry)
-	return nil
-}
 
-func (s *DiskIndexStore) GetChild(f Fingerprint, parent *IndexNode) (*IndexNode, error) {
-	path := s.childPathForFingerprint(f, parent.path)
-	return s.getNodeByPath(path)
+	fmt.Printf("AddEntry - Saving [%s] %d %d\n", nodeFingerprint.String(), len(node.childFingerprints), len(node.entries))
+	return s.nodes.Put(nodeFingerprint.String(), node)
 }
 
-func (s *DiskIndexStore) GetOrCreateChild(f Fingerprint, parent *IndexNode) (*IndexNode, error) {
-	fmt.Printf("GetOrCreateChild()\n")
-	childPath := s.childPathForFingerprint(f, parent.path)
-
-	node, err := s.getNodeByPath(childPath)
-	if err != nil {
-		return nil, err
-	}
+func (s *DiskIndexStore) Close() error {
+	return s.nodes.Close()
+}
 
-	if node == nil {
-		fmt.Printf("Creating child\n")
+func (s *DiskIndexStore) GetChild(f Fingerprint, parent *IndexNode) (*IndexNode, error) {
+	var node IndexNode
 
-		node = &IndexNode{
-			path: childPath,
-			childrenByFingerprint: make(map[string]*IndexNodeHandle),
-		}
+	err := s.nodes.Get(f.String(), &node)
+	if err == keva.ErrValueNotFound {
+		return nil, nil
 
-		err := s.saveNode(node, f)
+	} else if err == nil {
+		err = s.loadThumbnails(&node)
 		if err != nil {
 			return nil, err
 		}
 
-		parent.registerChild(node, f)
+	} else {
+		return nil, err
 	}
 
-	return node, nil
+	return &node, nil
 }
 
-func (s *DiskIndexStore) GetRoot() (*IndexNode, error) {
-	return s.getNodeByPath(s.RootPath)
-}
+func (s *DiskIndexStore) GetOrCreateChild(f Fingerprint, parent *IndexNode, parentFingerprint Fingerprint) (*IndexNode, error) {
+	fmt.Printf("GetOrCreateChild() %s\n", f.String())
 
-func (s *DiskIndexStore) RemoveEntries(node *IndexNode) error {
-	entriesDir := filepath.Join(node.path, nodeEntriesDir)
-	err := os.RemoveAll(entriesDir)
-	if err != nil {
-		return err
-	}
+	nodeKey := f.String()
 
-	node.removeEntries()
-	return nil
-}
+	var node IndexNode
+	err := s.nodes.Get(nodeKey, &node)
 
-func (s *DiskIndexStore) childPathForFingerprint(f Fingerprint, parentPath string) string {
-	fingerprintHash := sha256.Sum256(f.Bytes())
-	childDirName := hex.EncodeToString(fingerprintHash[:8])
-	return filepath.Join(parentPath, childDirName)
-}
+	if err == keva.ErrValueNotFound {
+		fmt.Printf("Creating child\n")
 
-func (s *DiskIndexStore) fingerprintForChild(childPath string) (Fingerprint, error) {
-	childFingerprint := Fingerprint{}
-	childFingerprintFile := filepath.Join(childPath, nodeFingerprintFile)
+		node = IndexNode{
+			childFingerprintsByString: make(map[string]*Fingerprint),
+		}
 
-	file, err := os.Open(childFingerprintFile)
-	if err != nil {
-		return childFingerprint, err
-	}
-	defer file.Close()
+		fmt.Printf("GetOrCreateChild - Saving [%s] %d %d\n", nodeKey, len(node.childFingerprints), len(node.entries))
+		err = s.nodes.Put(nodeKey, &node)
+		if err != nil {
+			return nil, err
+		}
 
-	fileInfo, err := file.Stat()
-	if err != nil {
-		return childFingerprint, err
-	}
+		parent.registerChild(f)
+		fmt.Printf("GetOrCreateChild - Parent - Saving [%s] %d %d\n", parentFingerprint.String(), len(parent.childFingerprints), len(parent.entries))
+		err = s.nodes.Put(parentFingerprint.String(), parent)
+		if err != nil {
+			return nil, err
+		}
 
-	fingerprintBytes := make([]byte, fileInfo.Size(), fileInfo.Size())
-	_, err = file.Read(fingerprintBytes)
-	if err != nil {
-		return childFingerprint, err
-	}
+	} else if err == nil {
+		err = s.loadThumbnails(&node)
+		if err != nil {
+			return nil, err
+		}
 
-	childFingerprint.UnmarshalBytes(fingerprintBytes)
+	} else {
+		return nil, err
+	}
 
-	return childFingerprint, nil
+	return &node, nil
 }
 
-func (s *DiskIndexStore) getNodeByPath(path string) (*IndexNode, error) {
+func (s *DiskIndexStore) GetRoot() (*IndexNode, error) {
+	var rootKey = Fingerprint{}.String()
 
-	_, err := os.Stat(path)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
+	var root IndexNode
+	err := s.nodes.Get(rootKey, &root)
+
+	if err == keva.ErrValueNotFound {
+		fmt.Printf("Root node not found - creating it\n")
+		root = IndexNode{
+			childFingerprintsByString: make(map[string]*Fingerprint),
 		}
-		return nil, err
-	}
 
-	node := &IndexNode{
-		path: path,
-		childrenByFingerprint: make(map[string]*IndexNodeHandle),
-	}
+	} else if err == nil {
+		fmt.Printf("Found root node with %d children and %d entries\n", len(root.childFingerprints), len(root.entries))
 
-	err = s.loadAllChildren(node)
-	if err != nil {
-		return nil, err
-	}
+		err = s.loadThumbnails(&root)
+		if err != nil {
+			return nil, err
+		}
 
-	err = s.loadAllEntries(node)
-	if err != nil {
+	} else {
 		return nil, err
 	}
 
-	return node, nil
+	return &root, nil
 }
 
-func (s *DiskIndexStore) loadAllChildren(n *IndexNode) error {
-	dir, err := os.Open(n.path)
-	if err != nil {
-		return err
-	}
-	defer dir.Close()
-
-	for fileInfos, err := dir.Readdir(1); err == nil && len(fileInfos) > 0; fileInfos, err = dir.Readdir(1) {
-		for _, info := range fileInfos {
-			if info.IsDir() && info.Name() != nodeEntriesDir {
-				child, err := s.loadChild(n, info.Name())
-				if err != nil {
-					return err
-				}
-
-				n.registerChildByHandle(child)
-			}
-		}
-	}
-
-	return nil
+func (s *DiskIndexStore) RemoveEntries(node *IndexNode, nodeFingerprint Fingerprint) error {
+	node.removeEntries()
+	fmt.Printf("RemoveEntries - Saving [%s] %d %d\n", nodeFingerprint.String(), len(node.childFingerprints), len(node.entries))
+	return s.nodes.Put(nodeFingerprint.String(), node)
 }
 
-func (s *DiskIndexStore) loadAllEntries(n *IndexNode) error {
-	entriesDir := filepath.Join(n.path, nodeEntriesDir)
-
-	dir, err := os.Open(entriesDir)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	defer dir.Close()
-
-	for fileInfos, err := dir.Readdir(1); err == nil && len(fileInfos) > 0; fileInfos, err = dir.Readdir(1) {
-		for _, fileInfo := range fileInfos {
-			if strings.HasSuffix(fileInfo.Name(), ".entry") {
-				entry, err := NewIndexEntryFromFile(filepath.Join(entriesDir, fileInfo.Name()))
-				if err != nil {
-					return err
-				}
-
-				n.registerEntry(entry)
-			}
-		}
-	}
-
-	return nil
+func (s *DiskIndexStore) loadThumbnails(n *IndexNode) error {
+	return n.withEachEntry(func(entry *IndexEntry) error {
+		return entry.loadThumbnail(s.pathForThumbnail(entry))
+	})
 }
 
-func (s *DiskIndexStore) loadChild(n *IndexNode, childDirName string) (*IndexNodeHandle, error) {
-	childPath := filepath.Join(n.path, childDirName)
-	childFingerprint, err := s.fingerprintForChild(childPath)
-	if err != nil {
-		return nil, err
-	}
-
-	return &IndexNodeHandle{Path: childPath, Fingerprint: childFingerprint}, nil
+func (s *DiskIndexStore) pathForThumbnail(entry *IndexEntry) string {
+	thumbnailHash := sha256.Sum256(entry.MaxFingerprint.Bytes())
+	thumbnailHex := hex.EncodeToString(thumbnailHash[:])
+	return path.Join(s.rootPath, thumbnailsDir, thumbnailHex[0:2], thumbnailHex[2:4], thumbnailHex[4:])
 }
 
-func (s *DiskIndexStore) saveNode(n *IndexNode, f Fingerprint) error {
-	fmt.Printf("Saving node %s\n", n.path)
+func NewDiskIndexStore(rootPath string) (*DiskIndexStore, error) {
+	thumbnailsDir := path.Join(rootPath, thumbnailsDir)
+	os.MkdirAll(thumbnailsDir, os.FileMode(0700))
 
-	os.Mkdir(n.path, os.FileMode(0700))
-
-	// Save the actual (non-truncated) fingerprint
-	fingerprintFile := filepath.Join(n.path, nodeFingerprintFile)
-	file, err := os.Create(fingerprintFile)
+	nodeStore, err := keva.NewStore(path.Join(rootPath, "nodes"))
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	defer file.Close()
-
-	_, err = file.Write(f.Bytes())
-	return err
+	return &DiskIndexStore{
+		rootPath: rootPath,
+		nodes:    nodeStore,
+	}, nil
 }