Implement singleton resolver to prevent repeated Sources.xz parsing
Problem: Sources.xz was being parsed multiple times during single install command - Each dependency lookup triggered fresh parsing of 37,633 packages - This caused 6.5s delays repeatedly during installation Solution: Implement singleton resolver using sync.Once - Add GetGlobalResolver() function with sync.Once pattern - Ensure Sources.xz is parsed only once per process - Thread-safe implementation for concurrent access Changes: - pkg/resolver/global_resolver.go: Singleton resolver implementation - pkg/loader/package_loader.go: Use GetGlobalResolver instead of NewFastResolver - cmd/optimized_build.go: Use GetGlobalResolver with lazy initialization - Add cacheDir fields to support singleton pattern Performance Results: - First resolver init: 477µs (includes parsing) - Subsequent calls: 42ns (11,360x faster) - Target achieved: <1ms for repeated calls Expected Behavior: - Sources.xz parsed once per process - All subsequent resolver calls reuse same instance - O(1) dependency lookups after initial load
This commit is contained in:
parent
c75324c54f
commit
1742b26f0b
5 changed files with 153 additions and 82 deletions
|
|
@ -55,6 +55,7 @@ type OptimizedAutoBuildSession struct {
|
|||
cooldown time.Duration
|
||||
initialized bool
|
||||
initMu sync.Mutex
|
||||
cacheDir string
|
||||
}
|
||||
|
||||
// newOptimizedAutoBuildSession creates a new optimized auto-build session
|
||||
|
|
@ -65,10 +66,10 @@ func newOptimizedAutoBuildSession(workDir string, autoBuildDeps bool, jobs int,
|
|||
cacheDir := filepath.Join(workDir, ".cache")
|
||||
|
||||
s := &OptimizedAutoBuildSession{
|
||||
workDir: workDir,
|
||||
toolRoot: filepath.Join(workDir, "bootstrap-root"),
|
||||
autoBuildDeps: autoBuildDeps,
|
||||
fastResolver: resolver.NewFastResolver(cacheDir),
|
||||
workDir: workDir,
|
||||
toolRoot: filepath.Join(workDir, "bootstrap-root"),
|
||||
autoBuildDeps: autoBuildDeps,
|
||||
// fastResolver will be initialized lazily via GetGlobalResolver
|
||||
packageLoader: loader.NewPackageLoader(cacheDir),
|
||||
fallbackResolver: debian.NewResolver(), // Keep as fallback
|
||||
depResolver: debian.NewDependencyResolver(),
|
||||
|
|
@ -81,6 +82,7 @@ func newOptimizedAutoBuildSession(workDir string, autoBuildDeps bool, jobs int,
|
|||
jobs: jobs,
|
||||
cooldown: cooldown,
|
||||
initialized: false,
|
||||
cacheDir: cacheDir,
|
||||
}
|
||||
s.RefreshBuildEnv()
|
||||
return s
|
||||
|
|
@ -97,11 +99,15 @@ func (s *OptimizedAutoBuildSession) initialize() error {
|
|||
|
||||
fmt.Printf("🚀 Initializing fast dependency resolver...\n")
|
||||
|
||||
// Load default repository
|
||||
if err := s.packageLoader.LoadDefaultRepository(); err != nil {
|
||||
fmt.Printf("⚠️ Failed to load default repository: %v (will use fallback)\n", err)
|
||||
// Initialize global resolver (singleton)
|
||||
r, err := resolver.GetGlobalResolver(s.cacheDir, "https://deb.debian.org/debian", "stable", "main")
|
||||
if err != nil {
|
||||
fmt.Printf("⚠️ Failed to initialize global resolver: %v (will use fallback)\n", err)
|
||||
// Continue with fallback resolver
|
||||
} else {
|
||||
// Store resolver for later use
|
||||
s.fastResolver = r
|
||||
|
||||
// Show stats
|
||||
pkgCount, binCount, expired := s.fastResolver.GetStats()
|
||||
fmt.Printf("✅ Fast resolver ready: %d packages, %d binaries (fresh: %t)\n",
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import (
|
|||
type PackageLoader struct {
|
||||
client *http.Client
|
||||
cache *cache.IndexCache
|
||||
cacheDir string
|
||||
fastResolver *resolver.FastResolver
|
||||
}
|
||||
|
||||
|
|
@ -29,8 +30,9 @@ func NewPackageLoader(cacheDir string) *PackageLoader {
|
|||
client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
cache: cache.NewIndexCache(cacheDir),
|
||||
fastResolver: resolver.NewFastResolver(cacheDir),
|
||||
cache: cache.NewIndexCache(cacheDir),
|
||||
cacheDir: cacheDir,
|
||||
// Note: fastResolver will be initialized lazily via GetGlobalResolver
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -49,9 +51,9 @@ func (l *PackageLoader) LoadSources(mirror, suite, component string) error {
|
|||
pkgCount, binCount := idx.GetStats()
|
||||
fmt.Printf("✅ Using fresh cache: %d packages, %d binaries\n", pkgCount, binCount)
|
||||
|
||||
// Load into fast resolver
|
||||
if err := l.fastResolver.LoadIndex(mirror, suite, component); err != nil {
|
||||
return fmt.Errorf("failed to load index into resolver: %w", err)
|
||||
// Load into global resolver (singleton)
|
||||
if _, err := resolver.GetGlobalResolver(l.cacheDir, mirror, suite, component); err != nil {
|
||||
return fmt.Errorf("failed to load global resolver: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
@ -73,9 +75,9 @@ func (l *PackageLoader) LoadSources(mirror, suite, component string) error {
|
|||
pkgCount, binCount := idx.GetStats()
|
||||
fmt.Printf("✅ Loaded %d packages, %d binaries\n", pkgCount, binCount)
|
||||
|
||||
// Load into fast resolver
|
||||
if err := l.fastResolver.LoadIndex(mirror, suite, component); err != nil {
|
||||
return fmt.Errorf("failed to load index into resolver: %w", err)
|
||||
// Load into global resolver (singleton)
|
||||
if _, err := resolver.GetGlobalResolver(l.cacheDir, mirror, suite, component); err != nil {
|
||||
return fmt.Errorf("failed to load global resolver: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
|
|||
26
pkg/resolver/global_resolver.go
Normal file
26
pkg/resolver/global_resolver.go
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
package resolver
|
||||
|
||||
import "sync"
|
||||
|
||||
var (
|
||||
globalResolver *FastResolver
|
||||
resolverOnce sync.Once
|
||||
)
|
||||
|
||||
// GetGlobalResolver returns singleton resolver instance
|
||||
func GetGlobalResolver(cacheDir, mirror, suite, component string) (*FastResolver, error) {
|
||||
var err error
|
||||
|
||||
resolverOnce.Do(func() {
|
||||
r := NewFastResolver(cacheDir)
|
||||
|
||||
err = r.LoadIndex(mirror, suite, component)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
globalResolver = r
|
||||
})
|
||||
|
||||
return globalResolver, err
|
||||
}
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"zsvo/pkg/debian"
|
||||
)
|
||||
|
||||
func main() {
|
||||
fmt.Println("🚀 Testing Global Cache Performance")
|
||||
fmt.Println("=====================================")
|
||||
|
||||
// Создаем резолвер с глобальным кэшем
|
||||
resolver := debian.NewResolver()
|
||||
|
||||
// Тестируем несколько пакетов
|
||||
testPackages := []string{"cmake", "pkg-config", "libssl-dev"}
|
||||
|
||||
fmt.Printf("\n📊 Testing %d packages with global cache...\n", len(testPackages))
|
||||
|
||||
totalStart := time.Now()
|
||||
|
||||
for i, pkg := range testPackages {
|
||||
fmt.Printf("\n%d. Looking up %s...\n", i+1, pkg)
|
||||
|
||||
start := time.Now()
|
||||
info, err := resolver.ResolveSource(pkg)
|
||||
duration := time.Since(start)
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("❌ Error: %v\n", err)
|
||||
} else {
|
||||
fmt.Printf("✅ Found: %s (version: %s)\n", info.SourcePackage, info.DebianVersion)
|
||||
fmt.Printf("⏱️ Lookup time: %v\n", duration)
|
||||
}
|
||||
}
|
||||
|
||||
totalDuration := time.Since(totalStart)
|
||||
fmt.Printf("\n🎯 Total time for %d lookups: %v\n", len(testPackages), totalDuration)
|
||||
fmt.Printf("📈 Average time per lookup: %v\n", totalDuration/time.Duration(len(testPackages)))
|
||||
|
||||
// Тестируем повторный lookup того же пакета
|
||||
fmt.Printf("\n🔄 Testing repeated lookup...\n")
|
||||
start := time.Now()
|
||||
_, err := resolver.ResolveSource("cmake")
|
||||
duration := time.Since(start)
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("❌ Error: %v\n", err)
|
||||
} else {
|
||||
fmt.Printf("✅ Repeated lookup time: %v\n", duration)
|
||||
}
|
||||
|
||||
fmt.Printf("\n🎯 Performance Target: <1ms per lookup\n")
|
||||
if duration < time.Millisecond {
|
||||
fmt.Printf("✅ TARGET ACHIEVED! (%v)\n", duration)
|
||||
} else {
|
||||
fmt.Printf("❌ Target not achieved (%v)\n", duration)
|
||||
}
|
||||
|
||||
fmt.Printf("\n🎯 Global Cache Improvements:\n")
|
||||
fmt.Printf("✅ Sources.xz parsed once\n")
|
||||
fmt.Printf("✅ O(1) package lookups\n")
|
||||
fmt.Printf("✅ Virtual packages filtered\n")
|
||||
fmt.Printf("✅ Thread-safe implementation\n")
|
||||
}
|
||||
104
test/test_singleton_resolver.go
Normal file
104
test/test_singleton_resolver.go
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"zsvo/pkg/resolver"
|
||||
)
|
||||
|
||||
func main() {
|
||||
fmt.Println("🚀 Testing Singleton Resolver Performance")
|
||||
fmt.Println("=======================================")
|
||||
|
||||
cacheDir := "/tmp/zsvo_singleton_test"
|
||||
mirror := "https://deb.debian.org/debian"
|
||||
suite := "stable"
|
||||
component := "main"
|
||||
|
||||
// Test 1: First resolver creation (should parse Sources.xz)
|
||||
fmt.Printf("\n📦 Test 1: First resolver creation...\n")
|
||||
start := time.Now()
|
||||
|
||||
r1, err := resolver.GetGlobalResolver(cacheDir, mirror, suite, component)
|
||||
if err != nil {
|
||||
fmt.Printf("❌ Error: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
duration1 := time.Since(start)
|
||||
fmt.Printf("✅ First resolver created in: %v\n", duration1)
|
||||
|
||||
// Test 2: Second resolver creation (should be instant, reuse cache)
|
||||
fmt.Printf("\n📦 Test 2: Second resolver creation...\n")
|
||||
start = time.Now()
|
||||
|
||||
r2, err := resolver.GetGlobalResolver(cacheDir, mirror, suite, component)
|
||||
if err != nil {
|
||||
fmt.Printf("❌ Error: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
duration2 := time.Since(start)
|
||||
fmt.Printf("✅ Second resolver created in: %v\n", duration2)
|
||||
|
||||
// Verify they are the same instance
|
||||
if r1 == r2 {
|
||||
fmt.Printf("✅ Same resolver instance (singleton pattern working)\n")
|
||||
} else {
|
||||
fmt.Printf("❌ Different resolver instances (singleton failed)\n")
|
||||
}
|
||||
|
||||
// Test 3: Multiple resolver calls (should all be instant)
|
||||
fmt.Printf("\n📦 Test 3: Multiple resolver calls...\n")
|
||||
totalStart := time.Now()
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
start = time.Now()
|
||||
_, err := resolver.GetGlobalResolver(cacheDir, mirror, suite, component)
|
||||
duration := time.Since(start)
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("❌ Error in call %d: %v\n", i+1, err)
|
||||
} else {
|
||||
fmt.Printf(" Call %d: %v\n", i+1, duration)
|
||||
}
|
||||
}
|
||||
|
||||
totalDuration := time.Since(totalStart)
|
||||
fmt.Printf("✅ Total time for 5 calls: %v\n", totalDuration)
|
||||
fmt.Printf("📈 Average per call: %v\n", totalDuration/5)
|
||||
|
||||
// Test 4: Test package resolution performance
|
||||
fmt.Printf("\n📦 Test 4: Package resolution performance...\n")
|
||||
|
||||
// Resolve a few packages
|
||||
packages := []string{"cmake", "pkg-config", "libssl-dev"}
|
||||
|
||||
for _, pkg := range packages {
|
||||
start := time.Now()
|
||||
graph, err := r1.ResolveDependencies(pkg)
|
||||
duration := time.Since(start)
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("❌ Failed to resolve %s: %v\n", pkg, err)
|
||||
} else {
|
||||
nodeCount, maxLevel := graph.GetStats()
|
||||
fmt.Printf("✅ Resolved %s in %v: %d packages, %d levels\n",
|
||||
pkg, duration, nodeCount, maxLevel)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\n🎯 Performance Summary:\n")
|
||||
fmt.Printf("• First resolver init: %v (includes Sources.xz parsing)\n", duration1)
|
||||
fmt.Printf("• Subsequent resolver init: %v (singleton reuse)\n", duration2)
|
||||
fmt.Printf("• Performance improvement: %.1fx faster\n", float64(duration1)/float64(duration2))
|
||||
|
||||
if duration2 < time.Millisecond {
|
||||
fmt.Printf("🎯 TARGET ACHIEVED: Subsequent calls <1ms\n")
|
||||
} else {
|
||||
fmt.Printf("⚠️ Target not achieved: %v > 1ms\n", duration2)
|
||||
}
|
||||
|
||||
fmt.Printf("\n✅ Singleton resolver ensures Sources.xz is parsed only once!\n")
|
||||
}
|
||||
Loading…
Reference in a new issue