package loader import ( "bufio" "compress/gzip" "fmt" "io" "net/http" "path" "strings" "time" "zsvo/pkg/cache" "zsvo/pkg/resolver" "github.com/ulikunitz/xz" ) // PackageLoader populates the cache from Debian Sources files type PackageLoader struct { client *http.Client cache *cache.IndexCache cacheDir string fastResolver *resolver.FastResolver } // NewPackageLoader creates a new package loader func NewPackageLoader(cacheDir string) *PackageLoader { return &PackageLoader{ client: &http.Client{ Timeout: 30 * time.Second, }, cache: cache.NewIndexCache(cacheDir), cacheDir: cacheDir, // Note: fastResolver will be initialized lazily via GetGlobalResolver } } // LoadSources loads and parses Sources.xz from the given repository func (l *PackageLoader) LoadSources(mirror, suite, component string) error { fmt.Printf("📦 Loading package index for %s/%s/%s...\n", mirror, suite, component) // Load or create index idx, err := l.cache.GetIndex(mirror, suite, component) if err != nil { return fmt.Errorf("failed to get index: %w", err) } // Check if cache is fresh (less than 24 hours old) if !idx.IsExpired(24 * time.Hour) { pkgCount, binCount := idx.GetStats() fmt.Printf("✅ Using fresh cache: %d packages, %d binaries\n", pkgCount, binCount) // Load into global resolver (singleton) if _, err := resolver.GetResolver(l.cacheDir, mirror, suite, component); err != nil { return fmt.Errorf("failed to load global resolver: %w", err) } return nil } fmt.Printf("🔄 Cache expired or missing, downloading fresh data...\n") // Download Sources.xz if err := l.downloadAndParseSources(mirror, suite, component, idx); err != nil { return fmt.Errorf("failed to download and parse sources: %w", err) } // Save to disk cacheFile := l.cache.CacheFilePath(mirror, suite, component) if err := idx.Save(cacheFile); err != nil { return fmt.Errorf("failed to save cache: %w", err) } pkgCount, binCount := idx.GetStats() fmt.Printf("✅ Loaded %d packages, %d binaries\n", pkgCount, binCount) // Load into global resolver (singleton) if _, err := resolver.GetResolver(l.cacheDir, mirror, suite, component); err != nil { return fmt.Errorf("failed to load global resolver: %w", err) } return nil } // downloadAndParseSources downloads and parses Sources.xz file func (l *PackageLoader) downloadAndParseSources(mirror, suite, component string, idx *cache.PackageIndex) error { // Try different compression formats formats := []struct { ext string decoder func(io.Reader) (io.Reader, error) }{ {"xz", func(r io.Reader) (io.Reader, error) { return xz.NewReader(r) }}, {"gz", func(r io.Reader) (io.Reader, error) { return gzip.NewReader(r) }}, {"", func(r io.Reader) (io.Reader, error) { return r, nil }}, } for _, format := range formats { url := l.buildSourcesURL(mirror, suite, component, format.ext) resp, err := l.client.Get(url) if err != nil { continue // Try next format } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { continue // Try next format } // Decode and parse reader, err := format.decoder(resp.Body) if err != nil { continue // Try next format } return l.parseSourcesFile(reader, idx) } return fmt.Errorf("failed to download Sources file in any format") } // parseSourcesFile parses a Sources file and populates the index func (l *PackageLoader) parseSourcesFile(reader io.Reader, idx *cache.PackageIndex) error { scanner := bufio.NewScanner(reader) scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) paragraph := make([]string, 0, 32) count := 0 flush := func() error { if len(paragraph) == 0 { return nil } entry, err := l.parseSourcesParagraph(paragraph) paragraph = paragraph[:0] if err != nil { return nil // Skip invalid entries } // Add to index cacheEntry := &cache.PackageEntry{ Package: entry.Package, Version: entry.Version, Directory: entry.Directory, DSCName: entry.DSCName, DSCSHA256: entry.DSCSHA256, Binaries: entry.Binaries, BuildDepends: entry.BuildDepends, } idx.AddPackage(cacheEntry) count++ // Progress indicator if count%1000 == 0 { fmt.Printf(" Parsed %d packages...\n", count) } return nil } for scanner.Scan() { line := scanner.Text() if strings.TrimSpace(line) == "" { if err := flush(); err != nil { return err } continue } paragraph = append(paragraph, line) } // Flush last paragraph if err := flush(); err != nil { return err } if err := scanner.Err(); err != nil { return err } fmt.Printf(" Parsed %d packages total\n", count) return nil } // parseSourcesParagraph parses a single package paragraph func (l *PackageLoader) parseSourcesParagraph(lines []string) (*SourceEntry, error) { fields := make(map[string]string) var currentKey string for _, raw := range lines { if strings.HasPrefix(raw, " ") || strings.HasPrefix(raw, "\t") { if currentKey == "" { continue } fields[currentKey] += "\n" + strings.TrimSpace(raw) continue } idx := strings.IndexByte(raw, ':') if idx <= 0 { continue } key := strings.TrimSpace(raw[:idx]) value := strings.TrimSpace(raw[idx+1:]) fields[key] = value currentKey = key } pkg := fields["Package"] ver := fields["Version"] dir := fields["Directory"] if pkg == "" || ver == "" || dir == "" { return nil, fmt.Errorf("missing required fields") } dscName, dscHash := l.parseChecksumsForDSC(fields["Checksums-Sha256"]) if dscName == "" { return nil, fmt.Errorf("no dsc in checksums") } dir = strings.Trim(strings.TrimSpace(dir), "/") return &SourceEntry{ Package: pkg, Version: ver, Directory: dir, DSCName: dscName, DSCSHA256: dscHash, Binaries: l.parseCommaSeparatedField(fields["Binary"]), BuildDepends: l.parseCommaSeparatedField(fields["Build-Depends"]), }, nil } // SourceEntry represents a parsed source package entry type SourceEntry struct { Package string Version string Directory string DSCName string DSCSHA256 string Binaries []string BuildDepends []string } // parseCommaSeparatedField parses a comma-separated field func (l *PackageLoader) parseCommaSeparatedField(raw string) []string { parts := strings.Split(strings.TrimSpace(raw), ",") out := make([]string, 0, len(parts)) seen := make(map[string]struct{}, len(parts)) for _, part := range parts { part = strings.TrimSpace(strings.ToLower(part)) if part == "" { continue } if _, exists := seen[part]; exists { continue } seen[part] = struct{}{} out = append(out, part) } return out } // parseChecksumsForDSC extracts DSC file info from checksums func (l *PackageLoader) parseChecksumsForDSC(raw string) (string, string) { for _, line := range strings.Split(raw, "\n") { fields := strings.Fields(strings.TrimSpace(line)) if len(fields) < 3 { continue } name := fields[2] if strings.HasSuffix(strings.ToLower(name), ".dsc") { return name, strings.ToLower(fields[0]) } } return "", "" } // buildSourcesURL builds the URL for Sources file func (l *PackageLoader) buildSourcesURL(mirror, suite, component, ext string) string { mirror = strings.TrimRight(strings.TrimSpace(mirror), "/") suite = strings.Trim(strings.TrimSpace(suite), "/") component = strings.Trim(strings.TrimSpace(component), "/") name := "Sources" if ext != "" { name += "." + ext } return mirror + "/" + path.Join("dists", suite, component, "source", name) } // GetFastResolver returns the loaded fast resolver func (l *PackageLoader) GetFastResolver() *resolver.FastResolver { return l.fastResolver } // LoadDefaultRepository loads the default Debian repository func (l *PackageLoader) LoadDefaultRepository() error { mirror := "https://deb.debian.org/debian" suite := "stable" component := "main" return l.LoadSources(mirror, suite, component) }