From f07f62244c3474048145f4259c267f5514586ca0 Mon Sep 17 00:00:00 2001 From: itexpert228 <67105314+fdaser1337@users.noreply.github.com> Date: Sat, 14 Mar 2026 22:03:19 +0300 Subject: [PATCH] pisechka --- cmd/install.go | 67 +++++---- pkg/builder/builder.go | 5 +- pkg/debian/source.go | 303 ++++++++++++++++++++++++++++++++++++----- 3 files changed, 313 insertions(+), 62 deletions(-) diff --git a/cmd/install.go b/cmd/install.go index dd8502b..8068e6d 100644 --- a/cmd/install.go +++ b/cmd/install.go @@ -186,7 +186,8 @@ func looksLikeFilePath(target string) bool { } const maxAutoBuildDepth = 15 // Увеличим для сложных цепочек зависимостей -const parallelWorkers = 4 // Количество параллельных воркеров для сборки +const parallelWorkers = 20 // Параллельный поиск зависимостей (HTTP) +const buildWorkers = 4 // Параллельное построение пакетов (CPU intensive) type autoBuildSession struct { workDir string @@ -641,6 +642,7 @@ func (s *autoBuildSession) installBuildDependency(dep, packagePath string) error } // collectAllDependencies recursively collects all dependencies into a graph without building +// Now uses parallel workers (200 concurrent) for faster dependency resolution func (s *autoBuildSession) collectAllDependencies(rootPkg string, graph *depGraph, stack []string) error { rootPkg = normalizePackageName(rootPkg) if rootPkg == "" { @@ -679,7 +681,8 @@ func (s *autoBuildSession) collectAllDependencies(rootPkg string, graph *depGrap node.recipe = rcp node.buildDepends = srcInfo.BuildDepends - // Process build dependencies + // Collect all dependency names first + var depsToProcess []string for _, dep := range srcInfo.BuildDepends { depName := extractPackageNameFromConstraint(dep) if depName == "" { @@ -698,10 +701,41 @@ func (s *autoBuildSession) collectAllDependencies(rootPkg string, graph *depGrap // Add dependency relationship graph.addDependency(pkgName, sourcePkg) - // Recursively collect this dependency's dependencies - if err := s.collectAllDependencies(sourcePkg, graph, append(stack, pkgName)); err != nil { - // Log but don't fail - some deps might be optional - fmt.Printf(" ⚠️ Could not collect dependency %s: %v\n", sourcePkg, err) + // Check if already in graph + depNode := graph.getOrCreateNode(sourcePkg) + if depNode.srcInfo == nil { + depsToProcess = append(depsToProcess, sourcePkg) + } + } + + // Process dependencies in parallel using worker pool + if len(depsToProcess) > 0 { + var wg sync.WaitGroup + errChan := make(chan error, len(depsToProcess)) + + // Semaphore to limit concurrent workers (200 workers) + semaphore := make(chan struct{}, parallelWorkers) + + for _, dep := range depsToProcess { + wg.Add(1) + go func(depName string) { + defer wg.Done() + + semaphore <- struct{}{} + defer func() { <-semaphore }() + + if err := s.collectAllDependencies(depName, graph, append(stack, pkgName)); err != nil { + errChan <- fmt.Errorf("could not collect dependency %s: %w", depName, err) + } + }(dep) + } + + wg.Wait() + close(errChan) + + // Log errors but don't fail + for err := range errChan { + fmt.Printf(" ⚠️ %v\n", err) } } @@ -749,7 +783,7 @@ func (s *autoBuildSession) buildDependenciesParallel(graph *depGraph) error { errChan := make(chan error, len(nodesToBuild)) // Limit concurrent builds - semaphore := make(chan struct{}, parallelWorkers) + semaphore := make(chan struct{}, buildWorkers) for _, node := range nodesToBuild { wg.Add(1) @@ -857,6 +891,7 @@ func joinPathListUnique(parts []string) string { } var simplePkgNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9+.-]*[a-z0-9]$`) +var versionNumberPattern = regexp.MustCompile(`[0-9]+\.[0-9]+`) var missingCommandPatterns = []*regexp.Regexp{ regexp.MustCompile(`(?m)(?:^|[\s:])(?:/bin/)?sh:\s*(?:\d+:\s*)?([a-zA-Z0-9+_.-]+):\s*(?:command not found|not found)\b`), regexp.MustCompile(`(?m)\b([a-zA-Z0-9+_.-]+):\s*command not found\b`), @@ -1171,8 +1206,7 @@ func mapDebianPackageToSource(pkg string) string { // Remove lib prefix for source packages base = strings.TrimPrefix(base, "lib") // Convert numbers like 5.1, 2.0 etc - re := regexp.MustCompile(`[0-9]+\.[0-9]+`) - base = re.ReplaceAllString(base, "") + base = versionNumberPattern.ReplaceAllString(base, "") if base != "" { return base } @@ -1320,20 +1354,7 @@ func toolAlreadyAvailable(dep string) bool { return false } - // Для тестов на macOS, предположим что базовые инструменты уже доступны - basicTools := map[string]bool{ - "gcc": true, "clang": true, "cc": true, - "make": true, "cmake": true, "git": true, - "pkg-config": true, "pkgconf": true, - "flex": true, "bison": true, "m4": true, - "autoconf": true, "automake": true, "libtool": true, - "python3": true, "perl": true, - } - - if basicTools[dep] { - return true - } - + // Check if command exists in PATH commands := commandHintsByDep[dep] if len(commands) == 0 { commands = []string{dep} diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index a0e591b..a57f81e 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -255,9 +255,6 @@ func (b *Builder) executeCommand(workDir, command string, env []string) error { log.Printf("Executing command: %s", command) - cmd := exec.Command("sh", "-c", command) - cmd.Dir = workDir - cmd.Env = env if b.quiet { // Create a context with timeout to prevent hanging - use 2 hours for large packages like gcc/llvm ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour) @@ -280,7 +277,7 @@ func (b *Builder) executeCommand(workDir, command string, env []string) error { // For non-quiet mode, still set a reasonable timeout - use 2 hours for large packages ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour) defer cancel() - cmd = exec.CommandContext(ctx, "sh", "-c", command) + cmd := exec.CommandContext(ctx, "sh", "-c", command) cmd.Dir = workDir cmd.Env = env diff --git a/pkg/debian/source.go b/pkg/debian/source.go index f339e26..32598c3 100644 --- a/pkg/debian/source.go +++ b/pkg/debian/source.go @@ -6,9 +6,12 @@ import ( "fmt" "io" "net/http" + "os" "path" + "path/filepath" "regexp" "strings" + "sync" "time" "github.com/ulikunitz/xz" @@ -19,8 +22,8 @@ var packageNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9+.-]*$`) const defaultDebianMirror = "https://deb.debian.org/debian" var ( - defaultSuites = []string{"stable", "testing", "unstable"} - defaultComponents = []string{"main", "contrib", "non-free", "non-free-firmware"} + defaultSuites = []string{"stable"} // Только stable для скорости + defaultComponents = []string{"main"} // Только main для скорости ) // SourceInfo describes Debian source package coordinates resolved over HTTP. @@ -37,14 +40,26 @@ type SourceInfo struct { } const maxAutoBuildDepth = 15 // Увеличим для сложных цепочек зависимостей -const parallelWorkers = 1 // Временно отключаем параллельную сборку (1 воркер = последовательно) +const parallelWorkers = 20 // Параллельный поиск зависимостей (HTTP запросы) +const buildWorkers = 4 // Параллельное построение пакетов + +// CachedSources holds parsed Sources data for fast lookups +type CachedSources struct { + packages map[string]*sourceRecord // key: package name + path string // cache file path + mu sync.RWMutex +} // Resolver queries Debian source metadata over HTTP. type Resolver struct { - client *http.Client - mirrors []string - suites []string - components []string + client *http.Client + mirrors []string + suites []string + components []string + cache map[string]*SourceInfo // Кеш найденных пакетов + cacheMu sync.RWMutex // Защита кеша + cachedSources map[string]*CachedSources // Кешированные Sources файлы по ключу + sourcesMu sync.RWMutex // Защита cachedSources map } // ResolverOption customizes resolver behavior. @@ -87,10 +102,12 @@ func NewResolver(opts ...ResolverOption) *Resolver { }, }, mirrors: []string{ - defaultDebianMirror, + defaultDebianMirror, // deb.debian.org (CDN) }, - suites: append([]string(nil), defaultSuites...), - components: append([]string(nil), defaultComponents...), + suites: append([]string(nil), defaultSuites...), + components: append([]string(nil), defaultComponents...), + cache: make(map[string]*SourceInfo), + cachedSources: make(map[string]*CachedSources), } for _, opt := range opts { @@ -118,6 +135,14 @@ func (r *Resolver) ResolveSource(pkg string) (*SourceInfo, error) { return nil, fmt.Errorf("invalid package name %q", pkg) } + // Проверяем кеш + r.cacheMu.RLock() + if cached, ok := r.cache[pkg]; ok { + r.cacheMu.RUnlock() + return cached, nil + } + r.cacheMu.RUnlock() + fmt.Printf(" [resolver] Looking up %s...\n", pkg) start := time.Now() defer func() { @@ -136,8 +161,7 @@ func (r *Resolver) ResolveSource(pkg string) (*SourceInfo, error) { continue } - fmt.Printf(" [resolver] ✓ Found %s in %s/%s/%s\n", pkg, mirror, suite, component) - return &SourceInfo{ + result := &SourceInfo{ RequestedPackage: pkg, SourcePackage: record.Package, DSCURL: strings.TrimRight(mirror, "/") + "/" + path.Join(record.Directory, record.DSCName), @@ -147,7 +171,15 @@ func (r *Resolver) ResolveSource(pkg string) (*SourceInfo, error) { Suite: suite, Component: component, BuildDepends: record.BuildDepends, - }, nil + } + + // Сохраняем в кеш + r.cacheMu.Lock() + r.cache[pkg] = result + r.cacheMu.Unlock() + + fmt.Printf(" [resolver] ✓ Found %s in %s/%s/%s\n", pkg, mirror, suite, component) + return result, nil } } } @@ -170,31 +202,18 @@ type sourceRecord struct { } func (r *Resolver) findPackageInIndex(mirror, suite, component, pkg string) (*sourceRecord, error) { - variants := []struct { - ext string - decoder func(io.Reader) (io.Reader, error) - }{ - {ext: "xz", decoder: decodeXZ}, - {ext: "gz", decoder: decodeGzip}, - {ext: "", decoder: passthrough}, + // First, ensure Sources file is loaded into memory + cached, err := r.loadCachedSources(mirror, suite, component) + if err != nil { + return nil, err } - var firstErr error - for _, v := range variants { - indexURL := buildSourcesURL(mirror, suite, component, v.ext) - record, err := r.findInSingleIndex(indexURL, v.decoder, pkg) - if err == nil { - return record, nil - } - if firstErr == nil { - firstErr = err - } + // Lookup from in-memory map (O(1) instead of HTTP scan) + if rec, found := r.lookupFromCache(pkg, cached); found { + return rec, nil } - if firstErr == nil { - firstErr = fmt.Errorf("could not read sources index") - } - return nil, firstErr + return nil, fmt.Errorf("package not found in cache") } func (r *Resolver) findInSingleIndex(indexURL string, decoder func(io.Reader) (io.Reader, error), pkg string) (*sourceRecord, error) { @@ -389,7 +408,9 @@ func normalizeUpstreamVersion(debianVersion string) string { v = v[idx+1:] } - if idx := strings.LastIndex(v, "-"); idx > 0 { + // Find the last dash that separates upstream version from Debian revision + // For versions like "1.0-1-ubuntu1", we want to cut at the first dash to get "1.0" + if idx := strings.Index(v, "-"); idx >= 0 { v = v[:idx] } @@ -421,3 +442,215 @@ func normalizeList(in []string) []string { } return out } + +// cacheDir returns the cache directory for zsvo +func cacheDir() string { + if dir := os.Getenv("ZSVO_CACHE"); dir != "" { + return dir + } + if home, err := os.UserHomeDir(); err == nil { + return filepath.Join(home, ".cache", "zsvo") + } + return "/var/cache/zsvo" +} + +// ensureCacheDir creates the cache directory if it doesn't exist +func ensureCacheDir() error { + dir := cacheDir() + return os.MkdirAll(dir, 0755) +} + +// cachePath returns the path for a cached Sources file +func (r *Resolver) cachePath(mirror, suite, component string) string { + host := strings.ReplaceAll(strings.TrimPrefix(mirror, "https://"), "/", "_") + return filepath.Join(cacheDir(), fmt.Sprintf("Sources_%s_%s_%s.xz", host, suite, component)) +} + +// cacheKey returns a unique key for mirror/suite/component combination +func (r *Resolver) cacheKey(mirror, suite, component string) string { + return fmt.Sprintf("%s:%s:%s", mirror, suite, component) +} + +// loadCachedSources loads Sources.xz from cache or downloads it if not exists +func (r *Resolver) loadCachedSources(mirror, suite, component string) (*CachedSources, error) { + key := r.cacheKey(mirror, suite, component) + + // Fast path: check if already loaded + r.sourcesMu.RLock() + if cached, exists := r.cachedSources[key]; exists { + r.sourcesMu.RUnlock() + return cached, nil + } + r.sourcesMu.RUnlock() + + // Slow path: load with write lock + r.sourcesMu.Lock() + defer r.sourcesMu.Unlock() + + // Double-check after acquiring write lock + if cached, exists := r.cachedSources[key]; exists { + return cached, nil + } + + cachePath := r.cachePath(mirror, suite, component) + cached := &CachedSources{ + packages: make(map[string]*sourceRecord), + path: cachePath, + } + + // Try to load from disk cache first + if _, err := os.Stat(cachePath); err == nil { + // File exists on disk, parse it + if err := r.parseSourcesFileToCache(cachePath, mirror, cached); err == nil { + r.cachedSources[key] = cached + return cached, nil + } + } + + // Download from HTTP + url := buildSourcesURL(mirror, suite, component, "xz") + if err := r.downloadSources(url, cachePath); err != nil { + // Try gz + url = buildSourcesURL(mirror, suite, component, "gz") + cachePath = strings.TrimSuffix(cachePath, ".xz") + ".gz" + if err := r.downloadSources(url, cachePath); err != nil { + // Try uncompressed + url = buildSourcesURL(mirror, suite, component, "") + cachePath = strings.TrimSuffix(cachePath, ".gz") + if err := r.downloadSources(url, cachePath); err != nil { + return nil, err + } + } + } + + // Parse the downloaded file + if err := r.parseSourcesFileToCache(cachePath, mirror, cached); err != nil { + return nil, err + } + + r.cachedSources[key] = cached + return cached, nil +} + +// downloadSources downloads a Sources file from URL to local path +func (r *Resolver) downloadSources(url, localPath string) error { + fmt.Printf(" [resolver] Downloading %s...\n", url) + start := time.Now() + + resp, err := r.client.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("http %d", resp.StatusCode) + } + + if err := ensureCacheDir(); err != nil { + return err + } + + file, err := os.Create(localPath) + if err != nil { + return err + } + defer file.Close() + + if _, err := io.Copy(file, resp.Body); err != nil { + os.Remove(localPath) + return err + } + + fmt.Printf(" [resolver] Downloaded in %v\n", time.Since(start)) + return nil +} + +// parseSourcesFileToCache parses a local Sources file into specific CachedSources instance +func (r *Resolver) parseSourcesFileToCache(path, mirror string, cached *CachedSources) error { + fmt.Printf(" [resolver] Parsing %s...\n", filepath.Base(path)) + start := time.Now() + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + var reader io.Reader = file + + // Decompress if needed + if strings.HasSuffix(path, ".xz") { + r, err := xz.NewReader(file) + if err != nil { + return err + } + reader = r + } else if strings.HasSuffix(path, ".gz") { + r, err := gzip.NewReader(file) + if err != nil { + return err + } + defer r.Close() + reader = r + } + + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + + paragraph := make([]string, 0, 32) + count := 0 + + flush := func() { + if len(paragraph) == 0 { + return + } + rec, err := parseSourcesParagraph(paragraph) + paragraph = paragraph[:0] + if err != nil { + return + } + + cached.mu.Lock() + cached.packages[rec.Package] = &rec + // Also index by binary names + for _, bin := range rec.Binaries { + bin = strings.TrimSpace(strings.ToLower(bin)) + if bin != "" && bin != rec.Package { + cached.packages[bin] = &rec + } + } + cached.mu.Unlock() + count++ + } + + for scanner.Scan() { + line := scanner.Text() + if strings.TrimSpace(line) == "" { + flush() + continue + } + paragraph = append(paragraph, line) + } + + flush() + + if err := scanner.Err(); err != nil { + return err + } + + cached.path = path + fmt.Printf(" [resolver] Parsed %d packages in %v\n", count, time.Since(start)) + return nil +} + +// lookupFromCache searches for a package in the in-memory cache +func (r *Resolver) lookupFromCache(pkg string, cached *CachedSources) (*sourceRecord, bool) { + cached.mu.RLock() + defer cached.mu.RUnlock() + + if rec, ok := cached.packages[pkg]; ok { + return rec, true + } + return nil, false +}