symbolizer: shell out to addr2line

README.md

@@ -127,6 +127,9 @@ Flags:
                                  Mode to demangle C++ symbols. Default mode
                                  is simplified: no parameters, no templates,
                                  no return type
+      --symbolizer-external-addr-2-line-path=""
+                                 Path to addr2line utility, to be used for
+                                 symbolization instead of native implementation
       --symbolizer-number-of-tries=3
                                  Number of tries to attempt to symbolize an
                                  unsybolized location

pkg/parca/parca.go

@@ -151,8 +151,9 @@ type FlagsStorage struct {
 }
 
 type FlagsSymbolizer struct {
-	DemangleMode  string `default:"simple" help:"Mode to demangle C++ symbols. Default mode is simplified: no parameters, no templates, no return type" enum:"simple,full,none,templates"`
-	NumberOfTries int    `default:"3" help:"Number of tries to attempt to symbolize an unsybolized location"`
+	DemangleMode          string `default:"simple" help:"Mode to demangle C++ symbols. Default mode is simplified: no parameters, no templates, no return type" enum:"simple,full,none,templates"`
+	ExternalAddr2linePath string `default:"" help:"Path to addr2line utility, to be used for symbolization instead of native implementation"`
+	NumberOfTries         int    `default:"3" help:"Number of tries to attempt to symbolize an unsybolized location"`
 }
 
 // FlagsDebuginfo configures the Parca Debuginfo client.
@@ -415,6 +416,7 @@ func Run(ctx context.Context, logger log.Logger, reg *prometheus.Registry, flags
 			symbolizer.NewBadgerCache(db),
 			debuginfo.NewFetcher(debuginfodClients, debuginfoBucket),
 			flags.Debuginfo.CacheDir,
+			flags.Symbolizer.ExternalAddr2linePath,
 			symbolizer.WithDemangleMode(flags.Symbolizer.DemangleMode),
 		),
 		memory.DefaultAllocator,

pkg/symbolizer/symbolizer.go

@@ -14,18 +14,23 @@
 package symbolizer
 
 import (
+	"bufio"
 	"context"
 	"debug/elf"
 	"errors"
 	"fmt"
 	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"strconv"
+	"strings"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 
 	debuginfopb "github.com/parca-dev/parca/gen/proto/go/parca/debuginfo/v1alpha1"
+	profilepb "github.com/parca-dev/parca/gen/proto/go/parca/metastore/v1alpha1"
 	"github.com/parca-dev/parca/pkg/debuginfo"
 	"github.com/parca-dev/parca/pkg/profile"
 	"github.com/parca-dev/parca/pkg/symbol/addr2line"
@@ -63,9 +68,10 @@ func WithDemangleMode(mode string) Option {
 type Symbolizer struct {
 	logger log.Logger
 
-	debuginfo DebuginfoFetcher
-	cache     SymbolizerCache
-	metadata  DebuginfoMetadata
+	debuginfo             DebuginfoFetcher
+	cache                 SymbolizerCache
+	metadata              DebuginfoMetadata
+	externalAddr2linePath string
 
 	demangler *demangle.Demangler
 
@@ -89,19 +95,21 @@ func New(
 	cache SymbolizerCache,
 	debuginfo DebuginfoFetcher,
 	tmpDir string,
+	externalAddr2linePath string,
 	opts ...Option,
 ) *Symbolizer {
 	const (
 		defaultDemangleMode = "simple"
 	)
 
 	s := &Symbolizer{
-		logger:    log.With(logger, "component", "symbolizer"),
-		cache:     cache,
-		debuginfo: debuginfo,
-		tmpDir:    tmpDir,
-		metadata:  metadata,
-		demangler: demangle.NewDemangler(defaultDemangleMode, false),
+		logger:                log.With(logger, "component", "symbolizer"),
+		cache:                 cache,
+		debuginfo:             debuginfo,
+		externalAddr2linePath: externalAddr2linePath,
+		tmpDir:                tmpDir,
+		metadata:              metadata,
+		demangler:             demangle.NewDemangler(defaultDemangleMode, false),
 	}
 
 	for _, opt := range opts {
@@ -126,7 +134,7 @@ func (s *Symbolizer) Symbolize(
 	req SymbolizationRequest,
 ) error {
 	if err := s.symbolize(ctx, req); err != nil {
-		level.Debug(s.logger).Log("msg", "failed to symbolize", "err", err)
+		level.Debug(s.logger).Log("msg", fmt.Sprintf("failed to symbolize: BuildID %s", req.BuildID), "err", err)
 	}
 
 	return nil
@@ -272,19 +280,159 @@ func (s *Symbolizer) getDebuginfo(ctx context.Context, buildID string) (string,
 }
 
 type cachedLiner struct {
-	logger    log.Logger
-	demangler *demangle.Demangler
-	filepath  string
-	f         *elf.File
-	quality   *debuginfopb.DebuginfoQuality
-	buildID   string
+	logger                log.Logger
+	demangler             *demangle.Demangler
+	filepath              string
+	f                     *elf.File
+	quality               *debuginfopb.DebuginfoQuality
+	buildID               string
+	externalAddr2linePath string
 
 	// this is the concrete liner
 	liner liner
 
 	cache SymbolizerCache
 }
 
+type externalExecutableLiner struct {
+	cmd       exec.Cmd
+	inputPipe io.WriteCloser
+	scanner   *bufio.Scanner
+
+	logger    log.Logger
+	demangler *demangle.Demangler
+}
+
+func (c *cachedLiner) newExternalExecutableLiner(
+	filepath string,
+) (liner, error) {
+	// Create cmd with pipes and scanner for reading output line by line
+	addr2lineCmd := exec.Command(c.externalAddr2linePath, "--exe", filepath, "-afiC")
+	addr2lineInput, err := addr2lineCmd.StdinPipe()
+	if err != nil {
+		return nil, fmt.Errorf("error creating input pipe for addr2line: %w", err)
+	}
+	addr2lineOutput, err := addr2lineCmd.StdoutPipe()
+	if err != nil {
+		return nil, fmt.Errorf("error creating output pipe for addr2line: %w", err)
+	}
+
+	// 100 KiB buffer, addr2line output can be very large sometimes
+	buf := make([]byte, 100*1024)
+	scanner := bufio.NewScanner(addr2lineOutput)
+	scanner.Buffer(buf, 100*1024)
+
+	// Start command to load executable info and wait for input
+	err = addr2lineCmd.Start()
+	if err != nil {
+		return nil, fmt.Errorf("error starting addr2line: %w", err)
+	}
+
+	return &externalExecutableLiner{
+		cmd:       *addr2lineCmd,
+		inputPipe: addr2lineInput,
+		scanner:   scanner,
+		logger:    c.logger,
+		demangler: c.demangler,
+	}, nil
+}
+
+func (l *externalExecutableLiner) Close() error {
+	err := l.inputPipe.Close()
+	if err != nil {
+		level.Warn(l.logger).Log("msg", "error closing input pipe of addr2line process", "err", err)
+		return err
+	}
+
+	err = l.cmd.Wait()
+	if err != nil {
+		level.Warn(l.logger).Log("msg", "error waiting for addr2line process to exit", "err", err)
+		return err
+	}
+	level.Debug(l.logger).Log("msg", "addr2line process exited successfully, closed external executable liner")
+
+	return nil
+}
+
+func (l *externalExecutableLiner) PCToLines(ctx context.Context, pc uint64) ([]profile.LocationLine, error) {
+	// Write the address to addr2line stdin twice in order to get addresses as a proper delimiter
+	for i := 0; i < 2; i++ {
+		_, err := io.WriteString(l.inputPipe, fmt.Sprintf("0x%x\n", pc))
+		if err != nil {
+			return nil, fmt.Errorf("error writing to addr2line stdin: %w\n", err)
+		}
+	}
+
+	// Auxiliary function to call Scan() and handle errors
+	scan := func() error {
+		if !l.scanner.Scan() {
+			if err := l.scanner.Err(); err != nil {
+				return fmt.Errorf("error scanning addr2line output: %w", err)
+			}
+			return fmt.Errorf("unexpected EOF while scanning addr2line output")
+		}
+		return nil
+	}
+
+	lines := []profile.LocationLine{}
+	// addr2line output consists of a variable number of pairs of lines per address
+	// The first line of each block of output is the address, if we have to skip additional lines
+	// it means the previous read loop left some output unread
+	err := scan()
+	if err != nil {
+		return nil, err
+	}
+	addressLine := l.scanner.Text()
+	for !strings.HasPrefix(addressLine, "0x") {
+		err := scan()
+		if err != nil {
+			return nil, err
+		}
+		addressLine = l.scanner.Text()
+	}
+
+	keepReading := true
+	err = scan()
+	if err != nil {
+		return nil, err
+	}
+	for keepReading {
+		line1 := l.scanner.Text()
+
+		err := scan()
+		if err != nil {
+			return nil, err
+		}
+		line2 := l.scanner.Text()
+
+		line1 = strings.TrimSuffix(line1, "\n")
+		line2 = strings.TrimSuffix(line2, "\n")
+
+		fileLocation := strings.Split(line2, ":")
+		lineNum, err := strconv.Atoi(fileLocation[1])
+		if err != nil {
+			lineNum = 0
+		}
+
+		lines = append(lines, profile.LocationLine{
+			Line: int64(lineNum),
+			Function: &profilepb.Function{
+				StartLine: int64(lineNum),
+				Filename:  fileLocation[0],
+				Name:      line1,
+			},
+		})
+
+		err = scan()
+		if err != nil {
+			return nil, err
+		}
+		keepReading = !strings.HasPrefix(l.scanner.Text(), "0x")
+	}
+
+	return lines, nil
+}
+
 // newConcreteLiner creates a new liner for the given mapping and object file path.
 func (s *Symbolizer) newLiner(
 	filepath string,
@@ -293,12 +441,13 @@ func (s *Symbolizer) newLiner(
 	buildID string,
 ) liner {
 	return &cachedLiner{
-		logger:    s.logger,
-		demangler: s.demangler,
-		filepath:  filepath,
-		f:         f,
-		quality:   quality,
-		buildID:   buildID,
+		logger:                s.logger,
+		demangler:             s.demangler,
+		filepath:              filepath,
+		f:                     f,
+		quality:               quality,
+		buildID:               buildID,
+		externalAddr2linePath: s.externalAddr2linePath,
 
 		cache: s.cache,
 	}
@@ -344,6 +493,9 @@ func (c *cachedLiner) PCToLines(ctx context.Context, pc uint64) ([]profile.Locat
 
 func (c *cachedLiner) newConcreteLiner(filepath string, f *elf.File, quality *debuginfopb.DebuginfoQuality) (liner, error) {
 	switch {
+	case c.externalAddr2linePath != "":
+		level.Debug(c.logger).Log("msg", fmt.Sprintf("using external addr2line liner with binary: %s", c.externalAddr2linePath))
+		return c.newExternalExecutableLiner(filepath)
 	case quality.HasDwarf:
 		lnr, err := addr2line.DWARF(c.logger, c.filepath, c.f, c.demangler)
 		if err != nil {

pkg/symbolizer/symbolizer_test.go

@@ -67,6 +67,7 @@ func TestSymbolizer(t *testing.T) {
 		&NoopSymbolizerCache{},
 		debuginfo.NewFetcher(debuginfodClient, bucket),
 		symbolizerCacheDir,
+		"",
 	)
 
 	ctx := context.Background()