From 29f88279ba7911d2b6cb06be3992e7ba4881b73a Mon Sep 17 00:00:00 2001 From: Dan Willemsen Date: Sat, 18 Feb 2017 18:12:41 -0800 Subject: [PATCH] Filter kati output to collapse verbose messages 1. Collapses the "including .../Android.mk ..." lines like ninja does, so that we overwrite the last line if there wasn't anything else to print. 2. Strips ansi control codes so that log files don't include unreadable characters. Test: m -j Test: m -j | tee output.log (with colored output) Change-Id: Ib18437f6f9d37084360097a9d586800c833072c5 --- ui/build/Android.bp | 7 ++++ ui/build/context.go | 7 ++++ ui/build/kati.go | 73 ++++++++++++++++++++++++++++++++++-- ui/build/util.go | 82 +++++++++++++++++++++++++++++++++++++++++ ui/build/util_darwin.go | 21 +++++++++++ ui/build/util_linux.go | 21 +++++++++++ ui/build/util_test.go | 62 +++++++++++++++++++++++++++++++ 7 files changed, 270 insertions(+), 3 deletions(-) create mode 100644 ui/build/util_darwin.go create mode 100644 ui/build/util_linux.go create mode 100644 ui/build/util_test.go diff --git a/ui/build/Android.bp b/ui/build/Android.bp index e4044e186..51aed2c18 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -33,5 +33,12 @@ bootstrap_go_package { ], testSrcs: [ "environment_test.go", + "util_test.go", ], + darwin: { + srcs: ["util_darwin.go"], + }, + linux: { + srcs: ["util_linux.go"], + }, } diff --git a/ui/build/context.go b/ui/build/context.go index 8144e586e..f85bb6c0c 100644 --- a/ui/build/context.go +++ b/ui/build/context.go @@ -95,3 +95,10 @@ func (c ContextImpl) ImportNinjaLog(filename string, startOffset time.Time) { c.Tracer.ImportNinjaLog(c.Thread, filename, startOffset) } } + +func (c ContextImpl) IsTerminal() bool { + if term, ok := os.LookupEnv("TERM"); ok { + return term != "dumb" && isTerminal(c.Stdout()) && isTerminal(c.Stderr()) + } + return false +} diff --git a/ui/build/kati.go b/ui/build/kati.go index 423bcbc9d..86db8878a 100644 --- a/ui/build/kati.go +++ b/ui/build/kati.go @@ -15,11 +15,14 @@ package build import ( + "bufio" "crypto/md5" "fmt" + "io" "io/ioutil" "os/exec" "path/filepath" + "regexp" "strconv" "strings" ) @@ -94,10 +97,20 @@ func runKati(ctx Context, config Config) { cmd := exec.CommandContext(ctx.Context, executable, args...) cmd.Env = config.Environment().Environ() - cmd.Stdout = ctx.Stdout() - cmd.Stderr = ctx.Stderr() + pipe, err := cmd.StdoutPipe() + if err != nil { + ctx.Fatalln("Error getting output pipe for ckati:", err) + } + cmd.Stderr = cmd.Stdout + ctx.Verboseln(cmd.Path, cmd.Args) - if err := cmd.Run(); err != nil { + if err := cmd.Start(); err != nil { + ctx.Fatalln("Failed to run ckati:", err) + } + + katiRewriteOutput(ctx, pipe) + + if err := cmd.Wait(); err != nil { if e, ok := err.(*exec.ExitError); ok { ctx.Fatalln("ckati failed with:", e.ProcessState.String()) } else { @@ -105,3 +118,57 @@ func runKati(ctx Context, config Config) { } } } + +var katiIncludeRe = regexp.MustCompile(`^(\[\d+/\d+] )?including [^ ]+ ...$`) + +func katiRewriteOutput(ctx Context, pipe io.ReadCloser) { + haveBlankLine := true + smartTerminal := ctx.IsTerminal() + + scanner := bufio.NewScanner(pipe) + for scanner.Scan() { + line := scanner.Text() + verbose := katiIncludeRe.MatchString(line) + + // For verbose lines, write them on the current line without a newline, + // then overwrite them if the next thing we're printing is another + // verbose line. + if smartTerminal && verbose { + // Limit line width to the terminal width, otherwise we'll wrap onto + // another line and we won't delete the previous line. + // + // Run this on every line in case the window has been resized while + // we're printing. This could be optimized to only re-run when we + // get SIGWINCH if it ever becomes too time consuming. + if max, ok := termWidth(ctx.Stdout()); ok { + if len(line) > max { + // Just do a max. Ninja elides the middle, but that's + // more complicated and these lines aren't that important. + line = line[:max] + } + } + + // Move to the beginning on the line, print the output, then clear + // the rest of the line. + fmt.Fprint(ctx.Stdout(), "\r", line, "\x1b[K") + haveBlankLine = false + continue + } else if smartTerminal && !haveBlankLine { + // If we've previously written a verbose message, send a newline to save + // that message instead of overwriting it. + fmt.Fprintln(ctx.Stdout()) + haveBlankLine = true + } else if !smartTerminal { + // Most editors display these as garbage, so strip them out. + line = string(stripAnsiEscapes([]byte(line))) + } + + // Assume that non-verbose lines are important enough for stderr + fmt.Fprintln(ctx.Stderr(), line) + } + + // Save our last verbose line. + if !haveBlankLine { + fmt.Fprintln(ctx.Stdout()) + } +} diff --git a/ui/build/util.go b/ui/build/util.go index ad084da66..37ac6b9a7 100644 --- a/ui/build/util.go +++ b/ui/build/util.go @@ -15,9 +15,13 @@ package build import ( + "bytes" + "io" "os" "path/filepath" "strings" + "syscall" + "unsafe" ) // indexList finds the index of a string in a []string @@ -77,3 +81,81 @@ func decodeKeyValue(str string) (string, string, bool) { } return str[:idx], str[idx+1:], true } + +func isTerminal(w io.Writer) bool { + if f, ok := w.(*os.File); ok { + var termios syscall.Termios + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, f.Fd(), + ioctlGetTermios, uintptr(unsafe.Pointer(&termios)), + 0, 0, 0) + return err == 0 + } + return false +} + +func termWidth(w io.Writer) (int, bool) { + if f, ok := w.(*os.File); ok { + var winsize struct { + ws_row, ws_column uint16 + ws_xpixel, ws_ypixel uint16 + } + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, f.Fd(), + syscall.TIOCGWINSZ, uintptr(unsafe.Pointer(&winsize)), + 0, 0, 0) + return int(winsize.ws_column), err == 0 + } + return 0, false +} + +// stripAnsiEscapes strips ANSI control codes from a byte array in place. +func stripAnsiEscapes(input []byte) []byte { + // read represents the remaining part of input that needs to be processed. + read := input + // write represents where we should be writing in input. + // It will share the same backing store as input so that we make our modifications + // in place. + write := input + + // advance will copy count bytes from read to write and advance those slices + advance := func(write, read []byte, count int) ([]byte, []byte) { + copy(write, read[:count]) + return write[count:], read[count:] + } + + for { + // Find the next escape sequence + i := bytes.IndexByte(read, 0x1b) + // If it isn't found, or if there isn't room for [, finish + if i == -1 || i+1 >= len(read) { + copy(write, read) + break + } + + // Not a CSI code, continue searching + if read[i+1] != '[' { + write, read = advance(write, read, i+1) + continue + } + + // Found a CSI code, advance up to the + write, read = advance(write, read, i) + + // Find the end of the CSI code + i = bytes.IndexFunc(read, func(r rune) bool { + return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') + }) + if i == -1 { + // We didn't find the end of the code, just remove the rest + i = len(read) - 1 + } + + // Strip off the end marker too + i = i + 1 + + // Skip the reader forward and reduce final length by that amount + read = read[i:] + input = input[:len(input)-i] + } + + return input +} diff --git a/ui/build/util_darwin.go b/ui/build/util_darwin.go new file mode 100644 index 000000000..254a9b877 --- /dev/null +++ b/ui/build/util_darwin.go @@ -0,0 +1,21 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "syscall" +) + +const ioctlGetTermios = syscall.TIOCGETA diff --git a/ui/build/util_linux.go b/ui/build/util_linux.go new file mode 100644 index 000000000..0a4e1d297 --- /dev/null +++ b/ui/build/util_linux.go @@ -0,0 +1,21 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "syscall" +) + +const ioctlGetTermios = syscall.TCGETS diff --git a/ui/build/util_test.go b/ui/build/util_test.go new file mode 100644 index 000000000..e85eadad2 --- /dev/null +++ b/ui/build/util_test.go @@ -0,0 +1,62 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import "testing" + +func TestStripAnsiEscapes(t *testing.T) { + testcases := []struct { + input string + output string + }{ + { + "", + "", + }, + { + "This is a test", + "This is a test", + }, + { + "interrupted: \x1b[12", + "interrupted: ", + }, + { + "other \x1bescape \x1b", + "other \x1bescape \x1b", + }, + { // from pretty-error macro + "\x1b[1mart/Android.mk: \x1b[31merror:\x1b[0m\x1b[1m art: test error \x1b[0m", + "art/Android.mk: error: art: test error ", + }, + { // from envsetup.sh make wrapper + "\x1b[0;31m#### make failed to build some targets (2 seconds) ####\x1b[00m", + "#### make failed to build some targets (2 seconds) ####", + }, + { // from clang (via ninja testcase) + "\x1b[1maffixmgr.cxx:286:15: \x1b[0m\x1b[0;1;35mwarning: \x1b[0m\x1b[1musing the result... [-Wparentheses]\x1b[0m", + "affixmgr.cxx:286:15: warning: using the result... [-Wparentheses]", + }, + } + for _, tc := range testcases { + got := string(stripAnsiEscapes([]byte(tc.input))) + if got != tc.output { + t.Errorf("output strings didn't match\n"+ + "input: %#v\n"+ + " want: %#v\n"+ + " got: %#v", tc.input, tc.output, got) + } + } +}