Make/makefile progress indication!

Question

Look at this makefile, it has some sort of primitive progress indication (could have been a progress bar).

Please give me suggestions/comments on it!

---

# BUILD is initially undefined
ifndef BUILD

# max equals 256 x's
sixteen := x x x x x x x x x x x x x x x x
MAX := $(foreach x,$(sixteen),$(sixteen))

# T estimates how many targets we are building by replacing BUILD with a special string
T := $(shell $(MAKE) -nrRf $(firstword $(MAKEFILE_LIST)) $(MAKECMDGOALS) \
            BUILD="COUNTTHIS" | grep -c "COUNTTHIS")

# N is the number of pending targets in base 1, well in fact, base x :-)
N := $(wordlist 1,$T,$(MAX))

# auto-decrementing counter that returns the number of pending targets in base 10
counter = $(words $N)$(eval N := $(wordlist 2,$(words $N),$N))

# BUILD is now defined to show the progress, this also avoids redefining T in loop
BUILD = @echo $(counter) of $(T)
endif

# dummy phony targets

.PHONY: all clean

all: target
    @echo done

clean:
    @rm -f target *.c

# dummy build rules

target: a.c b.c c.c d.c e.c f.c g.c
    @touch $@
    $(BUILD)

%.c:
    @touch $@
    $(BUILD)

---

All suggestions welcome!

Answer 1

This one is less intrusive and more awesome.

ifneq ($(words $(MAKECMDGOALS)),1)
.DEFAULT_GOAL = all
%:
        @$(MAKE) $@ --no-print-directory -rRf $(firstword $(MAKEFILE_LIST))
else
ifndef ECHO
T := $(shell $(MAKE) $(MAKECMDGOALS) --no-print-directory \
      -nrRf $(firstword $(MAKEFILE_LIST)) \
      ECHO="COUNTTHIS" | grep -c "COUNTTHIS")

N := x
C = $(words $N)$(eval N := x $N)
ECHO = echo "`expr " [\`expr $C '*' 100 / $T\`" : '.*\(....\)$$'`%]"
endif

.PHONY: all clean

all: target
        @$(ECHO) All done

clean:
        @rm -f target *.c
#       @$(ECHO) Clean done

target: a.c b.c c.c d.c e.c
        @$(ECHO) Linking $@
        @sleep 0.1
        @touch $@

%.c:
        @$(ECHO) Compiling $@
        @sleep 0.1
        @touch $@

endif

Answer 2

Nice trick! (-:

But not really scalable for growing projects that are distributed across many directories with lots of makefiles.

I'd be more inclined to have logging sprinkled through the [Mm]akefiles* in your project and use that to keep track of progress.

Just a thought. BTW Thanks for sharing this.

Edit: Just had a thought. This could be useful in a modified form to display a throbber to show progress while a long task proceeds, e.g unpacking a large distribution tarball instead of just specifying the -v option to the tar command. Still a bit of sugar coating but a bit of fun aswell. (-:

cheers,

Rob

Answer 3

There wasn't really a question so this is less of a standalone answer and more of an extension to Giovanni Funchai's solution. This question is the first google result for "GNU Make Progress" so I ended up here looking for how to do this.

As pointed out by Rob Wells, the solution doesn't work for <10%, but the technique can be extended with the print formatting done by a helper script in whatever language you feel is portable enough for your build. For example, using a python helper script:

echo_progress.py:

"""
Print makefile progress
"""

import argparse
import math
import sys

def main():
  parser = argparse.ArgumentParser(description=__doc__)
  parser.add_argument("--stepno", type=int, required=True)
  parser.add_argument("--nsteps", type=int, required=True)
  parser.add_argument("remainder", nargs=argparse.REMAINDER)
  args = parser.parse_args()

  nchars = int(math.log(args.nsteps, 10)) + 1
  fmt_str = "[{:Xd}/{:Xd}]({:6.2f}%)".replace("X", str(nchars))
  progress = 100 * args.stepno / args.nsteps
  sys.stdout.write(fmt_str.format(args.stepno, args.nsteps, progress))
  for item in args.remainder:
    sys.stdout.write(" ")
    sys.stdout.write(item)
  sys.stdout.write("\n")

if __name__ == "__main__":
  main()

And the modified Makefile:

_mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
I := $(patsubst %/,%,$(dir $(_mkfile_path)))

ifneq ($(words $(MAKECMDGOALS)),1)
.DEFAULT_GOAL = all
%:
    @$(MAKE) $@ --no-print-directory -rRf $(firstword $(MAKEFILE_LIST))
else
ifndef ECHO
T := $(shell $(MAKE) $(MAKECMDGOALS) --no-print-directory \
      -nrRf $(firstword $(MAKEFILE_LIST)) \
      ECHO="COUNTTHIS" | grep -c "COUNTTHIS")
N := x
C = $(words $N)$(eval N := x $N)
ECHO = python $(I)/echo_progress.py --stepno=$C --nsteps=$T
endif

.PHONY: all clean

all: target
    @$(ECHO) All done

clean:
    @rm -f target *.c
#       @$(ECHO) Clean done

target: a.c b.c c.c d.c e.c f.c g.c h.c i.c j.c k.c l.c m.c n.c o.c p.c q.c \
        r.c s.c t.c u.c v.c w.c x.c y.c z.c
    @$(ECHO) Linking $@
    @sleep 0.01
    @touch $@

%.c:
    @$(ECHO) Compiling $@
    @sleep 0.01
    @touch $@

endif

yields:

$ make
[ 1/28](  3.57%) Compiling a.c
[ 2/28](  7.14%) Compiling b.c
[ 3/28]( 10.71%) Compiling c.c
[ 4/28]( 14.29%) Compiling d.c
[ 5/28]( 17.86%) Compiling e.c
[ 6/28]( 21.43%) Compiling f.c
[ 7/28]( 25.00%) Compiling g.c
[ 8/28]( 28.57%) Compiling h.c
[ 9/28]( 32.14%) Compiling i.c
[10/28]( 35.71%) Compiling j.c
[11/28]( 39.29%) Compiling k.c
[12/28]( 42.86%) Compiling l.c
[13/28]( 46.43%) Compiling m.c
[14/28]( 50.00%) Compiling n.c
[15/28]( 53.57%) Compiling o.c
[16/28]( 57.14%) Compiling p.c
[17/28]( 60.71%) Compiling q.c
[18/28]( 64.29%) Compiling r.c
[19/28]( 67.86%) Compiling s.c
[20/28]( 71.43%) Compiling t.c
[21/28]( 75.00%) Compiling u.c
[22/28]( 78.57%) Compiling v.c
[23/28]( 82.14%) Compiling w.c
[24/28]( 85.71%) Compiling x.c
[25/28]( 89.29%) Compiling y.c
[26/28]( 92.86%) Compiling z.c
[27/28]( 96.43%) Linking target
[28/28](100.00%) All done

One could even print a fancy progress bar with unicode characters.

Modified echo_progress.py:

"""
Print makefile progress
"""

import argparse
import math
import sys

def get_progress_bar(numchars, fraction=None, percent=None):
  """
  Return a high resolution unicode progress bar
  """
  if percent is not None:
    fraction = percent / 100.0

  if fraction >= 1.0:
    return "█" * numchars

  blocks = [" ", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█"]
  length_in_chars = fraction * numchars
  n_full = int(length_in_chars)
  i_partial = int(8 * (length_in_chars - n_full))
  n_empty = max(numchars - n_full - 1, 0)
  return ("█" * n_full) + blocks[i_partial] + (" " * n_empty)

def main():
  parser = argparse.ArgumentParser(description=__doc__)
  parser.add_argument("--stepno", type=int, required=True)
  parser.add_argument("--nsteps", type=int, required=True)
  parser.add_argument("remainder", nargs=argparse.REMAINDER)
  args = parser.parse_args()

  nchars = int(math.log(args.nsteps, 10)) + 1
  fmt_str = "\r[{:Xd}/{:Xd}]({:6.2f}%) ".replace("X", str(nchars))
  progress = 100 * args.stepno / args.nsteps
  sys.stdout.write(fmt_str.format(args.stepno, args.nsteps, progress))
  sys.stdout.write(get_progress_bar(20, percent=progress))
  remainder_str = " ".join(args.remainder)
  sys.stdout.write(" {:20s}".format(remainder_str[:20]))
  if args.stepno == args.nsteps:
    sys.stdout.write("\n")

if __name__ == "__main__":
  main()

Which would result in something like this:

$ make clean && make
[12/28]( 42.86%) ███████▊             Compiling k.c

during progress and:

$ make clean && make
[28/28](100.00%) ████████████████████ All done 

upon completion.

Answer 4

This is a slight modification to @GiovanniFunchal's excellent answer.

So I wanted to understand this better and make it work for < 10% so I dug into the documentation and learned more about expr.

# PLACE AT THE TOP OF YOUR MAKEFILE
#---------------------------------
# Progress bar defs
#--------------------------------
#  words = count the number of words
ifneq ($(words $(MAKECMDGOALS)),1) # if no argument was given to make...
.DEFAULT_GOAL = all # set the default goal to all
#  http://www.gnu.org/software/make/manual/make.html
#  $@ = target name
#  %: = last resort recipe
#  --no-print-directory = don't print enter/leave messages for each output grouping
#  MAKEFILE_LIST = has a list of all the parsed Makefiles that can be found *.mk, Makefile, etc
#  -n = dry run, just print the recipes
#  -r = no builtin rules, disables implicit rules
#  -R = no builtin variables, disables implicit variables
#  -f = specify the name of the Makefile
%:                   # define a last resort default rule
      @$(MAKE) $@ --no-print-directory -rRf $(firstword $(MAKEFILE_LIST)) # recursive make call, 
else
ifndef ECHO
#  execute a dry run of make, defining echo beforehand, and count all the instances of "COUNTTHIS"
T := $(shell $(MAKE) $(MAKECMDGOALS) --no-print-directory \
      -nrRf $(firstword $(MAKEFILE_LIST)) \
      ECHO="COUNTTHIS" | grep -c "COUNTTHIS")
#  eval = evaluate the text and read the results as makefile commands
N := x
#  Recursively expand C for each instance of ECHO to count more x's
C = $(words $N)$(eval N := x $N)
#  Multipy the count of x's by 100, and divide by the count of "COUNTTHIS"
#  Followed by a percent sign
#  And wrap it all in square brackets
ECHO = echo -ne "\r [`expr $C '*' 100 / $T`%]"
endif
#------------------
# end progress bar
#------------------

# REST OF YOUR MAKEFILE HERE

#----- Progressbar endif at end Makefile
endif

I got rid of the : '.*\(....\)$$' part. It would return the last 4 characters of the inner expr command, but would fail if it was less than 4. And now it works for sub 10%!

And here is the comment free version:

ifneq ($(words $(MAKECMDGOALS)),1) # if no argument was given to make...
.DEFAULT_GOAL = all # set the default goal to all
%:                   # define a last resort default rule
      @$(MAKE) $@ --no-print-directory -rRf $(firstword $(MAKEFILE_LIST)) # recursive make call, 
else
ifndef ECHO
T := $(shell $(MAKE) $(MAKECMDGOALS) --no-print-directory \
      -nrRf $(firstword $(MAKEFILE_LIST)) \
      ECHO="COUNTTHIS" | grep -c "COUNTTHIS")
N := x
C = $(words $N)$(eval N := x $N)
ECHO = echo -ne "\r [`expr $C '*' 100 / $T`%]"
endif

# ...

endif

Hope that helps.