From: Matt McCutchen Date: Mon, 11 Jun 2007 21:42:14 +0000 (-0400) Subject: Rewrite the Mage basics, temporarily removing the dependency-logging command X-Git-Url: https://mattmccutchen.net/mgear/mgear.git/commitdiff_plain/ff59f3cf9dd552edf18931d41a491b2c6e495c99 Rewrite the Mage basics, temporarily removing the dependency-logging command stuff. Now, all the processing happens in bar and bar.g is just an obfuscated target that checks for changed prerequisites. Added a bunch of comments; read them for details. --- diff --git a/demo/Makefile b/demo/Makefile index 947822e..b8a7ab7 100644 --- a/demo/Makefile +++ b/demo/Makefile @@ -6,10 +6,10 @@ include mage.mk all: -cmd-xgrep = grep 'X' '$<' >'$@' +cmd-xgrep = grep 'X' '$<' >'$t' && echo "Hello world from xgrep" $(call mg-define-rule,%.x,%,$(value cmd-xgrep)) -cmd-addheader = echo 'Header:' | cat - '$<' >'$@' +cmd-addheader = echo 'Header:' | cat - '$<' >'$t' $(call mg-define-rule,%.h,%,$(value cmd-addheader)) # Testing obfuscation diff --git a/experiments/prereq-predict.mk b/experiments/prereq-predict.mk new file mode 100644 index 0000000..8ea4514 --- /dev/null +++ b/experiments/prereq-predict.mk @@ -0,0 +1,32 @@ +.SECONDEXPANSION: + +#dir/x.foo: + +#ding=$(@:.foo=.bar) +# +#%.foo: $$(info got here) /./proc/self/cwd/$$(ding) +# echo "Prerequisites: $^" +# +#%.bar: +# echo "We want $@" + +# $(call save-and-use,/,prereqs) +#save-and-use=$(eval $@=$(if $1,$2,$(foreach p,$2,$(dir $@)$p)))$2 + +pr1=$(info in pr1, $$@ is $@)$(eval $@@gdeps:=$(subst %,$*,$1))$1 +pr2=$(info in pr2, $$@ is $@)$(eval $@@gdeps:=$(addprefix $(dir $@),$(subst %,$*,$1)))$1 + +FORCE: + +define define-rule +$(eval +$1: $(if $(findstring /,$1),$$$$(call pr1,$2),$$$$(call pr2,$2)) FORCE + $$(info +++ $$$$^ is $$^, gdeps is $$($$@@gdeps)) +) +endef + +$(call define-rule,%.foo,%.bar) + +x/%.bar: + $(info +++ Making $@) + diff --git a/src/mage.mk b/src/mage.mk index 3a5181a..d310c0d 100644 --- a/src/mage.mk +++ b/src/mage.mk @@ -2,34 +2,37 @@ # http://www.kepreon.com/~matt/mage/ # Remember the original default goal so we can restore it at the end of mage.mk. -mg-orig-default-goal := $(.DEFAULT_GOAL) +mg-orig-default-goal:=$(.DEFAULT_GOAL) + +# We use second-expansion heavily to dynamically compute prerequisites when they +# are needed. Second-expansion lets us follow make's implicit rule search +# instead of trying to anticipate which prerequisites it will need in advance. +.SECONDEXPANSION: + # TEXT UTILITIES -empty := -bs := \$(empty) +empty:= +bs:=\$(empty) +pct:=% define nl endef # Shell-quote: a'b => 'a'\''b' -sq = '$(subst ','\'',$1)' +sq='$(subst ','\'',$1)' # Make-quote: a$\nb => a$$$(nl)b # This is enough to assign the value, *but not to use it as an argument!* -mqas = $(subst $(nl),$$(nl),$(subst $$,$$$$,$1)) +mqas=$(subst $(nl),$$(nl),$(subst $$,$$$$,$1)) # Return nonempty if the strings are equal, empty otherwise. # If the strings have only nice characters, you can do $(filter x$1,x$2). -streq = $(findstring x$1,$(findstring x$2,x$1)) +streq=$(findstring x$1,$(findstring x$2,x$1)) # $(call fmt-make-assignment,foo,bar) # Output a make assignment that stores bar in variable foo. # The result is like `foo:=bar' but handles leading spaces, $, and # newlines appearing in bar safely. -fmt-make-assignment = $1:=$$(empty)$(call mqas,$2) +fmt-make-assignment=$1:=$$(empty)$(call mqas,$2) -# We use second-expansion heavily to dynamically compute prerequisites when they -# are needed. Second-expansion lets us follow make's implicit rule search -# instead of trying to anticipate which prerequisites it will need in advance. -.SECONDEXPANSION: # TARGET OBFUSCATION @@ -56,34 +59,82 @@ aname:=/proc/self/cwd/ nextoid:=/./. newoid=$(nextoid)$(eval nextoid:=$(subst /.//////,//./,$(patsubst /.//////%,/././%,$(nextoid:.=/.)))) -# High-priority implicit rule for obfuscated targets. Works for both always- -# exists and alternate-name targets. +# Target-specific variables for obfuscated targets. $(opfx)%: oid=$(word 1,$(subst $(aname), ,$@)) $(opfx)%: otgt=$(word 2,$(subst $(aname), ,$@)) + +# High-priority implicit rule for obfuscated targets. Works for both always- +# exists and alternate-name targets. $(opfx)%: $$($$(oid)@opr) $($(oid)@ocmd) +# Make won't use the same implicit rule more than once on its stack, so provide +# a second copy of the rule to allow two obfuscated targets on the stack. +# Needed for $(mg-genfile-oid)$(aname)bar.g <- $(mg-scout-oid)$(aname)bar . +# The prerequisites must look different before second-expansion so the second +# rule isn't discarded as a duplicate. +$(opfx)%: $$(empty) $$($$(oid)@opr) + $($(oid)@ocmd) + # MAIN BUILD LOGIC -# Target metadata variables for bar: (*: stored in bar.g) +# DESIGN: +# +# - To each generated file bar corresponds a "genfile" bar.g that contains some +# information about how bar was generated, including the command (for rebuild on +# command change) and the warnings (for replay). +# +# - bar.g's mtime is the last time Mage verified that bar was up to date. bar +# needs to be regenerated iff a prerequisite is newer than *bar.g* (not bar). +# If a prerequisite changes but the command gives the same contents for bar, +# bar.g is touched but bar itself is not touched because files that depend on it +# need not be regenerated. +# +# - If bar is newer than bar.g, the user has overridden it, and we should leave +# the override in place but warn the user about it. +# +# - The user's Makefile defines Mage rules by calling mg-define-rule. Each Mage +# rule becomes one underlying make rule with the same target and a little extra +# magic. This is important so that all implicit rule competition takes place at +# the same target. Additionally, the prerequisites are passed to an obfuscated +# target for bar.g to see if any are newer than bar.g. This is done by the +# single obfuscated implicit rule, keeping the number of implicit rules low. +# Then the command script for bar checks for overrides, command change, +# prerequisite change, etc. and acts accordingly. +# +# Target metadata variables for bar: (* means stored in bar.g) # # bar@cmd:=cat foo >bar.tmp # Generation command as given to the shell.* +# # bar@warnings:=$(empty)yikes! # Stuff the command printed to stdout or stderr.* +# # bar@deps:=included@x oops@ # If dependency-logging, list of filename@revision used. Revision is x for # exists and empty for doesn't exist. Later perhaps x will be the mtime.* +# # bar@gloaded:=1 # Set if Mage has loaded bar.g and hasn't changed it since then. +# +# bar@gdeps:=foo +# Static dependencies of bar, for checking by bar.g. +# +# bar@gq:=foo $(mg-scout-oid)$(aname)bar +# $? from the rule for bar.g; used by the rule for bar. +# # bar@uptodate:=1 # Set when Mage makes a file or depends on it being made. Used to determine # the next prerequisite to check for a dependency-logging command. -# bar@generated:=1 -# Set when Mage generates a file; suppresses warning replay. # -# If the rule for bar is overridden, we clear the information from bar.g so that -# it is as if bar.g didn't exist. +## If the rule for bar is overridden, we clear the information from bar.g so +## that it is as if bar.g didn't exist. -- Not currently needed +# +# Some make features with which Mage's compatibility has not been investigated: +# - Command-line options (especially --dry-run, --question, --touch, +# --always-make, --keep-going, --jobs, --assume-old, and --assume-new) +# - Static pattern rules +# - Vpath # $(call gload,foo.o) # Make sure foo.o's genfile, if any, has been loaded. @@ -97,79 +148,71 @@ $1@gloaded:=1 )) endef -# $(call mg-translate-cmd,touch $$@) -# Translates $@, $<, $^, $+, $* to Mage-ized variables if necessary. -# We won't support $%, $?, $|. -# There might be false matches, e.g., $$@ => $$(mg@) ; -# to prevent that, do $$$(empty)@ . +# bar.g: scout bar and its dependencies and store $?. When implicit rules +# compete for bar, we depend on the rule make uses being the last one it +# second-expands so that $(bar@gdeps) is still correct. +mg-genfile-oid:=$(newoid) +mg-scout-oid:=$(newoid) +$(mg-genfile-oid)@opr=$($(otgt:.g=)@gdeps) $(if $(wildcard $(otgt:.g=)),$(mg-scout-oid)$(aname)$(otgt:.g=),) +$(mg-genfile-oid)@ocmd=$(eval $(otgt:.g=)@gq:=$?) + +# Mage-ized automatic variables. +# $@: needs no translation +# NOTE: $(mg@) is *eventual* target. Commands must write to temp file, $t . +# $%: haven't thought about it much, but probably needs no translation +mg< = $(firstword $(mg^)) +mg? = $(filter-out $(opfx)%,$($@@gq)) +mg^ = $(filter-out MG-% $(opfx)%,$^) +mg+ = $(filter-out MG-% $(opfx)%,$+) +# $|: needs no translation +# $*: needs no translation + +# $(call mg-translate-cmd,cat $$< >$$t) +# Replaces references to automatic variables with references to their Mage-ized +# counterparts. There might be false matches, e.g., $$@ => $$(mg@) ; +# to prevent that, write $$$(empty)@ instead. (c.f. autoconf empty quadrigraph) define mg-translate-cmd -$(subst $$@,$$(mg@),$(subst $$<,$$(mg<),$(subst $$^,$$(mg^),$(subst $$+,$$(mg+),$1)))) +$(subst $$?,$$(mg?),$(subst $$<,$$(mg<),$(subst $$^,$$(mg^),$(subst $$+,$$(mg+),$1)))) endef -# $(call mg-define-rule,target,prerequisite,cmd) -# Defines a rule. -# If cmd uses $@, quote if necessary so this function sees $@, etc. -# -# When we generate bar from foo using a Mage rule: -# Division of work: -# - "bar.g: foo" generates bar if necessary. -# - "bar: bar.g" loads new genfile and replays warnings if bar.g was -# already up to date. -# Cases: -# - bar is managed and up to date => replay -# - bar doesn't exist or is managed and out of date => generate -# - bar is unmanaged => warn about override +# $(call prereq-predict,target,prerequisites) +# Expands to code that does the following at second-expansion time: +# 1. Computes the actual prerequisites from the given prerequisite patterns by +# translating % to $* and prepending a directory if appropriate. +# 2. Saves both existing ($+) and newly computed prerequisites to $(bar@gdeps) +# where the rule for bar.g can get them. +# Factor out set-gdeps because make gets confused if the macro for the +# prerequisites has a colon. Make replaces the first % with the stem *before +# expansion*, so use $(pct) to protect some %s. +set-gdeps=$(eval $@@gdeps:=$1) +prereq-predict=$$$$(call set-gdeps,$$$$+ $(if $(findstring /,$1),$$$$(subst $$$$(pct),$$$*,$2),$$$$(addprefix $$$$(dir $$$$@),$$$$(subst $$$$(pct),$$$$*,$2)))) + +# Used to make the rule for bar always run so we can act based on $(bar@gdeps) +# and check for command change. MG-FORCE: .PHONY: MG-FORCE -mg-scout-oid:=$(newoid) + +# $(call mg-define-rule,target,prerequisites,cmd) +# Defines a rule. cmd is expanded again when it is run, at which time +# Mage-ized automatic variables are available. define mg-define-rule $(eval -# Define some target-specific variables. -# It might look like we could use $*, but $1 most likely isn't %. -$1.g: target = $$(@:.g=) -$1.g: cmd = $(call mg-translate-cmd,$3) -$1.g: mg@ = $$(target).tmp -$1.g: mg^ = $$(filter-out MG-% $$(opfx)%,$$^) -$1.g: mg+ = $$(filter-out MG-% $$(opfx)%,$$+) -$1.g: mg< = $$(firstword $$(mg^)) - -# Rule for the genfile. Evidently all the prerequisites we want second-expanded -# have to go on the same rule. -$1.g: $2 $$$$(mg-scout-target) MG-FORCE - $$(mg-generate) - -$(mg-file-from-genfile) -) -endef - -define mg-file-from-genfile -# If the file was regenerated, load the new genfile. -# If not, replay any warnings. -# HMMM Maybe errors should go to stderr??? -$1: MG-FORCE | $1.g - $$(call gload,$$@) - $$(eval $$@@uptodate:=1) - $$(if $$($$@@warnings),$$(if $$($$@@changed),,$$(info $$($$@@cmd) # warning replay)$$(info $$($$@@warnings))),) -endef - -# If the target is unmanaged, we must run the rule; we'll see that the -# obfuscated target is in $? and complain. -mg-scout-target=$(if $(wildcard $(target)),$(mg-scout-oid)$(aname)$(target),) - -# If the command changed, we must regenerate. -mg-check-cmd=$(if $(call streq,$(cmd),$($(target)@cmd)),,x) +# Store the new command. +$1: cmd=$(call mg-translate-cmd,$3) +# Provide variable for the temporary file. FIX +$1: t=$$@.tmp -# Just add additional prerequisites. -# I don't think this can add patterns to implicit rules, but it should be able -# to add specific prerequisites to uses of implicit rules. -# TODO Make this work for dependency-logging commands if desired. -define mg-define-prereq -$(eval -$1.g: $2 +# Rule for the target. Set $(bar@gdeps) to all prerequisites. Apply the new +# prerequisites. The command script always runs. Depend on bar.g to get +# $(bar@gq). +$1: $(call prereq-predict,$1,$2) $2 MG-FORCE $(mg-genfile-oid)$(aname)$$$$@.g + $$(mg-rule-cmd) ) endef +# TODO Provide a way to define static pattern rules. + # Procedure to generate bar. Remember, $@ is bar.g. # If an override: # 1. Complain. (Should we stop "`bar' is up to date" using @:; ?) @@ -190,62 +233,39 @@ endef # 7. Now that bar is known to be up to date, move in the new bar.g. # 8. Defensively remove both temporary files and exit with the exit code # from before the removal. (trap EXIT) -define mg-generate - $(call gload,$(target))\ - $(if $(filter $(mg-scout-oid)$(aname)$(target),$?),\ - $(info mage: warning: Manually created/modified file at $(target) overrides rule.)\ - $(eval $(target)@cmd:=)$(eval $(target)@warnings:=)$(eval $(target)@deps:=)\ - ,\ - $(if $?$(if $(wildcard $(target)),,x)$(mg-check-cmd),\ - $(eval $(target)@gloaded:=)$(eval $(target)@changed:=1)$(info $(cmd))\ - @trap 'rm -f $(target).tmp $@.tmp' EXIT &&\ - exec 3>$@.tmp && $(mg-assign-cmd) >&3 &&\ - set -o pipefail && { $(mg-run-cmd) | tee /dev/fd/4 | $(mg-wrap-warnings) >&3; } 4>&1 &&\ - $(mg-maybe-move-target) && mv -f $@.tmp $@\ - )) -endef - -# Pieces of mg-generate that I factored out to make mg-generate more readable. -mg-assign-cmd=echo $(call sq,$(call fmt-make-assignment,$(target)@cmd,$(cmd))) -mg-run-cmd={ ($(cmd)) 2>&1 && { [ -r $(target).tmp ] || { echo 'mage: error: Command for $(target) succeeded without creating it!'; false; }; }; } -mg-wrap-warnings=sed -re '1s/^/$(target)@warnings:=$$(empty)/; 1!s/^/$(target)@warnings+=$$(nl)/' -mg-maybe-move-target={ cmp -s $(target) $(target).tmp || echo >$@ && mv -f $(target).tmp $(target); } - -# $(call mg-rule,target,static prerequisites,cmd,[dep converter]) -# Defines a rule with a dependency-logging command. -# I haven't decided on the format for the dep converter yet. -dlc-static-run-oid:=$(newoid) -define mg-define-rule-dlc -$(eval - -# Copied stuff from mg-define-rule to modify as necessary - -## Define some target-specific variables. -## It might look like we could use $*, but $1 most likely isn't %. -#$1.g: target = $$(@:.g=) -#$1.g: cmd = $(call mg-translate-cmd,$3) -#$1.g: mg@ = $$(target).tmp -#$1.g: mg^ = $$(filter-out MG-% /./%,$$^) -#$1.g: mg+ = $$(filter-out MG-% /./%,$$+) -#$1.g: mg< = $$(firstword $$(mg^)) - -# -$1.g: MG-FORCE | $$$$(dlc-first-run-oid)$$$$(aname)$1.g $$$$(call dlc-setup-tgt,$(newoid)) - -## Rule for the genfile. Evidently all the prerequisites we want second-expanded -## have to go on the same rule. -#$1.g: $2 $$$$(mg-scout-target) MG-FORCE -# $$(mg-generate) - - -$(mg-file-from-genfile) -) +# If not: +# 1. If there were warnings, replay them. (HMMM To stderr?) +# HMMM .tmp in displayed command looks ugly +define mg-rule-cmd + $(if $(filter $(mg-scout-oid)$(aname)$@,$($@@gq)),\ + $(info Mage: warning: Manually created/modified file at $@ overrides rule.)\ + ,$(call gload,$@)$(if $($@@gq)$(if $(wildcard $@),,x)$(mg-check-cmd),\ + $(eval $@@gloaded:=)$(info $(cmd))\ + @trap 'rm -f $@.tmp $@.g.tmp' EXIT &&\ + exec 3>$@.g.tmp && $(mg-assign-cmd) >&3 &&\ + set -o pipefail && { $(mg-run-cmd) | tee /dev/fd/4 | $(mg-wrap-warnings) >&3; } 4>&1 &&\ + $(mg-maybe-move-target) && mv -f $@.g.tmp $@.g\ + ,$(if $($@@warnings),\ + $(info $($@@cmd) # Mage warning replay$(nl)$($@@warnings))\ + ))) endef -define dlc-setup-tgt -$(eval +# If the command changed, we must regenerate. +mg-check-cmd=$(if $(call streq,$(cmd),$($@@cmd)),,x) -)$1 +# Pieces of mg-generate that I factored out to make mg-generate more readable. +mg-assign-cmd=echo $(call sq,$(call fmt-make-assignment,$@@cmd,$(cmd))) +mg-run-cmd={ ($(cmd)) 2>&1 && { [ -r $@.tmp ] || { echo 'Mage: error: Command for $@ succeeded without creating it!'; false; }; }; } +mg-wrap-warnings=sed -re '1s/^/$@@warnings:=$$(empty)/; 1!s/^/$@@warnings+=$$(nl)/' +mg-maybe-move-target={ cmp -s $@ $@.tmp || echo >$@.g && mv -f $@.tmp $@; } + +# Just add additional prerequisites. This cannot add prerequisite patterns to +# an implicit rule, but it can add specific prerequisites to an individual use +# of an implicit rule. Currently, Mage picks up the target's prerequisites from +# make, so this just attaches the given prerequisites to the target, but the +# implementation might change in the future. +define mg-define-prereq +$(eval $1: $2) endef # END