lkml.org 
[lkml]   [2008]   [Jun]   [8]   [last100]   RSS Feed
Views: [more markup]  [less markup]  [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    DateSun, 8 Jun 2008 11:47:30 +0200
    FromSam Ravnborg <>
    Subject[PATCH] Speed up "make headers_*"
    This is just a heads up patch if anyone is interested.
    I finally took the time needed to optimize the
    make headers_* targets.
    
    On my box it now takes less than 10 seconds to run
    the full install + check cycle.
    And it generates roughtly one screen full of output.
    
    Compare that to ~31 seconds and output filling up
    my scroll back buffer.
    
    This fixes a long standing complaint from Linus that this is
    too simple to warrant such long execution time - yet
    there are room for improvements.
    
    Note that this patchset removes support for ALTARCH - only remaining
    user is sparc and patches are pending to remove ALTARCH usage form sparc too.
    
    I will do a proper split-up of the patch later and add it
    to kbuild-next.git.
    
    Comments (especially to the perl scripts) are welcome.
    
    	Sam
    
     b/Makefile                     |   41 ++++---
     b/scripts/Makefile.headersinst |  218 ++++++++++++-----------------------------
     b/scripts/headers_check.pl     |   48 +++++++++
     b/scripts/headers_install.pl   |   39 +++++++
     scripts/hdrcheck.sh            |   10 -
     5 files changed, 176 insertions(+), 180 deletions(-)
    diff --git a/Makefile b/Makefile
    index 8db70fe..8f8fd6f 100644
    --- a/Makefile
    +++ b/Makefile
    @@ -996,36 +996,45 @@ depend dep:
     
     # ---------------------------------------------------------------------------
     # Kernel headers
    -INSTALL_HDR_PATH=$(objtree)/usr
    -export INSTALL_HDR_PATH
     
    -HDRFILTER=generic i386 x86_64
    -HDRARCHES=$(filter-out $(HDRFILTER),$(patsubst $(srctree)/include/asm-%/Kbuild,%,$(wildcard $(srctree)/include/asm-*/Kbuild)))
    +#Default location for installed headers
    +export INSTALL_HDR_PATH = $(objtree)/usr
     
    -PHONY += headers_install_all
    -headers_install_all: include/linux/version.h scripts_basic FORCE
    +hdr-filter := generic um ppc
    +hdr-archs  := $(filter-out $(hdr-filter),                           \
    +                  $(patsubst $(srctree)/include/asm-%/Kbuild,%,     \
    +                      $(wildcard $(srctree)/include/asm-*/Kbuild)))
    +hdr-inst := -rR -f $(srctree)/scripts/Makefile.headersinst obj
    +
    +PHONY += __headers
    +__headers: include/linux/version.h scripts_basic FORCE
     	$(Q)$(MAKE) $(build)=scripts scripts/unifdef
    -	$(Q)for arch in $(HDRARCHES); do \
    -	 $(MAKE) ARCH=$$arch -f $(srctree)/scripts/Makefile.headersinst obj=include BIASMDIR=-bi-$$arch ;\
    +
    +PHONY += headers_install_all
    +headers_install_all: __headers
    +	$(Q)set -e; for arch in $(hdr-archs); do \
    +	 $(MAKE) ARCH=$$arch SRCARCH=$$arch $(hdr-inst)=include \
    +	         BIASMDIR=-bi-$$arch ;\
     	 done
     
     PHONY += headers_install
    -headers_install: include/linux/version.h scripts_basic FORCE
    -	@if [ ! -r $(srctree)/include/asm-$(SRCARCH)/Kbuild ]; then \
    +headers_install: __headers
    +	$(Q)if [ ! -r $(srctree)/include/asm-$(SRCARCH)/Kbuild ]; then \
     	  echo '*** Error: Headers not exportable for this architecture ($(SRCARCH))'; \
    -	  exit 1 ; fi
    -	$(Q)$(MAKE) $(build)=scripts scripts/unifdef
    -	$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.headersinst ARCH=$(SRCARCH) obj=include
    +	  exit 1 ; \
    +	fi
    +	$(Q)$(MAKE) $(hdr-inst)=include ARCH=$(SRCARCH)
     
     PHONY += headers_check_all
     headers_check_all: headers_install_all
    -	$(Q)for arch in $(HDRARCHES); do \
    -	 $(MAKE) ARCH=$$arch -f $(srctree)/scripts/Makefile.headersinst obj=include BIASMDIR=-bi-$$arch HDRCHECK=1 ;\
    +	$(Q)set -e; for arch in $(hdr-archs); do \
    +	 $(MAKE) ARCH=$$arch SRCARCH=$$arch $(hdr-inst)=include \
    +	         BIASMDIR=-bi-$$arch HDRCHECK=1 ;\
     	 done
     
     PHONY += headers_check
     headers_check: headers_install
    -	$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.headersinst ARCH=$(SRCARCH) obj=include HDRCHECK=1
    +	$(Q)$(MAKE) $(hdr-inst)=include ARCH=$(SRCARCH) HDRCHECK=1
     
     # ---------------------------------------------------------------------------
     # Modules
    diff --git a/scripts/Makefile.headersinst b/scripts/Makefile.headersinst
    index 53dae3e..80df6f3 100644
    --- a/scripts/Makefile.headersinst
    +++ b/scripts/Makefile.headersinst
    @@ -7,188 +7,98 @@
     #
     # ==========================================================================
     
    -UNIFDEF := scripts/unifdef -U__KERNEL__
    -
    -# Eliminate the contents of (and inclusions of) compiler.h
    -HDRSED  := sed 	-e "s/ inline / __inline__ /g" \
    -		-e "s/[[:space:]]__user[[:space:]]\{1,\}/ /g" \
    -		-e "s/(__user[[:space:]]\{1,\}/ (/g" \
    -		-e "s/[[:space:]]__force[[:space:]]\{1,\}/ /g" \
    -		-e "s/(__force[[:space:]]\{1,\}/ (/g" \
    -		-e "s/[[:space:]]__iomem[[:space:]]\{1,\}/ /g" \
    -		-e "s/(__iomem[[:space:]]\{1,\}/ (/g" \
    -		-e "s/[[:space:]]__attribute_const__[[:space:]]\{1,\}/\ /g" \
    -		-e "s/[[:space:]]__attribute_const__$$//" \
    -		-e "/^\#include <linux\/compiler.h>/d"
    -
     _dst := $(if $(dst),$(dst),$(obj))
     
    -ifeq (,$(patsubst include/asm/%,,$(obj)/))
    -# For producing the generated stuff in include/asm for biarch builds, include
    -# both sets of Kbuild files; we'll generate anything which is mentioned in
    -# _either_ arch, and recurse into subdirectories which are mentioned in either
    -# arch. Since some directories may exist in one but not the other, we must
    -# use $(wildcard...). 
    -GENASM := 1
    -archasm	   := $(subst include/asm,asm-$(ARCH),$(obj))
    -altarchasm := $(subst include/asm,asm-$(ALTARCH),$(obj))
    -KBUILDFILES := $(wildcard $(srctree)/include/$(archasm)/Kbuild $(srctree)/include/$(altarchasm)/Kbuild)
    -else
    -KBUILDFILES := $(srctree)/$(obj)/Kbuild
    -endif
    +kbuild-file := $(srctree)/$(obj)/Kbuild
    +include $(kbuild-file)
     
    -include $(KBUILDFILES)
    +include scripts/Kbuild.include
     
    -include scripts/Kbuild.include 
    -
    -# If this is include/asm-$(ARCH) and there's no $(ALTARCH), then
    -# override $(_dst) so that we install to include/asm directly.
    +# If this is include/asm-$(ARCH) then override $(_dst) so that
    +# we install to include/asm directly.
     # Unless $(BIASMDIR) is set, in which case we're probably doing
     # a 'headers_install_all' build and we should keep the -$(ARCH)
     # in the directory name.
    -ifeq ($(obj)$(ALTARCH),include/asm-$(ARCH)$(BIASMDIR))
    +ifeq ($(obj),include/asm-$(ARCH)$(BIASMDIR))
          _dst := include/asm
     endif
     
    -header-y	:= $(sort $(header-y))
    -unifdef-y	:= $(sort $(unifdef-y))
    -subdir-y	:= $(patsubst %/,%,$(filter %/, $(header-y)))
    -header-y	:= $(filter-out %/, $(header-y))
    -header-y	:= $(filter-out $(unifdef-y),$(header-y))
    +install      := $(INSTALL_HDR_PATH)/$(_dst)
     
    -# stamp files for header checks
    -check-y		:= $(patsubst %,.check.%,$(header-y) $(unifdef-y) $(objhdr-y))
    +header-y     := $(sort $(header-y) $(unifdef-y))
    +subdirs      := $(patsubst %/,%,$(filter %/, $(header-y)))
    +header-y     := $(filter-out %/, $(header-y))
     
    -# Work out what needs to be removed
    -oldheaders	:= $(patsubst $(INSTALL_HDR_PATH)/$(_dst)/%,%,$(wildcard $(INSTALL_HDR_PATH)/$(_dst)/*.h))
    -unwanted	:= $(filter-out $(header-y) $(unifdef-y) $(objhdr-y),$(oldheaders))
    +# files used to track state of install/check
    +install-file := $(install)/.install
    +check-file   := $(install)/.check
    +check-dep    := $(install)/.check.d
     
    -oldcheckstamps	:= $(patsubst $(INSTALL_HDR_PATH)/$(_dst)/%,%,$(wildcard $(INSTALL_HDR_PATH)/$(_dst)/.check.*.h))
    -unwanted	+= $(filter-out $(check-y),$(oldcheckstamps))
    +# all headers files for this dir
    +all-files    := $(header-y) $(objhdr-y)
     
    -# Prefix them all with full paths to $(INSTALL_HDR_PATH)
    -header-y 	:= $(patsubst %,$(INSTALL_HDR_PATH)/$(_dst)/%,$(header-y))
    -unifdef-y 	:= $(patsubst %,$(INSTALL_HDR_PATH)/$(_dst)/%,$(unifdef-y))
    -objhdr-y 	:= $(patsubst %,$(INSTALL_HDR_PATH)/$(_dst)/%,$(objhdr-y))
    -check-y 	:= $(patsubst %,$(INSTALL_HDR_PATH)/$(_dst)/%,$(check-y))
    +# Work out what needs to be removed
    +oldheaders   := $(patsubst $(install)/%,%,$(wildcard $(install)/*.h))
    +unwanted     := $(filter-out $(all-files),$(oldheaders))
     
    +# Prefix all with full paths to $(INSTALL_HDR_PATH)
    +header-y     := $(addprefix $(install)/, $(header-y))
    +objhdr-y     := $(addprefix $(install)/, $(objhdr-y))
    +unwanted     := $(addprefix $(install)/, $(unwanted))
     
    -ifdef ALTARCH
    -ifeq ($(obj),include/asm-$(ARCH))
    -altarch-y	:= altarch-dir
    -endif
    -endif
    +printdir = $(patsubst $(INSTALL_HDR_PATH)/%/,%,$(dir $@))
     
    -# Make the definitions visible for recursive make invocations
    -export ALTARCH
    -export ARCHDEF
    -export ALTARCHDEF
    -
    -quiet_cmd_o_hdr_install   = INSTALL $(patsubst $(INSTALL_HDR_PATH)/%,%,$@)
    -      cmd_o_hdr_install   = cp $(patsubst $(INSTALL_HDR_PATH)/$(_dst)/%,$(objtree)/$(obj)/%,$@) \
    -			    $(INSTALL_HDR_PATH)/$(_dst)
    -
    -quiet_cmd_headers_install = INSTALL $(patsubst $(INSTALL_HDR_PATH)/%,%,$@)
    -      cmd_headers_install = $(HDRSED) $(patsubst $(INSTALL_HDR_PATH)/$(_dst)/%,$(srctree)/$(obj)/%,$@)	\
    -			    > $@
    -
    -quiet_cmd_unifdef	  = UNIFDEF $(patsubst $(INSTALL_HDR_PATH)/%,%,$@)
    -      cmd_unifdef	  = $(UNIFDEF) $(patsubst $(INSTALL_HDR_PATH)/$(_dst)/%,$(srctree)/$(obj)/%,$@) \
    -				   | $(HDRSED) > $@ || :
    -
    -quiet_cmd_check		  = CHECK   $(patsubst $(INSTALL_HDR_PATH)/$(_dst)/.check.%,$(_dst)/%,$@)
    -      cmd_check		  = $(CONFIG_SHELL) $(srctree)/scripts/hdrcheck.sh \
    -                              $(INSTALL_HDR_PATH)/include $(subst /.check.,/,$@) $@
    -
    -quiet_cmd_remove	  = REMOVE  $(_dst)/$@
    -      cmd_remove	  = rm -f $(INSTALL_HDR_PATH)/$(_dst)/$@
    -
    -quiet_cmd_mkdir		  = MKDIR   $(patsubst $(INSTALL_HDR_PATH)/%,%,$@)
    -      cmd_mkdir		  = mkdir -p $@
    -
    -quiet_cmd_gen		  = GEN     $(patsubst $(INSTALL_HDR_PATH)/%,%,$@)
    -      cmd_gen		  = \
    -FNAME=$(patsubst $(INSTALL_HDR_PATH)/$(_dst)/%,%,$@);			\
    -STUBDEF=__ASM_STUB_`echo $$FNAME | tr a-z.- A-Z__`;			\
    -(echo "/* File autogenerated by 'make headers_install' */" ;		\
    -echo "\#ifndef $$STUBDEF" ;						\
    -echo "\#define $$STUBDEF" ;						\
    -echo "\# if $(ARCHDEF)" ;						\
    -if [ -r $(subst /$(_dst)/,/include/$(archasm)/,$@) ]; then		\
    -	echo "\#  include <$(archasm)/$$FNAME>" ;			\
    -else									\
    -	echo "\#  error $(archasm)/$$FNAME does not exist in"		\
    -			"the $(ARCH) architecture" ;			\
    -fi ;									\
    -echo "\# elif $(ALTARCHDEF)" ;						\
    -if [ -r $(subst /$(_dst)/,/include/$(altarchasm)/,$@) ]; then		\
    -	echo "\#  include <$(altarchasm)/$$FNAME>" ;			\
    -else									\
    -	echo "\#  error $(altarchasm)/$$FNAME does not exist in"	\
    -			"the $(ALTARCH) architecture" ;			\
    -fi ;									\
    -echo "\# else" ;							\
    -echo "\#  warning This machine appears to be"				\
    -		 "neither $(ARCH) nor $(ALTARCH)." ;			\
    -echo "\# endif" ;							\
    -echo "\#endif /* $$STUBDEF */" ;					\
    -) > $@
    -
    -.PHONY: __headersinst __headerscheck
    +quiet_cmd_install = INSTALL $(printdir) ($(words $(all-files)) files)
    +      cmd_install = $(PERL) $(srctree)/scripts/headers_install.pl \
    +                            $(obj) $(install) $(all-files)
     
    -ifdef HDRCHECK
    -__headerscheck: $(subdir-y) $(check-y)
    -	@true
    +quiet_cmd_remove = REMOVE  $(patsubst $(INSTALL_HDR_PATH)/%,%,$(unwanted))
    +      cmd_remove = rm -f $(unwanted)
    +
    +quiet_cmd_check = CHECK   $(printdir) ($(words $(all-files)) files)
    +      cmd_check = $(PERL) $(srctree)/scripts/headers_check.pl \
    +                  $(INSTALL_HDR_PATH)/include                 \
    +                  $(addprefix $(install)/, $(all-files))
     
    -$(check-y) : $(INSTALL_HDR_PATH)/$(_dst)/.check.%.h : $(INSTALL_HDR_PATH)/$(_dst)/%.h 
    -	$(call cmd,check)
    +PHONY += __headersinst __headerscheck
     
    -# Other dependencies for $(check-y)
    -include /dev/null $(wildcard $(check-y))
    +ifdef HDRCHECK
    +__headerscheck: $(subdirs) $(check-file)
    +	@:
     
    -# ... but leave $(check-y) as .PHONY for now until those deps are actually correct.
    -.PHONY: $(check-y)
    +targets += $(check-file)
    +$(check-file): $(addprefix $(srctree)/$(obj)/,$(all-files)) FORCE
    +	$(call if_changed,check)
    +	$(Q)touch $@
     
     else
     # Rules for installing headers
    -__headersinst: $(subdir-y) $(header-y) $(unifdef-y) $(altarch-y) $(objhdr-y)
    -	@true
    -
    -$(objhdr-y) $(subdir-y) $(header-y) $(unifdef-y): | $(INSTALL_HDR_PATH)/$(_dst) $(unwanted)
    -
    -$(INSTALL_HDR_PATH)/$(_dst):
    -	$(call cmd,mkdir)
    +__headersinst: $(subdirs) $(install-file)
    +	@:
     
    -.PHONY: $(unwanted)
    -$(unwanted):
    -	$(call cmd,remove)
    +targets += $(install-file)
    +$(install-file): $(addprefix $(srctree)/$(obj)/,$(all-files)) FORCE
    +	$(if $(unwanted),$(call cmd,remove),)
    +	$(if $(wildcard $(dir $@)),,$(shell mkdir -p $(dir $@)))
    +	$(call if_changed,install)
    +	$(Q)touch $@
     
    -ifdef GENASM
    -$(objhdr-y) $(header-y) $(unifdef-y): $(KBUILDFILES)
    -	$(call cmd,gen)
    +endif
     
    -else
    -$(objhdr-y) :		$(INSTALL_HDR_PATH)/$(_dst)/%.h: $(objtree)/$(obj)/%.h $(KBUILDFILES)
    -	$(call cmd,o_hdr_install)
    +# Recursion
    +hdr-inst := -rR -f $(srctree)/scripts/Makefile.headersinst obj
    +.PHONY: $(subdirs)
    +$(subdirs):
    +	$(Q)$(MAKE) $(hdr-inst)=$(obj)/$@ dst=$(_dst)/$@
     
    -$(header-y) :		$(INSTALL_HDR_PATH)/$(_dst)/%.h: $(srctree)/$(obj)/%.h $(KBUILDFILES)
    -	$(call cmd,headers_install)
    +targets := $(wildcard $(sort $(targets)))
    +cmd_files := $(wildcard \
    +             $(foreach f,$(targets),$(dir $(f)).$(notdir $(f)).cmd))
     
    -$(unifdef-y) :		$(INSTALL_HDR_PATH)/$(_dst)/%.h: $(srctree)/$(obj)/%.h $(KBUILDFILES)
    -	$(call cmd,unifdef)
    -endif
    +ifneq ($(cmd_files),)
    +	include $(cmd_files)
     endif
     
    -hdrinst := -rR -f $(srctree)/scripts/Makefile.headersinst obj
    -
    -.PHONY: altarch-dir
    -# All the files in the normal arch dir must be created first, since we test
    -# for their existence.
    -altarch-dir: $(subdir-y) $(header-y) $(unifdef-y) $(objhdr-y)
    -	$(Q)$(MAKE) $(hdrinst)=include/asm-$(ALTARCH) dst=include/asm-$(ALTARCH)
    -	$(Q)$(MAKE) $(hdrinst)=include/asm dst=include/asm$(BIASMDIR)
    -
    -# Recursion
    -.PHONY: $(subdir-y)
    -$(subdir-y):
    -	$(Q)$(MAKE) $(hdrinst)=$(obj)/$@ dst=$(_dst)/$@ rel=../$(rel)
    +.PHONY: $(PHONY)
    +PHONY += FORCE
    +FORCE: ;
    diff --git a/scripts/hdrcheck.sh b/scripts/hdrcheck.sh
    deleted file mode 100755
    index 3159858..0000000
    --- a/scripts/hdrcheck.sh
    +++ /dev/null
    @@ -1,10 +0,0 @@
    -#!/bin/sh
    -
    -for FILE in `grep '^[ \t]*#[ \t]*include[ \t]*<' $2 | cut -f2 -d\< | cut -f1 -d\> | egrep ^linux\|^asm` ; do
    -    if [ ! -r $1/$FILE ]; then
    -	echo $2 requires $FILE, which does not exist in exported headers
    -	exit 1
    -    fi
    -done
    -# FIXME: List dependencies into $3
    -touch $3
    diff --git a/scripts/headers_check.pl b/scripts/headers_check.pl
    new file mode 100644
    index 0000000..dfb2ec7
    --- /dev/null
    +++ b/scripts/headers_check.pl
    @@ -0,0 +1,48 @@
    +#!/usr/bin/perl
    +#
    +# headers_check.pl execute a number of trivial consistency checks
    +#
    +# Usage: headers_check.pl dir [files...]
    +# dir..:  dir to look for included files
    +# files:  list of files to check
    +#
    +# The script reads the supplied files line by line and:
    +#
    +# 1) for each include statement it checks if the
    +#    included file actually exists.
    +#    Only include files located in asm* and linux* are checked.
    +#    The rest are assumed to be system include files.
    +#
    +# 2) TODO
    +
    +my ($dir, @files) = @ARGV;
    +
    +my $ret = 0;
    +my $lineno = 0;
    +my $filename;
    +
    +foreach $file (@files) {
    +	$filename = $file;
    +	open(FILE, "< $filename") or die "$filename: $!\n";
    +	$lineno = 0;
    +	while ($line = <FILE>) {
    +		$lineno++;
    +		&check_include();
    +	}
    +	close(FILE);
    +}
    +exit($ret);
    +
    +my $includere = qr/^\s*#\s*include\s+<(.*)>/;
    +sub check_include()
    +{
    +	if ($line =~ m/^\s*#\s*include\s+<(.*)>/) {
    +		my $inc = $1;
    +		if ($inc =~ m/^asm/ || $inc =~ m/^linux/) {
    +			if (!(stat($dir . "/" . $inc))) {
    +				printf STDERR "$filename:$lineno: included file '$inc' is not exported\n";
    +				$ret = 1;
    +			}
    +		}
    +	}
    +}
    diff --git a/scripts/headers_install.pl b/scripts/headers_install.pl
    new file mode 100644
    index 0000000..8aa66af
    --- /dev/null
    +++ b/scripts/headers_install.pl
    @@ -0,0 +1,39 @@
    +#!/usr/bin/perl
    +#
    +# headers_install prepare the listed header files for use in
    +# user space and copy the files to their destination.
    +#
    +# Usage: headers_install.pl opendir installdir [files...]
    +# opendir: dir to open files
    +# install: dir to install the files
    +# files..: list of files to check
    +#
    +# Step in preparation for users space:
    +# 1) Drop all use of compiler.h definitions
    +# 2) Drop include of compiler.h
    +# 3) Drop all sections defined out by __KERNEL__
    +
    +my ($opendir, $installdir, @files) = @ARGV;
    +
    +my $ret = 0;
    +
    +foreach $file (@files) {
    +	open(INFILE, "< $opendir/$file") or die "$openddir/$file: $!\n";
    +	open(OUTFILE, "> $installdir/$file.tmp") or
    +		die "$installdir/$file.tmp: $!\n";
    +	while ($line = <INFILE>) {
    +		$line =~ s/([\s(])__user\s/\1/g;
    +		$line =~ s/([\s(])__force\s/\1/g;
    +		$line =~ s/([\s(])__iomem\s/\1/g;
    +		$line =~ s/\s__attribute_const__\s/ /g;
    +		$line =~ s/\s__attribute_const__$//g;
    +		$line =~ s/^#include <linux\/compiler.h>//;
    +		printf OUTFILE $line;
    +	}
    +	close(OUTFILE);
    +	close(INFILE);
    +	system "scripts/unifdef -U__KERNEL__ $installdir/$file.tmp > $installdir/$file"
    +}
    +
    +exit($ret);
    +
    
    
    \
     
     \ /
      Last update: 2008-06-08 11:49    [from the cache]
    ©2003-2008