lkml.org 
[lkml]   [1998]   [Mar]   [15]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patch in this message
/
Date
From
SubjectRe: [patch] x86, generic speedup, do we need 'cld'?

On Fri, 13 Mar 1998, Alan Cox wrote:

> > This should be fine as long as you verify that all kernel entry points do
> > have the cld - which I think they do. I'm nervous about it, though,
>
> And all the libc5 stuff that happens to inherit those definitions..

here is a new patch that fixes the libc5 issue. (by keeping the old way
for libc5. This also makes it easy to turn this feature off even in the
kernel).

cld cost on a PPro is 6 cycles, 3 on a P5, and it's used in some 1000
places in the kernel. I've reviewed all places where we switch to/from
some untrusted code. The following code was reviewed to esure correctness
of this patch:

arch/i386/boot/compressed/head.S (bootup): safe
arch/i386/kernel/entry.S (gates, traps, exceptions, NMI): safe
arch/i386/kernel/head.S (bootup): safe
arch/i386/kernel/process.c (reboot): safe
arch/i386/kernel/trampoline.S (SMP bootup): safe
arch/i386/kernel/irq.h (IRQ entries): safe
drivers/char/apm_bios.c (32 bit APM BIOS calls): safe

i'd like to ask all bored x86 assembly hackers to try to find any
unreviewed code.

the patch compiles/boots/works on 2.1.90 pre2.

-- mingo

--- linux/include/asm-i386/cld.h.orig Fri Mar 20 23:17:43 1998
+++ linux/include/asm-i386/cld.h Fri Mar 20 23:50:16 1998
@@ -0,0 +1,31 @@
+#ifndef _I386_CLD_H_
+#define _I386_CLD_H_
+
+/*
+ * So far _every_ str/mem/iosl function did a 'cld' before doing any of the
+ * 'rep' instructions. cld is quite expensive (3 cycles on a P5, __6__ cycles
+ * on a PPro, basically a limited pipeline flush), and there is no algorithmic
+ * need to use it in the kernel. Also it's used in some ~1000 places in a
+ * typical kernel image. So why not remove them?
+ *
+ * the theory is this: at all 'portals' where we enter the kernel from some
+ * untrusted code, we do an explicit 'cld'. Any kernel code can assume that
+ * the CPU is in 'cld' mode currently. Some functions do an explicit 'std',
+ * they all do a 'cld' later. (IRQ handling is safe as well)
+ *
+ * Unfortunately it's a very dangerous thing to remove cld's, since the
+ * 'direction flag' is rarely used in 'reverse direction' and bugs wrt.
+ * it's handling do not show up in a clear way. We've carefully reviewed
+ * the kernel source, but we cannot review all libc5 applications (string.h
+ * is used by libc5). Thus we only remove the 'cld' from the kernel, not
+ * from userspace uses.
+ */
+
+#ifndef __KERNEL__
+#define CLD "cld\n"
+#else
+#define CLD
+#endif
+
+#endif
+
--- linux/include/asm-i386/string.h.orig Thu Mar 19 05:46:45 1998
+++ linux/include/asm-i386/string.h Fri Mar 20 23:24:41 1998
@@ -14,6 +14,8 @@
#include <asm/string-486.h>
#else

+#include <asm/cld.h>
+
/*
* This string-include defines all string functions as inline
* functions. Use gcc. It also assumes ds=es=data space, this should be
@@ -31,6 +33,7 @@
extern inline char * strcpy(char * dest,const char *src)
{
__asm__ __volatile__(
+ CLD
"1:\tlodsb\n\t"
"stosb\n\t"
"testb %%al,%%al\n\t"
@@ -44,6 +47,7 @@
extern inline char * strncpy(char * dest,const char *src,size_t count)
{
__asm__ __volatile__(
+ CLD
"1:\tdecl %2\n\t"
"js 2f\n\t"
"lodsb\n\t"
@@ -62,6 +66,7 @@
extern inline char * strcat(char * dest,const char * src)
{
__asm__ __volatile__(
+ CLD
"repne\n\t"
"scasb\n\t"
"decl %1\n"
@@ -78,6 +83,7 @@
extern inline char * strncat(char * dest,const char * src,size_t count)
{
__asm__ __volatile__(
+ CLD
"repne\n\t"
"scasb\n\t"
"decl %1\n\t"
@@ -101,6 +107,7 @@
{
register int __res;
__asm__ __volatile__(
+ CLD
"1:\tlodsb\n\t"
"scasb\n\t"
"jne 2f\n\t"
@@ -120,6 +127,7 @@
{
register int __res;
__asm__ __volatile__(
+ CLD
"1:\tdecl %3\n\t"
"js 2f\n\t"
"lodsb\n\t"
@@ -141,6 +149,7 @@
{
register char * __res;
__asm__ __volatile__(
+ CLD
"movb %%al,%%ah\n"
"1:\tlodsb\n\t"
"cmpb %%ah,%%al\n\t"
@@ -159,6 +168,7 @@
{
register char * __res;
__asm__ __volatile__(
+ CLD
"movb %%al,%%ah\n"
"1:\tlodsb\n\t"
"cmpb %%ah,%%al\n\t"
@@ -175,6 +185,7 @@
{
register char * __res;
__asm__ __volatile__(
+ CLD
"movl %4,%%edi\n\t"
"repne\n\t"
"scasb\n\t"
@@ -200,6 +211,7 @@
{
register char * __res;
__asm__ __volatile__(
+ CLD
"movl %4,%%edi\n\t"
"repne\n\t"
"scasb\n\t"
@@ -225,6 +237,7 @@
{
register char * __res;
__asm__ __volatile__(
+ CLD
"movl %4,%%edi\n\t"
"repne\n\t"
"scasb\n\t"
@@ -253,6 +266,7 @@
{
register char * __res;
__asm__ __volatile__(
+ CLD
"movl %4,%%edi\n\t"
"repne\n\t"
"scasb\n\t"
@@ -281,6 +295,7 @@
{
register int __res;
__asm__ __volatile__(
+ CLD
"repne\n\t"
"scasb\n\t"
"notl %0\n\t"
@@ -302,6 +317,7 @@
"1:\txorl %0,%0\n\t"
"movl $-1,%%ecx\n\t"
"xorl %%eax,%%eax\n\t"
+ CLD
"movl %4,%%edi\n\t"
"repne\n\t"
"scasb\n\t"
@@ -352,6 +368,7 @@
extern inline void * __memcpy(void * to, const void * from, size_t n)
{
__asm__ __volatile__(
+ CLD
"rep ; movsl\n\t"
"testb $2,%b1\n\t"
"je 1f\n\t"
@@ -416,7 +433,8 @@
return to;
}
#define COMMON(x) \
-__asm__("rep ; movsl" \
+__asm__(CLD \
+ "rep ; movsl" \
x \
: /* no outputs */ \
: "c" (n/4),"D" ((long) to),"S" ((long) from) \
@@ -442,6 +460,7 @@
{
if (dest<src)
__asm__ __volatile__(
+ CLD
"rep\n\t"
"movsb"
: /* no output */
@@ -452,7 +471,7 @@
"std\n\t"
"rep\n\t"
"movsb\n\t"
- "cld"
+ "cld\n\t" /* cld is necessary here! */
: /* no output */
:"c" (n),
"S" (n-1+(const char *)src),
@@ -470,6 +489,7 @@
if (!count)
return NULL;
__asm__ __volatile__(
+ CLD
"repne\n\t"
"scasb\n\t"
"je 1f\n\t"
@@ -483,6 +503,7 @@
extern inline void * __memset_generic(void * s, char c,size_t count)
{
__asm__ __volatile__(
+ CLD
"rep\n\t"
"stosb"
: /* no output */
@@ -502,6 +523,7 @@
extern inline void * __constant_c_memset(void * s, unsigned long c, size_t count)
{
__asm__ __volatile__(
+ CLD
"rep ; stosl\n\t"
"testb $2,%b1\n\t"
"je 1f\n\t"
@@ -562,7 +584,8 @@
return s;
}
#define COMMON(x) \
-__asm__( "rep ; stosl" \
+__asm__(CLD \
+ "rep ; stosl" \
x \
: /* no outputs */ \
: "a" (pattern),"c" (count/4),"D" ((long) s) \
@@ -601,7 +624,8 @@
{
if (!size)
return addr;
- __asm__("repnz; scasb
+ __asm__( CLD
+ "repnz; scasb
jnz 1f
dec %%edi
1: "
--- linux/include/asm-i386/io.h.orig Thu Mar 19 05:50:09 1998
+++ linux/include/asm-i386/io.h Fri Mar 20 23:26:43 1998
@@ -14,6 +14,8 @@
* mistake somewhere.
*/

+#include <asm/cld.h>
+
/*
* Thanks to James van Artsdalen for a better timing-fix than
* the two short jumps: using outb's to a nonexistent port seems
@@ -66,12 +68,12 @@

#define __INS(s) \
extern inline void ins##s(unsigned short port, void * addr, unsigned long count) \
-{ __asm__ __volatile__ ("rep ; ins" #s \
+{ __asm__ __volatile__ ( CLD "rep ; ins" #s \
: "=D" (addr), "=c" (count) : "d" (port),"0" (addr),"1" (count)); }

#define __OUTS(s) \
extern inline void outs##s(unsigned short port, const void * addr, unsigned long count) \
-{ __asm__ __volatile__ ("rep ; outs" #s \
+{ __asm__ __volatile__ ( CLD "rep ; outs" #s \
: "=S" (addr), "=c" (count) : "d" (port),"0" (addr),"1" (count)); }

#define RETURN_TYPE unsigned char
--- linux/include/asm-i386/posix_types.h.orig Thu Mar 19 05:48:01 1998
+++ linux/include/asm-i386/posix_types.h Fri Mar 20 23:21:52 1998
@@ -7,6 +7,8 @@
* assume GCC is being used.
*/

+#include <asm/cld.h>
+
typedef unsigned short __kernel_dev_t;
typedef unsigned long __kernel_ino_t;
typedef unsigned short __kernel_mode_t;
@@ -57,7 +59,7 @@

#undef __FD_ZERO
#define __FD_ZERO(fdsetp) \
- __asm__ __volatile__("rep ; stosl" \
+ __asm__ __volatile__( CLD "rep ; stosl" \
:"=m" (*(__kernel_fd_set *) (fdsetp)) \
:"a" (0), "c" (__FDSET_LONGS), \
"D" ((__kernel_fd_set *) (fdsetp)) :"cx","di")
--- linux/include/asm-i386/bitops.h.orig Thu Mar 19 05:48:47 1998
+++ linux/include/asm-i386/bitops.h Fri Mar 20 23:21:15 1998
@@ -5,6 +5,8 @@
* Copyright 1992, Linus Torvalds.
*/

+#include <asm/cld.h>
+
/*
* These have to be done with inline assembly: that way the bit-setting
* is guaranteed to be atomic. All bit operations return 0 if the bit
@@ -131,7 +133,8 @@

if (!size)
return 0;
- __asm__("movl $-1,%%eax\n\t"
+ __asm__(CLD
+ "movl $-1,%%eax\n\t"
"xorl %%edx,%%edx\n\t"
"repe; scasl\n\t"
"je 1f\n\t"
--- linux/arch/i386/mm/init.c.orig Fri Mar 20 23:23:22 1998
+++ linux/arch/i386/mm/init.c Fri Mar 20 23:23:31 1998
@@ -49,7 +49,7 @@
{
extern char empty_bad_page_table[PAGE_SIZE];

- __asm__ __volatile__("cld ; rep ; stosl":
+ __asm__ __volatile__("rep ; stosl":
:"a" (pte_val(BAD_PAGE)),
"D" ((long) empty_bad_page_table),
"c" (PAGE_SIZE/4)
@@ -61,7 +61,7 @@
{
extern char empty_bad_page[PAGE_SIZE];

- __asm__ __volatile__("cld ; rep ; stosl":
+ __asm__ __volatile__("rep ; stosl":
:"a" (0),
"D" ((long) empty_bad_page),
"c" (PAGE_SIZE/4)
--- linux/include/asm-i386/string-486.h.orig Sat Mar 21 00:16:03 1998
+++ linux/include/asm-i386/string-486.h Fri Mar 20 23:20:44 1998
@@ -16,6 +16,8 @@
* Split into 2 CPU specific files by Alan Cox to keep #ifdef noise down.
*/

+#include <asm/cld.h>
+
#define __HAVE_ARCH_STRCPY
extern inline char * strcpy(char * dest,const char *src)
{
@@ -182,7 +184,7 @@
{
register char * __res;
__asm__ __volatile__(
- "cld\n\t"
+ CLD
"movb %%al,%%ah\n"
"1:\tlodsb\n\t"
"cmpb %%ah,%%al\n\t"
@@ -199,7 +201,7 @@
{
register char * __res;
__asm__ __volatile__(
- "cld\n\t"
+ CLD
"movl %4,%%edi\n\t"
"repne\n\t"
"scasb\n\t"
@@ -225,7 +227,7 @@
{
register char * __res;
__asm__ __volatile__(
- "cld\n\t"
+ CLD
"movl %4,%%edi\n\t"
"repne\n\t"
"scasb\n\t"
@@ -251,7 +253,7 @@
{
register char * __res;
__asm__ __volatile__(
- "cld\n\t"
+ CLD
"movl %4,%%edi\n\t"
"repne\n\t"
"scasb\n\t"
@@ -280,7 +282,7 @@
{
register char * __res;
__asm__ __volatile__(
- "cld\n\t" \
+ CLD
"movl %4,%%edi\n\t"
"repne\n\t"
"scasb\n\t"
@@ -359,7 +361,7 @@
"1:\txorl %0,%0\n\t"
"movl $-1,%%ecx\n\t"
"xorl %%eax,%%eax\n\t"
- "cld\n\t"
+ CLD
"movl %4,%%edi\n\t"
"repne\n\t"
"scasb\n\t"
@@ -466,7 +468,7 @@
{
register void *tmp = (void *)to;
__asm__ __volatile__ (
- "cld\n\t"
+ CLD
"shrl $1,%%ecx\n\t"
"jnc 1f\n\t"
"movsb\n"
@@ -488,7 +490,7 @@
register void *tmp = (void *)dest;
if (dest<src)
__asm__ __volatile__ (
- "cld\n\t"
+ CLD
"rep\n\t"
"movsb"
: /* no output */
@@ -499,7 +501,7 @@
"std\n\t"
"rep\n\t"
"movsb\n\t"
- "cld"
+ "cld" /* cld is necessary here! */
: /* no output */
:"c" (n), "S" (n-1+(const char *)src), "D" (n-1+(char *)tmp)
:"cx","si","di","memory");
@@ -510,7 +512,7 @@
{
register int __res;
__asm__ __volatile__(
- "cld\n\t"
+ CLD
"repe\n\t"
"cmpsb\n\t"
"je 1f\n\t"
@@ -529,7 +531,7 @@
if (!count)
return NULL;
__asm__ __volatile__(
- "cld\n\t"
+ CLD
"repne\n\t"
"scasb\n\t"
"je 1f\n\t"
@@ -683,8 +685,8 @@
{
if (!size)
return addr;
- __asm__("cld
- repnz; scasb
+ __asm__(CLD
+ "repnz; scasb
jnz 1f
dec %%edi
1: "

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.rutgers.edu

\
 
 \ /
  Last update: 2005-03-22 13:41    [from the cache]
©2003-2011 Jasper Spaans