From 6434b4b15fbb6561d876f09d666a2c122ebcae07 Mon Sep 17 00:00:00 2001 From: Joel Sherrill Date: Fri, 9 Mar 2012 09:38:23 -0600 Subject: [PATCH] Add other architecture specific in_cksum routines + arm, i386, mips, and sparc64 + freebsd-to-rtems.py: Updated + Makefile: Regenerated. --- Makefile | 17 + freebsd-to-rtems.py | 10 +- freebsd/arm/arm/in_cksum.c | 154 ++++++ freebsd/arm/arm/in_cksum_arm.S | 341 ++++++++++++ .../arm/include/freebsd/machine/in_cksum.h | 66 +++ freebsd/i386/i386/in_cksum.c | 493 ++++++++++++++++++ .../i386/include/freebsd/machine/in_cksum.h | 142 +++++ .../mips/include/freebsd/machine/in_cksum.h | 77 +++ freebsd/mips/mips/in_cksum.c | 250 +++++++++ .../include/freebsd/machine/in_cksum.h | 171 ++++++ freebsd/sparc64/sparc64/in_cksum.c | 264 ++++++++++ 11 files changed, 1980 insertions(+), 5 deletions(-) create mode 100644 freebsd/arm/arm/in_cksum.c create mode 100644 freebsd/arm/arm/in_cksum_arm.S create mode 100644 freebsd/arm/include/freebsd/machine/in_cksum.h create mode 100644 freebsd/i386/i386/in_cksum.c create mode 100644 freebsd/i386/include/freebsd/machine/in_cksum.h create mode 100644 freebsd/mips/include/freebsd/machine/in_cksum.h create mode 100644 freebsd/mips/mips/in_cksum.c create mode 100644 freebsd/sparc64/include/freebsd/machine/in_cksum.h create mode 100644 freebsd/sparc64/sparc64/in_cksum.c diff --git a/Makefile b/Makefile index eeed1a8f..759356e9 100644 --- a/Makefile +++ b/Makefile @@ -341,10 +341,27 @@ C_FILES += \ rtemsbsd/src/rtems-bsd-sysctlnametomib.c \ rtemsbsd/src/rtems-bsd-uma.c +ifeq ($(RTEMS_CPU),arm) +C_FILES += \ + freebsd/arm/arm/in_cksum.c \ + freebsd/arm/arm/in_cksum_arm.S +endif +ifeq ($(RTEMS_CPU),i386) +C_FILES += \ + freebsd/i386/i386/in_cksum.c +endif +ifeq ($(RTEMS_CPU),mips) +C_FILES += \ + freebsd/mips/mips/in_cksum.c +endif ifeq ($(RTEMS_CPU),powerpc) C_FILES += \ freebsd/powerpc/powerpc/in_cksum.c endif +ifeq ($(RTEMS_CPU),sparc64) +C_FILES += \ + freebsd/sparc64/sparc64/in_cksum.c +endif C_O_FILES = $(C_FILES:%.c=%.o) C_DEP_FILES = $(C_FILES:%.c=%.dep) diff --git a/freebsd-to-rtems.py b/freebsd-to-rtems.py index 2b16f512..c6918063 100755 --- a/freebsd-to-rtems.py +++ b/freebsd-to-rtems.py @@ -1767,7 +1767,7 @@ powerpcDependent.addSourceFiles( # SPARC64 Architecture Specific Files Module sparc64Dependent = Module('cpu_dependent') -sparc64Dependent.setTarget("powerpc") +sparc64Dependent.setTarget("sparc64") sparc64Dependent.addHeaderFiles( [ 'sparc64/include/in_cksum.h', @@ -1859,11 +1859,11 @@ mm.addModule(devUsbStorage) #mm.addModule(devUsbNet) # Now add CPU Architecture Dependent Modules -#mm.addModule(armDependent) -#mm.addModule(i386Dependent) -#mm.addModule(mipsDependent) +mm.addModule(armDependent) +mm.addModule(i386Dependent) +mm.addModule(mipsDependent) mm.addModule(powerpcDependent) -#mm.addModule(sparc64Dependent) +mm.addModule(sparc64Dependent) # Perform the actual file manipulation if isForward == True: diff --git a/freebsd/arm/arm/in_cksum.c b/freebsd/arm/arm/in_cksum.c new file mode 100644 index 00000000..db98915d --- /dev/null +++ b/freebsd/arm/arm/in_cksum.c @@ -0,0 +1,154 @@ +#include + +/* $NetBSD: in_cksum.c,v 1.7 1997/09/02 13:18:15 thorpej Exp $ */ + +/*- + * Copyright (c) 1988, 1992, 1993 + * The Regents of the University of California. All rights reserved. + * Copyright (c) 1996 + * Matt Thomas + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)in_cksum.c 8.1 (Berkeley) 6/10/93 + */ + +#include /* RCS ID & Copyright macro defns */ +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +/* + * Checksum routine for Internet Protocol family headers + * (Portable Alpha version). + * + * This routine is very heavily used in the network + * code and should be modified for each CPU to be as fast as possible. + */ + +#define ADDCARRY(x) (x > 65535 ? x -= 65535 : x) +#define REDUCE32 \ + { \ + q_util.q = sum; \ + sum = q_util.s[0] + q_util.s[1] + q_util.s[2] + q_util.s[3]; \ + } +#define REDUCE16 \ + { \ + q_util.q = sum; \ + l_util.l = q_util.s[0] + q_util.s[1] + q_util.s[2] + q_util.s[3]; \ + sum = l_util.s[0] + l_util.s[1]; \ + ADDCARRY(sum); \ + } + +union l_util { + u_int16_t s[2]; + u_int32_t l; +}; +union q_util { + u_int16_t s[4]; + u_int32_t l[2]; + u_int64_t q; +}; + +u_short +in_addword(u_short a, u_short b) +{ + u_int64_t sum = a + b; + + ADDCARRY(sum); + return (sum); +} + +static +uint64_t _do_cksum(void *addr, int len) +{ + uint64_t sum; + union q_util q_util; + + sum = do_cksum(addr, len); + REDUCE32; + return (sum); +} + +u_short +in_cksum_skip(struct mbuf *m, int len, int skip) +{ + u_int64_t sum = 0; + int mlen = 0; + int clen = 0; + caddr_t addr; + union q_util q_util; + union l_util l_util; + + len -= skip; + for (; skip && m; m = m->m_next) { + if (m->m_len > skip) { + mlen = m->m_len - skip; + addr = mtod(m, caddr_t) + skip; + goto skip_start; + } else { + skip -= m->m_len; + } + } + + for (; m && len; m = m->m_next) { + if (m->m_len == 0) + continue; + mlen = m->m_len; + addr = mtod(m, caddr_t); +skip_start: + if (len < mlen) + mlen = len; + + if ((clen ^ (int) addr) & 1) + sum += _do_cksum(addr, mlen) << 8; + else + sum += _do_cksum(addr, mlen); + + clen += mlen; + len -= mlen; + } + REDUCE16; + return (~sum & 0xffff); +} + +u_int in_cksum_hdr(const struct ip *ip) +{ + u_int64_t sum = do_cksum(ip, sizeof(struct ip)); + union q_util q_util; + union l_util l_util; + REDUCE16; + return (~sum & 0xffff); +} diff --git a/freebsd/arm/arm/in_cksum_arm.S b/freebsd/arm/arm/in_cksum_arm.S new file mode 100644 index 00000000..d4d1ac13 --- /dev/null +++ b/freebsd/arm/arm/in_cksum_arm.S @@ -0,0 +1,341 @@ +#include + +/* $NetBSD: in_cksum_arm.S,v 1.2 2003/09/23 10:01:36 scw Exp $ */ + +/*- + * Copyright 2003 Wasabi Systems, Inc. + * All rights reserved. + * + * Written by Steve C. Woodford for Wasabi Systems, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed for the NetBSD Project by + * Wasabi Systems, Inc. + * 4. The name of Wasabi Systems, Inc. may not be used to endorse + * or promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* + * Hand-optimised in_cksum() and in4_cksum() implementations for ARM/armv5e + */ + +#include + +#include +#include +__FBSDID("$FreeBSD$"); + +/* + * int in_cksum(struct mbuf *m, int len) + * + * Entry: + * r0 m + * r1 len + * + * NOTE: Assumes 'm' is *never* NULL. + */ +/* LINTSTUB: Func: int in_cksum(struct mbuf *, int) */ +ENTRY(in_cksum) + stmfd sp!, {r4-r11,lr} + mov r8, #0x00 + mov r9, r1 + mov r10, #0x00 + mov ip, r0 + +.Lin_cksum_loop: + ldr r1, [ip, #(M_LEN)] + ldr r0, [ip, #(M_DATA)] + ldr ip, [ip, #(M_NEXT)] +.Lin_cksum_entry4: + cmp r9, r1 + movlt r1, r9 + sub r9, r9, r1 + eor r11, r10, r0 + add r10, r10, r1 + adds r2, r1, #0x00 + blne _ASM_LABEL(L_cksumdata) + tst r11, #0x01 + movne r2, r2, ror #8 + adds r8, r8, r2 + adc r8, r8, #0x00 + cmp ip, #0x00 + bne .Lin_cksum_loop + + mov r1, #0xff + orr r1, r1, #0xff00 + and r0, r8, r1 + add r0, r0, r8, lsr #16 + add r0, r0, r0, lsr #16 + and r0, r0, r1 + eor r0, r0, r1 + ldmfd sp!, {r4-r11,pc} + + +ENTRY(do_cksum) + stmfd sp!, {r4-r7, lr} + bl L_cksumdata + mov r0, r2 + ldmfd sp!, {r4-r7, pc} +/* + * The main in*_cksum() workhorse... + * + * Entry parameters: + * r0 Pointer to buffer + * r1 Buffer length + * lr Return address + * + * Returns: + * r2 Accumulated 32-bit sum + * + * Clobbers: + * r0-r7 + */ +/* LINTSTUB: Ignore */ +ASENTRY_NP(L_cksumdata) +#ifdef _ARM_ARCH_5E + pld [r0] /* Pre-fetch the start of the buffer */ +#endif + mov r2, #0 + + /* We first have to word-align the buffer. */ + ands r7, r0, #0x03 + beq .Lcksumdata_wordaligned + rsb r7, r7, #0x04 + cmp r1, r7 /* Enough bytes left to make it? */ + blt .Lcksumdata_endgame + cmp r7, #0x02 + ldrb r4, [r0], #0x01 /* Fetch 1st byte */ + ldrgeb r5, [r0], #0x01 /* Fetch 2nd byte */ + movlt r5, #0x00 + ldrgtb r6, [r0], #0x01 /* Fetch 3rd byte */ + movle r6, #0x00 + /* Combine the three bytes depending on endianness and alignment */ +#ifdef __ARMEB__ + orreq r2, r5, r4, lsl #8 + orreq r2, r2, r6, lsl #24 + orrne r2, r4, r5, lsl #8 + orrne r2, r2, r6, lsl #16 +#else + orreq r2, r4, r5, lsl #8 + orreq r2, r2, r6, lsl #16 + orrne r2, r5, r4, lsl #8 + orrne r2, r2, r6, lsl #24 +#endif + subs r1, r1, r7 /* Update length */ + RETeq /* All done? */ + + /* Buffer is now word aligned */ +.Lcksumdata_wordaligned: +#ifdef _ARM_ARCH_5E + cmp r1, #0x04 /* Less than 4 bytes left? */ + blt .Lcksumdata_endgame /* Yup */ + + /* Now quad-align, if necessary */ + ands r7, r0, #0x04 + ldrne r7, [r0], #0x04 + subne r1, r1, #0x04 + subs r1, r1, #0x40 + blt .Lcksumdata_bigloop_end /* Note: C flag clear if branch taken */ + + /* + * Buffer is now quad aligned. Sum 64 bytes at a time. + * Note: First ldrd is hoisted above the loop, together with + * setting r6 to zero to avoid stalling for results in the + * loop. (r7 is live, from above). + */ + ldrd r4, [r0], #0x08 + mov r6, #0x00 +.Lcksumdata_bigloop: + pld [r0, #0x18] + adds r2, r2, r6 + adcs r2, r2, r7 + ldrd r6, [r0], #0x08 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldrd r4, [r0], #0x08 + adcs r2, r2, r6 + adcs r2, r2, r7 + ldrd r6, [r0], #0x08 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldrd r4, [r0], #0x08 + adcs r2, r2, r6 + adcs r2, r2, r7 + pld [r0, #0x18] + ldrd r6, [r0], #0x08 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldrd r4, [r0], #0x08 + adcs r2, r2, r6 + adcs r2, r2, r7 + ldrd r6, [r0], #0x08 + adcs r2, r2, r4 + adcs r2, r2, r5 + adc r2, r2, #0x00 + subs r1, r1, #0x40 + ldrged r4, [r0], #0x08 + bge .Lcksumdata_bigloop + + adds r2, r2, r6 /* r6/r7 still need summing */ +.Lcksumdata_bigloop_end: + adcs r2, r2, r7 + adc r2, r2, #0x00 + +#else /* !_ARM_ARCH_5E */ + + subs r1, r1, #0x40 + blt .Lcksumdata_bigloop_end + +.Lcksumdata_bigloop: + ldmia r0!, {r3, r4, r5, r6} + adds r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldmia r0!, {r3, r4, r5, r7} + adcs r2, r2, r6 + adcs r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldmia r0!, {r3, r4, r5, r6} + adcs r2, r2, r7 + adcs r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldmia r0!, {r3, r4, r5, r7} + adcs r2, r2, r6 + adcs r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + adcs r2, r2, r7 + adc r2, r2, #0x00 + subs r1, r1, #0x40 + bge .Lcksumdata_bigloop +.Lcksumdata_bigloop_end: +#endif + + adds r1, r1, #0x40 + RETeq + cmp r1, #0x20 + +#ifdef _ARM_ARCH_5E + ldrged r4, [r0], #0x08 /* Avoid stalling pld and result */ + blt .Lcksumdata_less_than_32 + pld [r0, #0x18] + ldrd r6, [r0], #0x08 + adds r2, r2, r4 + adcs r2, r2, r5 + ldrd r4, [r0], #0x08 + adcs r2, r2, r6 + adcs r2, r2, r7 + ldrd r6, [r0], #0x08 + adcs r2, r2, r4 + adcs r2, r2, r5 + adcs r2, r2, r6 /* XXX: Unavoidable result stall */ + adcs r2, r2, r7 +#else + blt .Lcksumdata_less_than_32 + ldmia r0!, {r3, r4, r5, r6} + adds r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldmia r0!, {r3, r4, r5, r7} + adcs r2, r2, r6 + adcs r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + adcs r2, r2, r7 +#endif + adc r2, r2, #0x00 + subs r1, r1, #0x20 + RETeq + +.Lcksumdata_less_than_32: + /* There are less than 32 bytes left */ + and r3, r1, #0x18 + rsb r4, r3, #0x18 + sub r1, r1, r3 + adds r4, r4, r4, lsr #1 /* Side effect: Clear carry flag */ + addne pc, pc, r4 + nop + +/* + * Note: We use ldm here, even on armv5e, since the combined issue/result + * latencies for ldm and ldrd are the same. Using ldm avoids needless #ifdefs. + */ + /* At least 24 bytes remaining... */ + ldmia r0!, {r4, r5} + adcs r2, r2, r4 + adcs r2, r2, r5 + + /* At least 16 bytes remaining... */ + ldmia r0!, {r4, r5} + adcs r2, r2, r4 + adcs r2, r2, r5 + + /* At least 8 bytes remaining... */ + ldmia r0!, {r4, r5} + adcs r2, r2, r4 + adcs r2, r2, r5 + + /* Less than 8 bytes remaining... */ + adc r2, r2, #0x00 + subs r1, r1, #0x04 + blt .Lcksumdata_lessthan4 + + ldr r4, [r0], #0x04 + sub r1, r1, #0x04 + adds r2, r2, r4 + adc r2, r2, #0x00 + + /* Deal with < 4 bytes remaining */ +.Lcksumdata_lessthan4: + adds r1, r1, #0x04 + RETeq + + /* Deal with 1 to 3 remaining bytes, possibly misaligned */ +.Lcksumdata_endgame: + ldrb r3, [r0] /* Fetch first byte */ + cmp r1, #0x02 + ldrgeb r4, [r0, #0x01] /* Fetch 2nd and 3rd as necessary */ + movlt r4, #0x00 + ldrgtb r5, [r0, #0x02] + movle r5, #0x00 + /* Combine the three bytes depending on endianness and alignment */ + tst r0, #0x01 +#ifdef __ARMEB__ + orreq r3, r4, r3, lsl #8 + orreq r3, r3, r5, lsl #24 + orrne r3, r3, r4, lsl #8 + orrne r3, r3, r5, lsl #16 +#else + orreq r3, r3, r4, lsl #8 + orreq r3, r3, r5, lsl #16 + orrne r3, r4, r3, lsl #8 + orrne r3, r3, r5, lsl #24 +#endif + adds r2, r2, r3 + adc r2, r2, #0x00 + RET diff --git a/freebsd/arm/include/freebsd/machine/in_cksum.h b/freebsd/arm/include/freebsd/machine/in_cksum.h new file mode 100644 index 00000000..5e3e9333 --- /dev/null +++ b/freebsd/arm/include/freebsd/machine/in_cksum.h @@ -0,0 +1,66 @@ +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from tahoe: in_cksum.c 1.2 86/01/05 + * from: @(#)in_cksum.c 1.3 (Berkeley) 1/19/91 + * from: Id: in_cksum.c,v 1.8 1995/12/03 18:35:19 bde Exp + * $FreeBSD$ + */ + +#ifndef _MACHINE_IN_CKSUM_HH_ +#define _MACHINE_IN_CKSUM_HH_ 1 + +#include + +#ifdef _KERNEL +u_short in_cksum(struct mbuf *m, int len); +u_short in_addword(u_short sum, u_short b); +u_short in_cksum_skip(struct mbuf *m, int len, int skip); +u_int do_cksum(const void *, int); +u_int in_cksum_hdr(const struct ip *); + +static __inline u_short +in_pseudo(u_int sum, u_int b, u_int c) +{ + __asm __volatile("adds %0, %0, %1\n" + "adcs %0, %0, %2\n" + "adc %0, %0, #0\n" + : "+r" (sum) + : "r" (b), "r" (c)); + sum = (sum & 0xffff) + (sum >> 16); + if (sum > 0xffff) + sum -= 0xffff; + return (sum); +} + +#endif /* _KERNEL */ +#endif /* _MACHINE_IN_CKSUM_HH_ */ diff --git a/freebsd/i386/i386/in_cksum.c b/freebsd/i386/i386/in_cksum.c new file mode 100644 index 00000000..0f663989 --- /dev/null +++ b/freebsd/i386/i386/in_cksum.c @@ -0,0 +1,493 @@ +#include + +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from tahoe: in_cksum.c 1.2 86/01/05 + * from: @(#)in_cksum.c 1.3 (Berkeley) 1/19/91 + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include + +#include +#include +#include + +#include + +/* + * Checksum routine for Internet Protocol family headers. + * + * This routine is very heavily used in the network + * code and should be modified for each CPU to be as fast as possible. + * + * This implementation is 386 version. + */ + +#undef ADDCARRY +#define ADDCARRY(x) if ((x) > 0xffff) (x) -= 0xffff +/* + * icc needs to be special cased here, as the asm code below results + * in broken code if compiled with icc. + */ +#if !defined(__GNUCLIKE_ASM) || defined(__INTEL_COMPILER) +/* non gcc parts stolen from sys/alpha/alpha/in_cksum.c */ +#define REDUCE32 \ + { \ + q_util.q = sum; \ + sum = q_util.s[0] + q_util.s[1] + q_util.s[2] + q_util.s[3]; \ + } +#define REDUCE16 \ + { \ + q_util.q = sum; \ + l_util.l = q_util.s[0] + q_util.s[1] + q_util.s[2] + q_util.s[3]; \ + sum = l_util.s[0] + l_util.s[1]; \ + ADDCARRY(sum); \ + } +#endif +#define REDUCE {sum = (sum & 0xffff) + (sum >> 16); ADDCARRY(sum);} + +#if !defined(__GNUCLIKE_ASM) || defined(__INTEL_COMPILER) +static const u_int32_t in_masks[] = { + /*0 bytes*/ /*1 byte*/ /*2 bytes*/ /*3 bytes*/ + 0x00000000, 0x000000FF, 0x0000FFFF, 0x00FFFFFF, /* offset 0 */ + 0x00000000, 0x0000FF00, 0x00FFFF00, 0xFFFFFF00, /* offset 1 */ + 0x00000000, 0x00FF0000, 0xFFFF0000, 0xFFFF0000, /* offset 2 */ + 0x00000000, 0xFF000000, 0xFF000000, 0xFF000000, /* offset 3 */ +}; + +union l_util { + u_int16_t s[2]; + u_int32_t l; +}; +union q_util { + u_int16_t s[4]; + u_int32_t l[2]; + u_int64_t q; +}; + +static u_int64_t +in_cksumdata(const u_int32_t *lw, int len) +{ + u_int64_t sum = 0; + u_int64_t prefilled; + int offset; + union q_util q_util; + + if ((3 & (long) lw) == 0 && len == 20) { + sum = (u_int64_t) lw[0] + lw[1] + lw[2] + lw[3] + lw[4]; + REDUCE32; + return sum; + } + + if ((offset = 3 & (long) lw) != 0) { + const u_int32_t *masks = in_masks + (offset << 2); + lw = (u_int32_t *) (((long) lw) - offset); + sum = *lw++ & masks[len >= 3 ? 3 : len]; + len -= 4 - offset; + if (len <= 0) { + REDUCE32; + return sum; + } + } +#if 0 + /* + * Force to cache line boundary. + */ + offset = 32 - (0x1f & (long) lw); + if (offset < 32 && len > offset) { + len -= offset; + if (4 & offset) { + sum += (u_int64_t) lw[0]; + lw += 1; + } + if (8 & offset) { + sum += (u_int64_t) lw[0] + lw[1]; + lw += 2; + } + if (16 & offset) { + sum += (u_int64_t) lw[0] + lw[1] + lw[2] + lw[3]; + lw += 4; + } + } +#endif + /* + * access prefilling to start load of next cache line. + * then add current cache line + * save result of prefilling for loop iteration. + */ + prefilled = lw[0]; + while ((len -= 32) >= 4) { + u_int64_t prefilling = lw[8]; + sum += prefilled + lw[1] + lw[2] + lw[3] + + lw[4] + lw[5] + lw[6] + lw[7]; + lw += 8; + prefilled = prefilling; + } + if (len >= 0) { + sum += prefilled + lw[1] + lw[2] + lw[3] + + lw[4] + lw[5] + lw[6] + lw[7]; + lw += 8; + } else { + len += 32; + } + while ((len -= 16) >= 0) { + sum += (u_int64_t) lw[0] + lw[1] + lw[2] + lw[3]; + lw += 4; + } + len += 16; + while ((len -= 4) >= 0) { + sum += (u_int64_t) *lw++; + } + len += 4; + if (len > 0) + sum += (u_int64_t) (in_masks[len] & *lw); + REDUCE32; + return sum; +} + +u_short +in_addword(u_short a, u_short b) +{ + u_int64_t sum = a + b; + + ADDCARRY(sum); + return (sum); +} + +u_short +in_pseudo(u_int32_t a, u_int32_t b, u_int32_t c) +{ + u_int64_t sum; + union q_util q_util; + union l_util l_util; + + sum = (u_int64_t) a + b + c; + REDUCE16; + return (sum); +} + +u_short +in_cksum_skip(struct mbuf *m, int len, int skip) +{ + u_int64_t sum = 0; + int mlen = 0; + int clen = 0; + caddr_t addr; + union q_util q_util; + union l_util l_util; + + len -= skip; + for (; skip && m; m = m->m_next) { + if (m->m_len > skip) { + mlen = m->m_len - skip; + addr = mtod(m, caddr_t) + skip; + goto skip_start; + } else { + skip -= m->m_len; + } + } + + for (; m && len; m = m->m_next) { + if (m->m_len == 0) + continue; + mlen = m->m_len; + addr = mtod(m, caddr_t); +skip_start: + if (len < mlen) + mlen = len; + if ((clen ^ (long) addr) & 1) + sum += in_cksumdata((const u_int32_t *)addr, mlen) << 8; + else + sum += in_cksumdata((const u_int32_t *)addr, mlen); + + clen += mlen; + len -= mlen; + } + REDUCE16; + return (~sum & 0xffff); +} + +u_int in_cksum_hdr(const struct ip *ip) +{ + u_int64_t sum = in_cksumdata((const u_int32_t *)ip, sizeof(struct ip)); + union q_util q_util; + union l_util l_util; + + REDUCE16; + return (~sum & 0xffff); +} +#else + +/* + * These asm statements require __volatile because they pass information + * via the condition codes. GCC does not currently provide a way to specify + * the condition codes as an input or output operand. + * + * The LOAD macro below is effectively a prefetch into cache. GCC will + * load the value into a register but will not use it. Since modern CPUs + * reorder operations, this will generally take place in parallel with + * other calculations. + */ +u_short +in_cksum_skip(m, len, skip) + struct mbuf *m; + int len; + int skip; +{ + register u_short *w; + register unsigned sum = 0; + register int mlen = 0; + int byte_swapped = 0; + union { char c[2]; u_short s; } su; + + len -= skip; + for (; skip && m; m = m->m_next) { + if (m->m_len > skip) { + mlen = m->m_len - skip; + w = (u_short *)(mtod(m, u_char *) + skip); + goto skip_start; + } else { + skip -= m->m_len; + } + } + + for (;m && len; m = m->m_next) { + if (m->m_len == 0) + continue; + w = mtod(m, u_short *); + if (mlen == -1) { + /* + * The first byte of this mbuf is the continuation + * of a word spanning between this mbuf and the + * last mbuf. + */ + + /* su.c[0] is already saved when scanning previous + * mbuf. sum was REDUCEd when we found mlen == -1 + */ + su.c[1] = *(u_char *)w; + sum += su.s; + w = (u_short *)((char *)w + 1); + mlen = m->m_len - 1; + len--; + } else + mlen = m->m_len; +skip_start: + if (len < mlen) + mlen = len; + len -= mlen; + /* + * Force to long boundary so we do longword aligned + * memory operations + */ + if (3 & (int) w) { + REDUCE; + if ((1 & (int) w) && (mlen > 0)) { + sum <<= 8; + su.c[0] = *(char *)w; + w = (u_short *)((char *)w + 1); + mlen--; + byte_swapped = 1; + } + if ((2 & (int) w) && (mlen >= 2)) { + sum += *w++; + mlen -= 2; + } + } + /* + * Advance to a 486 cache line boundary. + */ + if (4 & (int) w && mlen >= 4) { + __asm __volatile ( + "addl %1, %0\n" + "adcl $0, %0" + : "+r" (sum) + : "g" (((const u_int32_t *)w)[0]) + ); + w += 2; + mlen -= 4; + } + if (8 & (int) w && mlen >= 8) { + __asm __volatile ( + "addl %1, %0\n" + "adcl %2, %0\n" + "adcl $0, %0" + : "+r" (sum) + : "g" (((const u_int32_t *)w)[0]), + "g" (((const u_int32_t *)w)[1]) + ); + w += 4; + mlen -= 8; + } + /* + * Do as much of the checksum as possible 32 bits at at time. + * In fact, this loop is unrolled to make overhead from + * branches &c small. + */ + mlen -= 1; + while ((mlen -= 32) >= 0) { + /* + * Add with carry 16 words and fold in the last + * carry by adding a 0 with carry. + * + * The early ADD(16) and the LOAD(32) are to load + * the next 2 cache lines in advance on 486's. The + * 486 has a penalty of 2 clock cycles for loading + * a cache line, plus whatever time the external + * memory takes to load the first word(s) addressed. + * These penalties are unavoidable. Subsequent + * accesses to a cache line being loaded (and to + * other external memory?) are delayed until the + * whole load finishes. These penalties are mostly + * avoided by not accessing external memory for + * 8 cycles after the ADD(16) and 12 cycles after + * the LOAD(32). The loop terminates when mlen + * is initially 33 (not 32) to guaranteed that + * the LOAD(32) is within bounds. + */ + __asm __volatile ( + "addl %1, %0\n" + "adcl %2, %0\n" + "adcl %3, %0\n" + "adcl %4, %0\n" + "adcl %5, %0\n" + "mov %6, %%eax\n" + "adcl %7, %0\n" + "adcl %8, %0\n" + "adcl %9, %0\n" + "adcl $0, %0" + : "+r" (sum) + : "g" (((const u_int32_t *)w)[4]), + "g" (((const u_int32_t *)w)[0]), + "g" (((const u_int32_t *)w)[1]), + "g" (((const u_int32_t *)w)[2]), + "g" (((const u_int32_t *)w)[3]), + "g" (((const u_int32_t *)w)[8]), + "g" (((const u_int32_t *)w)[5]), + "g" (((const u_int32_t *)w)[6]), + "g" (((const u_int32_t *)w)[7]) + : "eax" + ); + w += 16; + } + mlen += 32 + 1; + if (mlen >= 32) { + __asm __volatile ( + "addl %1, %0\n" + "adcl %2, %0\n" + "adcl %3, %0\n" + "adcl %4, %0\n" + "adcl %5, %0\n" + "adcl %6, %0\n" + "adcl %7, %0\n" + "adcl %8, %0\n" + "adcl $0, %0" + : "+r" (sum) + : "g" (((const u_int32_t *)w)[4]), + "g" (((const u_int32_t *)w)[0]), + "g" (((const u_int32_t *)w)[1]), + "g" (((const u_int32_t *)w)[2]), + "g" (((const u_int32_t *)w)[3]), + "g" (((const u_int32_t *)w)[5]), + "g" (((const u_int32_t *)w)[6]), + "g" (((const u_int32_t *)w)[7]) + ); + w += 16; + mlen -= 32; + } + if (mlen >= 16) { + __asm __volatile ( + "addl %1, %0\n" + "adcl %2, %0\n" + "adcl %3, %0\n" + "adcl %4, %0\n" + "adcl $0, %0" + : "+r" (sum) + : "g" (((const u_int32_t *)w)[0]), + "g" (((const u_int32_t *)w)[1]), + "g" (((const u_int32_t *)w)[2]), + "g" (((const u_int32_t *)w)[3]) + ); + w += 8; + mlen -= 16; + } + if (mlen >= 8) { + __asm __volatile ( + "addl %1, %0\n" + "adcl %2, %0\n" + "adcl $0, %0" + : "+r" (sum) + : "g" (((const u_int32_t *)w)[0]), + "g" (((const u_int32_t *)w)[1]) + ); + w += 4; + mlen -= 8; + } + if (mlen == 0 && byte_swapped == 0) + continue; /* worth 1% maybe ?? */ + REDUCE; + while ((mlen -= 2) >= 0) { + sum += *w++; + } + if (byte_swapped) { + sum <<= 8; + byte_swapped = 0; + if (mlen == -1) { + su.c[1] = *(char *)w; + sum += su.s; + mlen = 0; + } else + mlen = -1; + } else if (mlen == -1) + /* + * This mbuf has odd number of bytes. + * There could be a word split betwen + * this mbuf and the next mbuf. + * Save the last byte (to prepend to next mbuf). + */ + su.c[0] = *(char *)w; + } + + if (len) + printf("%s: out of data by %d\n", __func__, len); + if (mlen == -1) { + /* The last mbuf has odd # of bytes. Follow the + standard (the odd byte is shifted left by 8 bits) */ + su.c[1] = 0; + sum += su.s; + } + REDUCE; + return (~sum & 0xffff); +} +#endif diff --git a/freebsd/i386/include/freebsd/machine/in_cksum.h b/freebsd/i386/include/freebsd/machine/in_cksum.h new file mode 100644 index 00000000..1f7dbac7 --- /dev/null +++ b/freebsd/i386/include/freebsd/machine/in_cksum.h @@ -0,0 +1,142 @@ +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from tahoe: in_cksum.c 1.2 86/01/05 + * from: @(#)in_cksum.c 1.3 (Berkeley) 1/19/91 + * from: Id: in_cksum.c,v 1.8 1995/12/03 18:35:19 bde Exp + * $FreeBSD$ + */ + +#ifndef _MACHINE_IN_CKSUM_HH_ +#define _MACHINE_IN_CKSUM_HH_ 1 + +#ifndef _SYS_CDEFS_HH_ +#error this file needs sys/cdefs.h as a prerequisite +#endif + +/* + * MP safe (alfred) + */ + +#include + +#define in_cksum(m, len) in_cksum_skip(m, len, 0) + +/* + * It it useful to have an Internet checksum routine which is inlineable + * and optimized specifically for the task of computing IP header checksums + * in the normal case (where there are no options and the header length is + * therefore always exactly five 32-bit words. + */ +#if defined(__GNUCLIKE_ASM) && !defined(__INTEL_COMPILER) +static __inline u_int +in_cksum_hdr(const struct ip *ip) +{ + u_int sum = 0; + + __asm( + "addl %1, %0\n" + "adcl %2, %0\n" + "adcl %3, %0\n" + "adcl %4, %0\n" + "adcl %5, %0\n" + "adcl $0, %0" + : "+r" (sum) + : "g" (((const u_int32_t *)ip)[0]), + "g" (((const u_int32_t *)ip)[1]), + "g" (((const u_int32_t *)ip)[2]), + "g" (((const u_int32_t *)ip)[3]), + "g" (((const u_int32_t *)ip)[4]) + : "cc" + ); + sum = (sum & 0xffff) + (sum >> 16); + if (sum > 0xffff) + sum -= 0xffff; + + return ~sum & 0xffff; +} + +static __inline void +in_cksum_update(struct ip *ip) +{ + int __tmpsum; + __tmpsum = (int)ntohs(ip->ip_sum) + 256; + ip->ip_sum = htons(__tmpsum + (__tmpsum >> 16)); +} + +static __inline u_short +in_addword(u_short sum, u_short b) +{ + __asm( + "addw %1, %0\n" + "adcw $0, %0" + : "+r" (sum) + : "g" (b) + : "cc" + ); + return (sum); +} + +static __inline u_short +in_pseudo(u_int sum, u_int b, u_int c) +{ + __asm( + "addl %1, %0\n" + "adcl %2, %0\n" + "adcl $0, %0" + : "+r" (sum) + : "g" (b), + "g" (c) + : "cc" + ); + sum = (sum & 0xffff) + (sum >> 16); + if (sum > 0xffff) + sum -= 0xffff; + return (sum); +} + +#else +#define in_cksum_update(ip) \ + do { \ + int __tmpsum; \ + __tmpsum = (int)ntohs(ip->ip_sum) + 256; \ + ip->ip_sum = htons(__tmpsum + (__tmpsum >> 16)); \ + } while(0) + +#endif + +#ifdef _KERNEL +#if !defined(__GNUCLIKE_ASM) || defined(__INTEL_COMPILER) +u_int in_cksum_hdr(const struct ip *ip); +u_short in_addword(u_short sum, u_short b); +u_short in_pseudo(u_int sum, u_int b, u_int c); +#endif +u_short in_cksum_skip(struct mbuf *m, int len, int skip); +#endif /* _KERNEL */ + +#endif /* _MACHINE_IN_CKSUM_HH_ */ diff --git a/freebsd/mips/include/freebsd/machine/in_cksum.h b/freebsd/mips/include/freebsd/machine/in_cksum.h new file mode 100644 index 00000000..4539d129 --- /dev/null +++ b/freebsd/mips/include/freebsd/machine/in_cksum.h @@ -0,0 +1,77 @@ +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from tahoe: in_cksum.c 1.2 86/01/05 + * from: @(#)in_cksum.c 1.3 (Berkeley) 1/19/91 + * from: Id: in_cksum.c,v 1.8 1995/12/03 18:35:19 bde Exp + * from: src/sys/alpha/include/in_cksum.h,v 1.7 2005/03/02 21:33:20 joerg + * $FreeBSD$ + */ + +#ifndef _MACHINE_IN_CKSUM_HH_ +#define _MACHINE_IN_CKSUM_HH_ 1 + +#include + +#define in_cksum(m, len) in_cksum_skip(m, len, 0) + +/* + * It it useful to have an Internet checksum routine which is inlineable + * and optimized specifically for the task of computing IP header checksums + * in the normal case (where there are no options and the header length is + * therefore always exactly five 32-bit words. + */ +#ifdef __CC_SUPPORTS___INLINE + +static __inline void +in_cksum_update(struct ip *ip) +{ + int __tmpsum; + __tmpsum = (int)ntohs(ip->ip_sum) + 256; + ip->ip_sum = htons(__tmpsum + (__tmpsum >> 16)); +} + +#else + +#define in_cksum_update(ip) \ + do { \ + int __tmpsum; \ + __tmpsum = (int)ntohs(ip->ip_sum) + 256; \ + ip->ip_sum = htons(__tmpsum + (__tmpsum >> 16)); \ + } while(0) + +#endif + +#ifdef _KERNEL +u_int in_cksum_hdr(const struct ip *ip); +u_short in_addword(u_short sum, u_short b); +u_short in_pseudo(u_int sum, u_int b, u_int c); +u_short in_cksum_skip(struct mbuf *m, int len, int skip); +#endif + +#endif /* _MACHINE_IN_CKSUM_HH_ */ diff --git a/freebsd/mips/mips/in_cksum.c b/freebsd/mips/mips/in_cksum.c new file mode 100644 index 00000000..b4a12df2 --- /dev/null +++ b/freebsd/mips/mips/in_cksum.c @@ -0,0 +1,250 @@ +#include + +/*- + * Copyright (c) 1988, 1992, 1993 + * The Regents of the University of California. All rights reserved. + * Copyright (c) 1996 + * Matt Thomas + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)in_cksum.c 8.1 (Berkeley) 6/10/93 + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +/* + * Checksum routine for Internet Protocol family headers + * (Portable Alpha version). + * + * This routine is very heavily used in the network + * code and should be modified for each CPU to be as fast as possible. + */ + +#define ADDCARRY(x) (x > 65535 ? x -= 65535 : x) +#define REDUCE32 \ + { \ + q_util.q = sum; \ + sum = q_util.s[0] + q_util.s[1] + q_util.s[2] + q_util.s[3]; \ + } +#define REDUCE16 \ + { \ + q_util.q = sum; \ + l_util.l = q_util.s[0] + q_util.s[1] + q_util.s[2] + q_util.s[3]; \ + sum = l_util.s[0] + l_util.s[1]; \ + ADDCARRY(sum); \ + } + +static const u_int32_t in_masks[] = { +#if _BYTE_ORDER == _LITTLE_ENDIAN + /*0 bytes*/ /*1 byte*/ /*2 bytes*/ /*3 bytes*/ + 0x00000000, 0x000000FF, 0x0000FFFF, 0x00FFFFFF, /* offset 0 */ + 0x00000000, 0x0000FF00, 0x00FFFF00, 0xFFFFFF00, /* offset 1 */ + 0x00000000, 0x00FF0000, 0xFFFF0000, 0xFFFF0000, /* offset 2 */ + 0x00000000, 0xFF000000, 0xFF000000, 0xFF000000, /* offset 3 */ +#else + /*0 bytes*/ /*1 byte*/ /*2 bytes*/ /*3 bytes*/ + 0x00000000, 0xFF000000, 0xFFFF0000, 0xFFFFFF00, /* offset 0 */ + 0x00000000, 0x00FF0000, 0x00FFFF00, 0x00FFFFFF, /* offset 1 */ + 0x00000000, 0x0000FF00, 0x0000FFFF, 0x0000FFFF, /* offset 2 */ + 0x00000000, 0x000000FF, 0x000000FF, 0x000000FF, /* offset 3 */ +#endif +}; + +union l_util { + u_int16_t s[2]; + u_int32_t l; +}; +union q_util { + u_int16_t s[4]; + u_int32_t l[2]; + u_int64_t q; +}; + +static u_int64_t +in_cksumdata(const void *buf, int len) +{ + const u_int32_t *lw = (const u_int32_t *) buf; + u_int64_t sum = 0; + u_int64_t prefilled; + int offset; + union q_util q_util; + + if ((3 & (long) lw) == 0 && len == 20) { + sum = (u_int64_t) lw[0] + lw[1] + lw[2] + lw[3] + lw[4]; + REDUCE32; + return sum; + } + + if ((offset = 3 & (long) lw) != 0) { + const u_int32_t *masks = in_masks + (offset << 2); + lw = (u_int32_t *) (((long) lw) - offset); + sum = *lw++ & masks[len >= 3 ? 3 : len]; + len -= 4 - offset; + if (len <= 0) { + REDUCE32; + return sum; + } + } +#if 0 + /* + * Force to cache line boundary. + */ + offset = 32 - (0x1f & (long) lw); + if (offset < 32 && len > offset) { + len -= offset; + if (4 & offset) { + sum += (u_int64_t) lw[0]; + lw += 1; + } + if (8 & offset) { + sum += (u_int64_t) lw[0] + lw[1]; + lw += 2; + } + if (16 & offset) { + sum += (u_int64_t) lw[0] + lw[1] + lw[2] + lw[3]; + lw += 4; + } + } +#endif + /* + * access prefilling to start load of next cache line. + * then add current cache line + * save result of prefilling for loop iteration. + */ + prefilled = lw[0]; + while ((len -= 32) >= 4) { + u_int64_t prefilling = lw[8]; + sum += prefilled + lw[1] + lw[2] + lw[3] + + lw[4] + lw[5] + lw[6] + lw[7]; + lw += 8; + prefilled = prefilling; + } + if (len >= 0) { + sum += prefilled + lw[1] + lw[2] + lw[3] + + lw[4] + lw[5] + lw[6] + lw[7]; + lw += 8; + } else { + len += 32; + } + while ((len -= 16) >= 0) { + sum += (u_int64_t) lw[0] + lw[1] + lw[2] + lw[3]; + lw += 4; + } + len += 16; + while ((len -= 4) >= 0) { + sum += (u_int64_t) *lw++; + } + len += 4; + if (len > 0) + sum += (u_int64_t) (in_masks[len] & *lw); + REDUCE32; + return sum; +} + +u_short +in_addword(u_short a, u_short b) +{ + u_int64_t sum = a + b; + + ADDCARRY(sum); + return (sum); +} + +u_short +in_pseudo(u_int32_t a, u_int32_t b, u_int32_t c) +{ + u_int64_t sum; + union q_util q_util; + union l_util l_util; + + sum = (u_int64_t) a + b + c; + REDUCE16; + return (sum); +} + +u_short +in_cksum_skip(struct mbuf *m, int len, int skip) +{ + u_int64_t sum = 0; + int mlen = 0; + int clen = 0; + caddr_t addr; + union q_util q_util; + union l_util l_util; + + len -= skip; + for (; skip && m; m = m->m_next) { + if (m->m_len > skip) { + mlen = m->m_len - skip; + addr = mtod(m, caddr_t) + skip; + goto skip_start; + } else { + skip -= m->m_len; + } + } + + for (; m && len; m = m->m_next) { + if (m->m_len == 0) + continue; + mlen = m->m_len; + addr = mtod(m, caddr_t); +skip_start: + if (len < mlen) + mlen = len; + + if ((clen ^ (uintptr_t) addr) & 1) + sum += in_cksumdata(addr, mlen) << 8; + else + sum += in_cksumdata(addr, mlen); + + clen += mlen; + len -= mlen; + } + REDUCE16; + return (~sum & 0xffff); +} + +u_int in_cksum_hdr(const struct ip *ip) +{ + u_int64_t sum = in_cksumdata(ip, sizeof(struct ip)); + union q_util q_util; + union l_util l_util; + REDUCE16; + return (~sum & 0xffff); +} diff --git a/freebsd/sparc64/include/freebsd/machine/in_cksum.h b/freebsd/sparc64/include/freebsd/machine/in_cksum.h new file mode 100644 index 00000000..c121f46d --- /dev/null +++ b/freebsd/sparc64/include/freebsd/machine/in_cksum.h @@ -0,0 +1,171 @@ +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/*- + * Copyright (c) 2001 by Thomas Moestl . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE + * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * from tahoe: in_cksum.c 1.2 86/01/05 + * from: @(#)in_cksum.c 1.3 (Berkeley) 1/19/91 + * from: Id: in_cksum.c,v 1.8 1995/12/03 18:35:19 bde Exp + * from: FreeBSD: src/sys/alpha/include/in_cksum.h,v 1.5 2000/05/06 + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_IN_CKSUM_HH_ +#define _MACHINE_IN_CKSUM_HH_ 1 + +#include + +#define in_cksum(m, len) in_cksum_skip(m, len, 0) + +static __inline void +in_cksum_update(struct ip *ip) +{ + int __tmp; + + __tmp = (int)ip->ip_sum + 1; + ip->ip_sum = __tmp + (__tmp >> 16); +} + +static __inline u_short +in_addword(u_short sum, u_short b) +{ + u_long __ret, __tmp; + + __asm( + "sll %2, 16, %0\n" + "sll %3, 16, %1\n" + "addcc %0, %1, %0\n" + "srl %0, 16, %0\n" + "addc %0, 0, %0\n" + : "=&r" (__ret), "=&r" (__tmp) : "r" (sum), "r" (b) : "cc"); + return (__ret); +} + +static __inline u_short +in_pseudo(u_int sum, u_int b, u_int c) +{ + u_long __tmp; + + __asm( + "addcc %0, %3, %0\n" + "addccc %0, %4, %0\n" + "addc %0, 0, %0\n" + "sll %0, 16, %1\n" + "addcc %0, %1, %0\n" + "srl %0, 16, %0\n" + "addc %0, 0, %0\n" + : "=r" (sum), "=&r" (__tmp) : "0" (sum), "r" (b), "r" (c) : "cc"); + return (sum); +} + +static __inline u_int +in_cksum_hdr(struct ip *ip) +{ + u_long __ret, __tmp1, __tmp2, __tmp3, __tmp4; + + /* + * Use 32-bit memory accesses and additions - addition with carry only + * works for 32 bits, and fixing up alignment issues for 64 is probably + * more trouble than it's worth. + * This may read outside of the ip header, but does not cross a page + * boundary in doing so, so that should be OK. + * Actually, this specialized implementation might be overkill - using + * a generic implementation for both in_cksum_skip and in_cksum_hdr + * should not be too much more expensive. + */ +#define __LD_ADD(addr, tmp, sum, offs, mod) \ + "lduw [" #addr " + " #offs "], " #tmp "\n" \ + "add" # mod " " #sum ", " #tmp ", " #sum "\n" + + __asm( + "and %5, 3, %3\n" + "andn %5, 3, %1\n" + "brz,pt %3, 0f\n" + " lduw [%1], %0\n" + "mov 4, %4\n" + "sub %4, %3, %4\n" + "sll %4, 3, %4\n" /* fix up unaligned buffers */ + "mov 1, %2\n" + "sll %2, %4, %4\n" + "sub %4, 1, %4\n" + "lduw [%1 + 20], %2\n" + "andn %2, %4, %2\n" + "and %0, %4, %0\n" + "or %0, %2, %0\n" + "0:\n" + __LD_ADD(%1, %2, %0, 4, cc) + __LD_ADD(%1, %2, %0, 8, ccc) + __LD_ADD(%1, %2, %0, 12, ccc) + __LD_ADD(%1, %2, %0, 16, ccc) + "addc %0, 0, %0\n" /* reduce */ + "1:\n" + "sll %0, 16, %2\n" + "addcc %0, %2, %0\n" + "srl %0, 16, %0\n" + "addc %0, 0, %0\n" + "andcc %3, 1, %3\n" /* need to byte-swap? */ + "clr %3\n" + "bne,a,pn %%xcc, 1b\n" + " sll %0, 8, %0\n" + "not %0\n" + "sll %0, 16, %0\n" + "srl %0, 16, %0\n" + : "=&r" (__ret), "=r" (__tmp1), "=&r" (__tmp2), "=&r" (__tmp3), + "=&r" (__tmp4) : "1" (ip) : "cc"); +#undef __LD_ADD + return (__ret); +} + +#ifdef _KERNEL +u_short in_cksum_skip(struct mbuf *m, int len, int skip); +#endif + +#endif /* _MACHINE_IN_CKSUM_HH_ */ diff --git a/freebsd/sparc64/sparc64/in_cksum.c b/freebsd/sparc64/sparc64/in_cksum.c new file mode 100644 index 00000000..5490d005 --- /dev/null +++ b/freebsd/sparc64/sparc64/in_cksum.c @@ -0,0 +1,264 @@ +#include + +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/*- + * Copyright (c) 2001 by Thomas Moestl . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE + * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * from tahoe: in_cksum.c 1.2 86/01/05 + * from: @(#)in_cksum.c 1.3 (Berkeley) 1/19/91 + * from: FreeBSD: src/sys/i386/i386/in_cksum.c,v 1.22 2000/11/25 + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include + +#include +#include +#include + +#include + +/* + * Checksum routine for Internet Protocol family headers. + * + * This routine is very heavily used in the network + * code and should be modified for each CPU to be as fast as possible. + * + * This implementation is a sparc64 version. Most code was taken over + * and adapted from the i386. Some optimizations were changed to achieve + * (hopefully) better performance. + * This uses 64-bit loads, but 32-bit additions due to the lack of a 64-bit + * add-with-carry operation. + */ + +/* + * REDUCE() is actually not used that frequently... maybe a C implementation + * would suffice. + */ +#define REDUCE(sum, tmp) __asm( \ + "sll %2, 16, %1\n" \ + "addcc %2, %1, %0\n" \ + "srl %0, 16, %0\n" \ + "addc %0, 0, %0" : "=r" (sum), "=&r" (tmp) : "0" (sum) : "cc") + +/* + * Note that some of these macros depend on the flags being preserved + * between calls, thus they have to be used within a single __asm(). + */ +#define LD64_ADD32(n, mod) \ + "ldx [%3 + " #n "], %1\n" \ + "add" #mod " %2, %1, %0\n" \ + "srlx %1, 32, %1\n" \ + "addccc %0, %1, %0\n" + +#define LD32_ADD32(n, mod) \ + "lduw [%3 + " #n "], %1\n" \ + "add" #mod " %2, %1, %0\n" + +#define MOP(sum, tmp, addr) \ + "addc %2, 0, %0" \ + : "=r" (sum), "=&r" (tmp) : "0" (sum), "r" (addr) : "cc" + +u_short +in_cksum_skip(struct mbuf *m, int len, int skip) +{ + u_short *w; + unsigned long tmp, sum = 0; + int mlen = 0; + int byte_swapped = 0; + u_short su = 0; + + len -= skip; + for (; skip > 0 && m != NULL; m = m->m_next) { + if (m->m_len > skip) { + mlen = m->m_len - skip; + w = (u_short *)(mtod(m, u_char *) + skip); + goto skip_start; + } else + skip -= m->m_len; + } + + for (; m != NULL && len > 0; m = m->m_next) { + if (m->m_len == 0) + continue; + w = mtod(m, u_short *); + if (mlen == -1) { + /* + * The first byte of this mbuf is the continuation + * of a word spanning between this mbuf and the + * last mbuf. + * + * The high order byte of su is already saved when + * scanning previous mbuf. sum was REDUCEd when we + * found mlen == -1 + */ + sum += su | *(u_char *)w; + w = (u_short *)((u_char *)w + 1); + mlen = m->m_len - 1; + len--; + } else + mlen = m->m_len; +skip_start: + if (len < mlen) + mlen = len; + len -= mlen; + /* + * Force to a 8-byte boundary first so that we can use + * LD64_ADD32. + */ + if (((u_long)w & 7) != 0) { + REDUCE(sum, tmp); + if (((u_long)w & 1) != 0 && mlen >= 1) { + sum <<= 8; + su = *(u_char *)w << 8; + w = (u_short *)((u_char *)w + 1); + mlen--; + byte_swapped = 1; + } + if (((u_long)w & 2) != 0 && mlen >= 2) { + sum += *w++; + mlen -= 2; + } + if (((u_long)w & 4) != 0 && mlen >= 4) { + __asm( + LD32_ADD32(0, cc) + MOP(sum, tmp, w) + ); + w += 2; + mlen -= 4; + } + } + /* + * Do as much of the checksum as possible 64 bits at at time. + * In fact, this loop is unrolled to make overhead from + * branches &c small. + */ + for (; mlen >= 64; mlen -= 64) { + __asm( + LD64_ADD32(0, cc) + LD64_ADD32(8, ccc) + LD64_ADD32(16, ccc) + LD64_ADD32(24, ccc) + LD64_ADD32(32, ccc) + LD64_ADD32(40, ccc) + LD64_ADD32(48, ccc) + LD64_ADD32(56, ccc) + MOP(sum, tmp, w) + ); + w += 32; + } + if (mlen >= 32) { + __asm( + LD64_ADD32(0, cc) + LD64_ADD32(8, ccc) + LD64_ADD32(16, ccc) + LD64_ADD32(24, ccc) + MOP(sum, tmp, w) + ); + w += 16; + mlen -= 32; + } + if (mlen >= 16) { + __asm( + LD64_ADD32(0, cc) + LD64_ADD32(8, ccc) + MOP(sum, tmp, w) + ); + w += 8; + mlen -= 16; + } + if (mlen >= 8) { + __asm( + LD64_ADD32(0, cc) + MOP(sum, tmp, w) + ); + w += 4; + mlen -= 8; + } + REDUCE(sum, tmp); + while ((mlen -= 2) >= 0) + sum += *w++; + if (byte_swapped) { + sum <<= 8; + byte_swapped = 0; + if (mlen == -1) { + su |= *(u_char *)w; + sum += su; + mlen = 0; + } else + mlen = -1; + } else if (mlen == -1) { + /* + * This mbuf has odd number of bytes. + * There could be a word split between + * this mbuf and the next mbuf. + * Save the last byte (to prepend to next mbuf). + */ + su = *(u_char *)w << 8; + } + } + + if (len) + printf("%s: out of data by %d\n", __func__, len); + if (mlen == -1) { + /* + * The last mbuf has odd # of bytes. Follow the + * standard (the odd byte is shifted left by 8 bits). + */ + sum += su & 0xff00; + } + REDUCE(sum, tmp); + return (~sum & 0xffff); +}