source: build-files/freebsd-update/scripts/build.subr @ 635dc54

9.2-releasereleng/10.0releng/10.0.1releng/10.0.2releng/10.0.3releng/10.1
Last change on this file since 635dc54 was 635dc54, checked in by Kris Moore <kris@…>, 17 months ago

Add our freebsd-update code to GIT

This will become a dumping ground for mods we may make to the freebsd-update
build process, as well as the specific patches we have going into a release

  • Property mode set to 100644
File size: 35.7 KB
Line 
1#-
2# Copyright 2006 Colin Percival
3# All rights reserved
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted providing that the following conditions
7# are met:
8# 1. Redistributions of source code must retain the above copyright
9#    notice, this list of conditions and the following disclaimer.
10# 2. Redistributions in binary form must reproduce the above copyright
11#    notice, this list of conditions and the following disclaimer in the
12#    documentation and/or other materials provided with the distribution.
13#
14# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
16# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
18# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
22# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
23# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
24# POSSIBILITY OF SUCH DAMAGE.
25
26# $FreeBSD$
27
28# FreeBSD Update build subroutines.
29#
30# All the actual work gets done by functions defined here
31# and optionally overridden by release-specific and/or
32# release-and-platform-specific functions.
33#
34# Sourcing this is the first thing which scripts do.
35
36### Initialization
37
38# Read global configuration and set global variables.  Run from
39# readconfig(), so most scripts don't need to call this directly.
40readglobalconfig () {
41        # Set paths
42        export SCRIPTDIR="`dirname $0`"
43        export BASEDIR="`realpath ${SCRIPTDIR}/..`"
44        export BINDIR=${BASEDIR}/bin
45        export SRCDIR=${BASEDIR}/src
46        export KEYDIR=${BASEDIR}/keys
47        export PRIVKEYDIR=${KEYDIR}/mnt
48
49        # Read global configuration
50        . ${SCRIPTDIR}/build.conf
51}
52
53# Read configuration and subroutine override files for FreeBSD/$1 $2.
54readconfig () {
55        # Make sure we have a target platform specified
56        export TARGET=$1
57        if [ -z "${TARGET}" ]; then
58                echo "Target platform must be set"
59                exit 1
60        fi
61
62        # Make sure we have a release specified
63        export REL=$2
64        if [ -z "${REL}" ]; then
65                echo "Release name must be set"
66                exit 1
67        fi
68
69        # Get global configuration and variables.
70        readglobalconfig
71
72        # Set paths
73        export PATCHDIR=${BASEDIR}/patches/${REL}
74        export WORKDIR=${BASEDIR}/work/${REL}/${TARGET}
75        export PUBDIR=${BASEDIR}/pub/${REL}/${TARGET}
76        export STAGEDIR=${WORKDIR}/stage
77        export TMPDIR=${WORKDIR}/tmp
78
79        # Default TARGET_ARCH is TARGET.  On pc98, this needs to be
80        # overridden in the platform-specific configuration.
81        export TARGET_ARCH=${TARGET}
82
83        # RELP is REL, unless it is changed to be REL-pX instead.
84        export RELP=${REL}
85
86        # Make sure the target/release pair make sense.
87        if ! [ -f ${SCRIPTDIR}/${REL}/${TARGET}/build.conf ]; then
88                echo "No configuration available for FreeBSD/${TARGET} ${REL}"
89                exit 1
90        fi
91
92        # Read release/platform specific configuration
93        . ${SCRIPTDIR}/${REL}/${TARGET}/build.conf
94
95        # Read release and release-and-platform specific routines, if any.
96        if [ -f ${SCRIPTDIR}/${REL}/build.subr ]; then
97                . ${SCRIPTDIR}/${REL}/build.subr
98        fi
99        if [ -f ${SCRIPTDIR}/${REL}/${TARGET}/build.subr ]; then
100                . ${SCRIPTDIR}/${REL}/${TARGET}/build.subr
101        fi
102}
103
104### Utility subroutines
105
106# Logging function
107log () {
108        echo "`date` $1 for FreeBSD/${TARGET} ${RELP}"
109}
110
111# Function for nuking a directory
112nuke () {
113        rm -rf ${WORKDIR}/$1 2>/dev/null || true
114        if [ -d ${WORKDIR}/$1 ]; then
115                chflags -R noschg ${WORKDIR}/$1
116                rm -rf ${WORKDIR}/$1
117        fi
118}
119
120# Nuke a world and delete its associated index.
121removeworld () {
122        nuke $1
123        rm -f ${WORKDIR}/$1-index
124}
125
126### Key handling
127
128# Create a small memory disk and mount it on ${PRIVKEYDIR}
129mountkeydir () {
130        if [ -e ${PRIVKEYDIR}/.snap ]; then
131                echo "Private key directory is already mounted"
132                exit 1
133        fi
134
135        KEYMD=`mdconfig -a -t malloc -s 1M -n`
136        newfs /dev/md${KEYMD} > /dev/null
137        mount /dev/md${KEYMD} ${PRIVKEYDIR}
138        chmod 700 ${PRIVKEYDIR}
139}
140
141# Unmount the mini-filesystem containing the unencrypted private key
142umountkeydir () {
143        if ! [ -e ${PRIVKEYDIR}/.snap ]; then
144                echo "Private key directory is not mounted"
145                exit 1
146        fi
147
148        KEYDEV=`df ${PRIVKEYDIR} | tail -1 | awk '{ print $1 }'`
149        case ${KEYDEV} in
150        /dev/md*)
151                ;;
152        *)
153                echo "${PRIVKEYDIR} isn't mounted on a memory disk!"
154                exit 1
155                ;;
156        esac
157
158        # unmount
159        umount ${KEYDEV}
160
161        # Overwrite the memory disk with random garbage before deleting it.
162        # This may prevent the key from lingering in memory; in particular,
163        # overwriting with zeros often won't, due to overly intelligent
164        # memory disk code.
165        dd if=/dev/urandom of=${KEYDEV} count=2048 2>/dev/null
166        mdconfig -d -u `echo ${KEYDEV} | cut -c 8-`
167}
168
169# Generate RSA key
170makekey () {
171        mountkeydir
172        openssl genrsa -F4 4096 > ${PRIVKEYDIR}/priv.ssl
173        echo
174        echo "Public key fingerprint:"
175        openssl rsa -in ${PRIVKEYDIR}/priv.ssl -pubout 2>/dev/null |
176            sha256
177        echo
178}
179
180# Make sure an unencrypted key is available
181checkkey () {
182        if ! [ -f ${PRIVKEYDIR}/priv.ssl ]; then
183                cat <<-EOF
184                        You must run mountkey.sh to make the signing
185                        key available for use first.
186                EOF
187                exit 1
188        fi
189}
190
191# Encrypt the key and write it to disk
192encryptkey () {
193        KEYOWNER=${SUDO_USER:-${USER}}
194
195        echo "Encrypting signing key for ${KEYOWNER}"
196        openssl enc -aes-256-cbc                                \
197            -in ${PRIVKEYDIR}/priv.ssl                          \
198            -out ${KEYDIR}/priv.ssl-${KEYOWNER}
199}
200
201# Decrypt the key
202decryptkey () {
203        KEYOWNER=${SUDO_USER:-${USER}}
204
205        echo "Decrypting signing key for ${KEYOWNER}"
206        openssl enc -d -aes-256-cbc                             \
207            -in ${KEYDIR}/priv.ssl-${KEYOWNER}                  \
208            -out ${PRIVKEYDIR}/priv.ssl
209}
210
211### Setup subroutines
212
213# Make directories needed once for all releases/platforms
214makeglobaldirs () {
215        mkdir -p ${BINDIR} ${BASEDIR}/logs ${KEYDIR} ${PRIVKEYDIR}
216        chmod 700 ${KEYDIR} ${PRIVKEYDIR}
217}
218
219# Make binaries
220makebin () {
221        ( cd ${SRCDIR} && make all install clean )
222}
223
224# Create directories needed for a specific release/platform.
225makedirs () {
226        mkdir -p ${WORKDIR} ${TMPDIR}                           \
227            ${WORKDIR}/oldfiles ${WORKDIR}/oldmeta              \
228            ${STAGEDIR} ${STAGEDIR}/f ${STAGEDIR}/m             \
229            ${STAGEDIR}/bp ${STAGEDIR}/tp ${STAGEDIR}/t         \
230            ${PUBDIR} ${PUBDIR}/f ${PUBDIR}/m                   \
231            ${PUBDIR}/bp ${PUBDIR}/tp ${PUBDIR}/t
232}
233
234# Create some empty databases.
235makedbs () {
236        touch ${WORKDIR}/INDEX-ALL ${WORKDIR}/INDEX-NEW         \
237            ${WORKDIR}/INDEX-OLD ${WORKDIR}/metadb              \
238            ${WORKDIR}/stampvalues ${WORKDIR}/stamplocations    \
239            ${WORKDIR}/stampedfiles ${WORKDIR}/stampedfiles.tgz
240}
241
242# Set up PUBDIR.
243makepub () {
244        # Create an "all files earlier than this have been uploaded" marker
245        # and wait a second to make sure other files will have timestamps
246        # after (rather than equal to) this.
247        touch ${PUBDIR}/uploaded
248        sleep 1
249
250        # Create some files which aren't used by FreeBSD Update but are
251        # useful nonetheless.  The Expires directives help caching HTTP
252        # proxies to save bandwidth without being out of date, while
253        # the 'Options -Indexes' directive works around a problem with
254        # Apache and large directory listings (Apache uses the regular
255        # memory allocator to allocate space for each directory entry,
256        # and has no way to tell the OS that it no longer needs the
257        # many MB which it allocated).
258        cat <<-EOF > ${PUBDIR}/.htaccess
259                ExpiresActive on
260                ExpiresDefault "access plus 60 seconds"
261                Options -Indexes
262        EOF
263        echo 'ExpiresDefault "access plus 1 week"' > ${PUBDIR}/f/.htaccess
264        echo 'ExpiresDefault "access plus 1 week"' > ${PUBDIR}/m/.htaccess
265        echo 'ExpiresDefault "access plus 1 week"' > ${PUBDIR}/bp/.htaccess
266        echo 'ExpiresDefault "access plus 1 week"' > ${PUBDIR}/tp/.htaccess
267        echo 'ExpiresDefault "access plus 1 week"' > ${PUBDIR}/t/.htaccess
268}
269
270### Operation order checking routines
271
272# Make sure that the findstamps binary has been built.  This more or
273# less automatically means that the unstamp binary has been built, so
274# we won't bother checking for that one.
275checkbins () {
276        if ! [ -f ${BINDIR}/findstamps ]; then
277                cat <<-EOF
278                        You must run make.sh to build some binaries
279                        before running init.sh.
280                EOF
281                exit 1
282        fi
283}
284
285# Make sure that init has been run before running diff
286checkinit () {
287        if ! [ -s ${WORKDIR}/INDEX-ALL ]; then
288                cat <<-EOF
289                        You must run init.sh to fetch and extract the
290                        release before running diff.sh.
291                EOF
292                exit 1
293        fi
294}
295
296### Main functions
297
298# Download and verify a release ISO image.
299fetchiso () {
300        log "Starting fetch"
301
302        # Figure out where the disc1 ISO image is
303        RELNUM=${REL%-*}
304        ISO=${FTP}/${TARGET}/ISO-IMAGES/${RELNUM}/${REL}-${TARGET}-disc1.iso
305        ISO=${FTP}/FreeBSD-${REL}-${TARGET}-disc1.iso
306
307        # Fetch the ISO image.  We consider the ISO image to be
308        # the One True Release and don't look at the files used
309        # for FTP installs.  The FreeBSD 4.7-RELEASE ISO and FTP
310        # files were not identical, but this should never happen
311        # again.
312        fetch -o ${WORKDIR}/iso.img -rR ${ISO} 2>&1
313
314        log "Verifying disc1 hash"
315
316        # Check that the downloaded ISO has the correct hash.
317        if ! [ "`sha256 -q ${WORKDIR}/iso.img`" = "${RELH}" ]; then
318                echo "FreeBSD ${REL}-${TARGET}-disc1.iso has incorrect hash."
319                rm ${WORKDIR}/iso.img
320                return 1
321        fi
322}
323
324# Extract the released trees and, if appropriate, construct a world (base
325# plus source code) in which to perform builds.
326extractiso () {
327        # Create and mount a md(4) attached to the ISO image.
328        ISOMD=`mdconfig -a -t vnode -f ${WORKDIR}/iso.img -n`
329        mkdir -p ${WORKDIR}/iso
330        mount -t cd9660 -o ro,nosuid /dev/md${ISOMD} ${WORKDIR}/iso
331
332        # Extract the various components into different directories
333        log "Extracting components"
334        for C in ${WORLDPARTS}; do
335                mkdir -p ${WORKDIR}/release/R/trees/world/${C}
336                cat ${WORKDIR}/iso/${REL}/${C}/${C}.?? |
337                    tar -xpzf - -C ${WORKDIR}/release/R/trees/world/${C}
338        done
339        for C in ${KERNELPARTS}; do
340                mkdir -p ${WORKDIR}/release/R/trees/kernel/${C}
341                cat ${WORKDIR}/iso/${REL}/kernels/${C}.?? |
342                    tar -xpzf - -C ${WORKDIR}/release/R/trees/kernel/${C}
343        done
344        for C in ${SOURCEPARTS}; do
345                mkdir -p ${WORKDIR}/release/R/trees/src/${C}
346                cat ${WORKDIR}/iso/${REL}/src/s${C}.?? |
347                    tar -xpzf - -C ${WORKDIR}/release/R/trees/src/${C}
348        done
349
350        # If the release ISO we're handling belongs to the platform
351        # we're running right now, create a world image for future use.
352        if [ ${TARGET} = ${HOSTPLATFORM} ]; then
353                log "Constructing world+src image"
354
355                # Create directory for world
356                mkdir ${WORKDIR}/world/
357
358                # Extract world and source distributions
359                for C in ${WORLDPARTS}; do
360                        cat ${WORKDIR}/iso/${REL}/${C}/${C}.?? |
361                            tar -xpzf - -C ${WORKDIR}/world/
362                done
363                for C in ${SOURCEPARTS}; do
364                        cat ${WORKDIR}/iso/${REL}/src/s${C}.?? |
365                            tar -xpzf - -C ${WORKDIR}/world/usr/src/
366                done
367
368                # build a single tarball of them.
369                tar -czf ${WORKDIR}/../world.tgz -C ${WORKDIR}/world .
370
371                # clean up
372                nuke world
373        fi
374
375        # Unmount and detach the ISO image md(4).
376        umount ${WORKDIR}/iso
377        rmdir ${WORKDIR}/iso
378        mdconfig -d -u ${ISOMD}
379}
380
381# Extract to ${WORKDIR}/$1 a world in which to perform builds.
382extractworld () {
383        # Clean old world, if necessary (e.g., if a build was interrupted)
384        if [ -d ${WORKDIR}/$1 ]; then
385                log "Removing old world+src"
386                removeworld $1
387        fi
388
389        log "Extracting world+src"
390        mkdir -p ${WORKDIR}/$1
391        tar -xpzf ${WORKDIR}/../world.tgz -C ${WORKDIR}/$1
392}
393
394# Apply a list of patches stored in $2 to a world in ${WORKDIR}/$1
395applypatches () {
396        while read PATCH; do
397                cat ${PATCHDIR}/${PATCH} |
398                        ( cd ${WORKDIR}/$1/usr/src && patch -b '.removeme' -E -p0 )
399        done < $2 2> ${WORKDIR}/$1-patch.log
400
401        oPwd=`pwd`
402        cd ${WORKDIR}/${1}/usr/src
403        for i in `find . | grep '.removeme'`
404        do
405           echo "Removing ${i}"
406           rm ${i}
407        done
408        cd $oPwd
409}
410
411# In a world at ${WORKDIR}/$1, edit src/sys/conf/newvers.sh to
412# 1. Add $2 to the end of the BRANCH= line; and
413# 2. Revert the RELEASE= line back to what it is in CVS.
414# Note that the RELEASE= line is modified by src/release/Makefile
415# at release time.
416patchnewvers () {
417        sed -i "" -E "s,^(BRANCH=.*)\",\1$2\","                 \
418            ${WORKDIR}/$1/usr/src/sys/conf/newvers.sh
419        sed -i "" -E 's,^(RELEASE=).*,\1"${REVISION}-${BRANCH}",'       \
420            ${WORKDIR}/$1/usr/src/sys/conf/newvers.sh
421}
422
423# Apply "release" patches to a world in ${WORKDIR}/$1.  These "release"
424# patches are special patches which we want to pretend were in the
425# release all along -- they don't result in newly built files being
426# distributed, since we're pretending that these patches weren't added
427# after the release.
428rpatchworld () {
429        ls ${PATCHDIR} |
430            grep -E '^0-' |
431            cat > ${WORKDIR}/patchlist.tmp
432        applypatches $1 ${WORKDIR}/patchlist.tmp
433        rm ${WORKDIR}/patchlist.tmp
434
435        # Edit src/sys/conf/newvers.sh
436        patchnewvers $1 ""
437}
438
439# Apply all available patches to a world in ${WORKDIR}/$1
440patchworld () {
441        # Apply patches, in order of increasing patch level.  The
442        # order matters in case we have multiple security advisories
443        # affecting the same code.
444        ls ${PATCHDIR} |
445            sort -n > ${WORKDIR}/patchlist.tmp
446        applypatches $1 ${WORKDIR}/patchlist.tmp
447        rm ${WORKDIR}/patchlist.tmp
448
449        # Edit src/sys/conf/newvers.sh
450        patchnewvers $1 "-p${PNUM}"
451}
452
453# Perform a build in ${WORKDIR}/$1 with BRANCH_OVERRIDE set to $2
454buildworld () {
455        # We need a devfs inside the jail.  Note that we are using a
456        # jail here in order to keep the environment as "clean" as
457        # possible, not for security reasons; we assume that the
458        # original source code plus patches we add personally will
459        # not do anything evil.
460        mount -t devfs devfs ${WORKDIR}/$1/dev
461
462        # We need to be able to set file flags
463        sysctl security.jail.chflags_allowed=1 >/dev/null
464
465        # Build stuff.
466        jail ${WORKDIR}/$1 ${BUILDHOSTNAME} 127.1.2.3                   \
467            /usr/bin/env -i PATH=${PATH} RELP=${RELP}                   \
468                JFLAG=${JFLAG}                                          \
469                BRANCH_OVERRIDE=$2                                      \
470                TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH}             \
471            /bin/sh -e <<-"EOF" 2>&1 >${WORKDIR}/$1-build.log
472                # Releases build and distribute catpages
473                export MANBUILDCAT=YES
474
475                # Compat flags, for pre-6.0
476                export COMPATFLAGS="COMPAT1X=yes COMPAT20=yes           \
477                    COMPAT21=yes COMPAT22=yes COMPAT3X=yes COMPAT4X=yes"
478
479                # Function for logging what we're doing
480                log () {
481                        echo "`date` $1 for FreeBSD/${TARGET} ${RELP}" 1>&2
482                }
483
484                # Build the world
485                log "Building world"
486                cd /usr/src &&
487                    make ${COMPATFLAGS} ${JFLAG} buildworld 2>&1
488
489                # Distribute the world
490                log "Distributing world"
491                cd /usr/src/release &&
492                    make obj &&
493                    make ${COMPATFLAGS} release.1 release.2 2>&1
494
495                # Build and distribute kernels
496                log "Building and distributing kernels"
497                cd /usr/src/release &&
498                    make ${JFLAG} release.3 2>&1
499
500                # Build "synthetic" world distributions
501                log "Constructing world components"
502                cd /usr/src/release &&
503                    make release.5 2>&1
504
505                # Distribute source
506                log "Distributing source"
507                cd /usr/src/release &&
508                    make release.7 2>&1
509        EOF
510
511        # Put all the components into the right places.  This could be
512        # merged with the previous block of jailed script, except that
513        # sh(1) seems to get confused and stop reading the here-document
514        # halfway through if I do that...
515        log "Moving components into staging area"
516        jail ${WORKDIR}/$1 ${BUILDHOSTNAME} 127.1.2.3                   \
517            /usr/bin/env -i PATH=${PATH}                                \
518                KERNELPARTS="${KERNELPARTS}"                            \
519                SOURCEPARTS="${SOURCEPARTS}"                            \
520            /bin/sh -e <<-"EOF" 2>&1 >>${WORKDIR}/$1-build.log
521                # Create area for uncompressed components
522                mkdir -p /R/trees
523
524                # Move world components into place
525                mv /R/stage/trees /R/trees/world
526
527                # Move kernel components into place
528                mkdir -p /R/trees/kernel
529                for C in ${KERNELPARTS}; do
530                        CTMP=`echo ${C} | tr 'a-z' 'A-Z'`
531                        mkdir -p /R/trees/kernel/${C}
532                        mv /R/stage/kernels/${CTMP} /R/trees/kernel/${C}
533                done
534
535                # Extract src components into place
536                mkdir -p /R/trees/src/
537                for C in ${SOURCEPARTS}; do
538                        mkdir -p /R/trees/src/${C}
539                        cat /R/stage/dists/src/s${C}.?? |
540                            tar -xpzf - -C /R/trees/src/${C}
541                done
542        EOF
543
544        # Get rid of the devfs we no longer need.
545        umount ${WORKDIR}/$1/dev
546}
547
548# Perform a build in ${WORKDIR}/$1, but with the date set 400 days
549# into the future.  Turn off NTP before we change the date and
550# turn it back on afterwards.
551futurebuildworld () {
552        # Turn off ntpd if necessary
553        if /etc/rc.d/ntpd status |
554            grep -q 'is running'; then
555                ntpd_was_running=1
556                log "Turning off NTP"
557                /etc/rc.d/ntpd stop >/dev/null
558        else
559                ntpd_was_running=0
560        fi
561
562        date -n `date -j -v+400d "+%y%m%d%H%M.%S"` >/dev/null
563        buildworld $1 FUTUREBUILD
564        date -n `date -j -v-400d "+%y%m%d%H%M.%S"` >/dev/null
565
566        # Turn ntpd back on, if appropriate
567        if [ ${ntpd_was_running} = 1 ]; then
568                log "Turning NTP back on"
569                /etc/rc.d/ntpd start >/dev/null
570        fi
571}
572
573# Compare ${WORKDIR}/release and ${WORKDIR}/$1, identify which parts
574# of the world|doc subcomponent are missing from the latter, and
575# build a tarball out of them.
576findextradocs () {
577        log "Identifying extra documentation"
578
579        ( cd ${WORKDIR}/release && find R ) |
580            sort > ${WORKDIR}/release-files
581        ( cd ${WORKDIR}/$1 && find R ) |
582            sort > ${WORKDIR}/$1-files
583        comm -23 ${WORKDIR}/release-files ${WORKDIR}/$1-files |
584            awk '
585                BEGIN { p = "/" }
586                { if ( match($0, p) != 1) {
587                        print $0
588                        p = sprintf("%s/", $0)
589                } }' |
590            grep -E '^R/trees/world/doc' |
591            cut -f 4- -d / > ${WORKDIR}/extradocs
592        rm ${WORKDIR}/release-files ${WORKDIR}/$1-files
593
594        echo
595        echo "Documentation not built from src:"
596        cut -f 2- -d / ${WORKDIR}/extradocs
597        echo
598
599        tar -czf ${WORKDIR}/extradocs.tgz               \
600            -C ${WORKDIR}/release/R/trees/world         \
601            -T ${WORKDIR}/extradocs
602        rm ${WORKDIR}/extradocs
603}
604
605# Add extra docs to ${WORKDIR}/$1
606addextradocs () {
607        log "Extracting extra docs"
608
609        cat ${WORKDIR}/extradocs.tgz |
610            chroot ${WORKDIR}/$1/                       \
611                tar -xpzf - -C /R/trees/world doc/
612}
613
614# Index ${WORKDIR}/$1/R/trees/ and write to ${WORKDIR}/$1-index.  The
615# index format is described below and the lines are in lexicographical
616# order.
617indexfiles () {
618        log "Indexing $1"
619        # Output lines of the form
620        #
621        # comp/scomp/path/to/file|type|inum|user|group|perm|flags|value
622        #
623        # where the fields have meaning as follows:
624        # comp = component (kernel, world, src)
625        # scomp = subcomponent (generic, smp, base, doc, sys, etc.)
626        # path/to/file = path to the file, symlink, or directory
627        # type = 'f' (file), 'L' (link), or 'd' (directory)
628        # inum = inode number
629        # user, group = owning user and group
630        # perm = permissions, in octal and including setuid bits
631        # flags = file flags, in octal
632        # value = link target (for a symlink), or SHA256 hash (for a file)
633
634        ( cd ${WORKDIR}/$1/R && find trees -mindepth 2 |
635                grep -vE '/obj$' |
636                while read F; do
637                        if [ -L ${F} ]; then
638                                echo -n "${F}|L|"
639                                stat -n -f '%i|%u|%g|%Mp%Lp|%Of|' ${F};
640                                readlink ${F};
641                        elif [ -f ${F} ]; then
642                                echo -n "${F}|f|"
643                                stat -n -f '%i|%u|%g|%Mp%Lp|%Of|' ${F};
644                                sha256 -q ${F};
645                        elif [ -d ${F} ]; then
646                                echo -n "${F}|d|"
647                                stat -f '%i|%u|%g|%Mp%Lp|%Of|' ${F};
648                        else
649                                echo "Unknown file type: ${F}"          \
650                                    >/dev/stderr
651                        fi
652                done |
653                sed -E 's,trees/([^/]+)/([^/|]+)/?,\1|\2|/,' |
654                sort -k 5,5 -t '|' > ${WORKDIR}/tmp-index
655        )
656
657        # By examining the inode numbers, convert the above index
658        # to one containing lines of the form
659        #
660        # comp|scomp|/path/to/file|type|user|group|perm|flags|value|hlink
661        #
662        # where the fields have meanings as above, plus
663        # hlink = (lexicographically) first file to which this file is
664        #         hard linked, or blank (if there is no earlier such file).
665
666        cut -f 3,5 -d '|' ${WORKDIR}/tmp-index |
667            sort -k 1,1 -t '|' |
668            sort -s -u -k 2,2 -t '|' |
669            join -1 2 -2 5 -t '|' - ${WORKDIR}/tmp-index |
670            awk -F \| -v OFS=\| '{
671                if ($2 == $5)
672                        print $3,$4,$5,$6,$7,$8,$9,$10,$11,""
673                else
674                        print $3,$4,$5,$6,$7,$8,$9,$10,$11,$2
675                }' |
676            sort > ${WORKDIR}/$1-index
677        rm ${WORKDIR}/tmp-index
678}
679
680# Compare release-index and $1-index and warn the user about discrepancies.
681diffwarn () {
682        # Generate lists of files
683        cut -f 1-3 -d '|' ${WORKDIR}/release-index |
684            sort > ${WORKDIR}/release-index-flist
685        cut -f 1-3 -d '|' ${WORKDIR}/$1-index |
686            sort > ${WORKDIR}/$1-index-flist
687
688        echo
689        echo "Files built but not released:"
690        comm -13 ${WORKDIR}/release-index-flist ${WORKDIR}/$1-index-flist |
691            tee ${WORKDIR}/tmp-unreleased-flist
692
693        echo "Files released but not built:"
694        comm -23 ${WORKDIR}/release-index-flist ${WORKDIR}/$1-index-flist
695
696        echo "Files which differ by more than contents:"
697        cut -f 1-8,10 -d '|' ${WORKDIR}/release-index |
698            sort > ${WORKDIR}/release-index-nohash
699        cut -f 1-8,10 -d '|' ${WORKDIR}/$1-index |
700            sort |
701            comm -23 - ${WORKDIR}/release-index-nohash |
702            cut -f 1-3 -d '|' |
703            sort |
704            comm -23 - ${WORKDIR}/tmp-unreleased-flist
705
706        echo "Files which differ between release and build:"
707        comm -23 ${WORKDIR}/$1-index ${WORKDIR}/release-index |
708            cut -f 1-3 -d '|' |
709            sort |
710            comm -23 - ${WORKDIR}/tmp-unreleased-flist
711        echo
712
713        # Clean up
714        rm ${WORKDIR}/$1-index-flist ${WORKDIR}/release-index-flist     \
715            ${WORKDIR}/release-index-nohash ${WORKDIR}/tmp-unreleased-flist
716}
717
718# Compare $1 and $2 to find stamps
719findstamps () {
720        log "Locating build stamps"
721
722        # Make sure that the only difference is in file hashes.
723        cut -f 1-8,10 -d '|' < ${WORKDIR}/$1-index              \
724            > ${WORKDIR}/$1-index-nohash
725        cut -f 1-8,10 -d '|' < ${WORKDIR}/$2-index              \
726            > ${WORKDIR}/$2-index-nohash
727        if ! cmp -s ${WORKDIR}/$1-index-nohash                  \
728            ${WORKDIR}/$1-index-nohash; then
729                echo -n "Current and future builds differ "
730                echo "by more than just hashes!"
731                exit 1
732        fi
733        rm ${WORKDIR}/$1-index-nohash ${WORKDIR}/$2-index-nohash
734
735        # Generate list of files with varying hashes.
736        comm -23 ${WORKDIR}/$1-index ${WORKDIR}/$2-index |
737            cut -f 1-3 -d '|' |
738            sort > ${WORKDIR}/stampedfiles.new
739
740        # Construct a tarball containing the stamped files.
741        cat ${WORKDIR}/stampedfiles.new |
742            tr -s '|' '/' |
743            tar -czf ${WORKDIR}/stampedfiles.tgz.new    \
744                -C ${WORKDIR}/$1/R/trees -T -
745
746        # Find stamps!
747        tr '|' ' ' < ${WORKDIR}/stampedfiles.new |
748            while read C SC F; do
749                FP="R/trees/${C}/${SC}${F}"
750                if file -b ${WORKDIR}/$1/${FP} | grep -q "text"; then
751                        ${BINDIR}/findstamps -t         \
752                            ${WORKDIR}/$1/${FP} ${WORKDIR}/$2/${FP} |
753                            lam -s "${C}|${SC}|${F}|t|" -
754                else
755                        ${BINDIR}/findstamps            \
756                            ${WORKDIR}/$1/${FP} ${WORKDIR}/$2/${FP} |
757                            lam -s "${C}|${SC}|${F}|b|" -
758                fi
759            done > ${WORKDIR}/stamplocations.new        \
760                2> ${WORKDIR}/stampvalues.new
761}
762
763# Compare $1 and release to find differences in non-stamped files.
764findnonstamps () {
765        # release-filemap contains lines of the form
766        # ${C}|${SC}|${FP}|${RH}|${BH}
767        # meaning that the file ${FP} in the ${C}|${SC} component has
768        # hash ${RH} in the release, but hash ${BH} in the local build.
769        comm -23 ${WORKDIR}/$1-index ${WORKDIR}/release-index |
770            cut -f 1-3 -d '|' |
771            sort |
772            comm -23 - ${WORKDIR}/stampedfiles.new |
773            tr '|' ' ' |
774            while read C SC F; do
775                FP="R/trees/${C}/${SC}${F}"
776
777                echo "${C}|${SC}|${F}|"
778                sha256 -q ${WORKDIR}/release/${FP}
779                echo "|"
780                sha256 -q ${WORKDIR}/$1/${FP}
781            done |
782            lam - - - - > ${WORKDIR}/release-filemap
783}
784
785# Fixup paths: kernels are installed under /boot/, while src is
786# installed under /usr/src/.
787indexpublish () {
788        sed -E '
789                s,src\|([^|]+)\|/,src|\1|/usr/src/,
790                s,src\|(.*)\|/([^|]*)$,src|\1|/usr/src/\2,
791                s,kernel\|([^|]+)\|/,kernel|\1|/boot/,
792                s,kernel\|(.*)\|/([^|]*)$,kernel|\1|/boot/\2,
793            '
794}
795
796# If a parameter is specified, move $1 to "newworld" and $1-index to
797# "newworld-index", removing anything previous newworld and index.
798# From the world "newworld" and associated index "newworld-index",
799# generate INDEX-*.new, copy data files into ${STAGEDIR}/f/ and
800# metadata files into ${STAGEDIR}/m/, and generate patches from earlier
801# data and metadata files into ${STAGEDIR}/bp/ and ${STAGEDIR}/tp/.
802stageworld () {
803        # If necessary, turn $1 into newworld
804        if ! [ $# -eq 0 ]; then
805                # Clean if necessary
806                if [ -d ${WORKDIR}/newworld ]; then
807                        removeworld newworld
808                fi
809
810                # Move $1 to newworld
811                mv ${WORKDIR}/$1 ${WORKDIR}/newworld
812                mv ${WORKDIR}/$1-index ${WORKDIR}/newworld-index
813        fi
814
815        # Anything in ${STAGEDIR} has been left behind from a
816        # previous build which was nor approved.
817        log "Cleaning staging area"
818        find ${STAGEDIR} -type f |
819            xargs rm
820
821        log "Preparing to copy files into staging area"
822
823        # INDEX-ALL.new is newworld-index.  Sort the index just in case
824        # it was modified by hand and unsorted at that point.
825        sort ${WORKDIR}/newworld-index > ${WORKDIR}/INDEX-ALL.new
826
827        # Generate INDEX-NEW.new.  Note that when performing an "init"
828        # run, this file will be overwritten later, since the files in
829        # the release aren't "new" to the user (although at this point
830        # they are new files as far as this script is concerned).
831        comm -23 ${WORKDIR}/INDEX-ALL ${WORKDIR}/INDEX-NEW |
832            comm -23 ${WORKDIR}/INDEX-ALL.new - > ${WORKDIR}/INDEX-NEW.new
833
834        # Generate INDEX-OLD.new.  This will contain:
835        # (a) Any lines which were at one point in INDEX-ALL but are no
836        # longer in that file, and
837        # (b) Lines of the form "${C}|${SC}|${F}|${T}||||||" with ${T}="-"
838        # and ${C}, ${SC}, ${F} corresponding to any such tuples in INDEX-ALL
839        # which at some point did not exist in INDEX-ALL (i.e., these lines
840        # represent formerly non-existant files/directories/symlinks).
841        comm -23 ${WORKDIR}/INDEX-ALL ${WORKDIR}/INDEX-ALL.new |
842            sort -u ${WORKDIR}/INDEX-OLD - > ${WORKDIR}/INDEX-OLD.tmp
843        cut -f 1-3 -d '|' < ${WORKDIR}/INDEX-ALL.new |
844            sort > ${WORKDIR}/INDEX-ALL.new.nodes
845        cut -f 1-3 -d '|' < ${WORKDIR}/INDEX-ALL |
846            sort |
847            comm -13 - ${WORKDIR}/INDEX-ALL.new.nodes |
848            lam - -s "|-||||||" |
849            sort -u ${WORKDIR}/INDEX-OLD.tmp - > ${WORKDIR}/INDEX-OLD.new
850        rm ${WORKDIR}/INDEX-OLD.tmp ${WORKDIR}/INDEX-ALL.new.nodes
851
852        # Copy bits of world into ${STAGEDIR}/f/
853        log "Copying data files into staging area"
854        comm -23 ${WORKDIR}/INDEX-NEW.new ${WORKDIR}/INDEX-NEW |
855            cut -f 1-4,9 -d '|' |
856            tr '|' ' ' |
857            while read C SC F T H; do
858                # We don't care about directories or symlinks
859                if ! [ ${T} = "f" ]; then
860                        continue;
861                fi
862
863                # Copy the file into the staging area
864                gzip < ${WORKDIR}/newworld/R/trees/${C}/${SC}/${F}      \
865                    > ${STAGEDIR}/f/${H}.gz
866
867                # Generate binary patches for it
868                look "${C}|${SC}|${F}|f|" ${WORKDIR}/INDEX-OLD.new |
869                    cut -f 9 -d '|' |
870                    while read OH; do
871                        gunzip < ${WORKDIR}/oldfiles/${OH}.gz           \
872                            > ${TMPDIR}/${OH}
873                        bsdiff ${TMPDIR}/${OH}                          \
874                            ${WORKDIR}/newworld/R/trees/${C}/${SC}/${F} \
875                            ${STAGEDIR}/bp/${OH}-${H}
876                        rm ${TMPDIR}/${OH}
877                    done
878            done
879
880        # Special handling of INDEX-(NEW|OLD) for "init" builds: If the
881        # currently existing INDEX-ALL is empty, we're populating our
882        # databases with the release files, none of which belong in the
883        # published list of "new" files; equally, there is no need to
884        # record the fact that formerly nonexistant nodes now exist.
885        if ! [ -s ${WORKDIR}/INDEX-ALL ]; then
886                : > ${WORKDIR}/INDEX-NEW.new
887                : > ${WORKDIR}/INDEX-OLD.new
888        fi
889
890        # Convert metadata into publishable format and build patches
891        log "Copying metadata files into staging area"
892        for M in INDEX-ALL INDEX-NEW INDEX-OLD; do
893                # Create publishable version
894                indexpublish < ${WORKDIR}/${M}.new              \
895                    > ${WORKDIR}/${M}.new.pub
896
897                # Copy to staging area
898                H=`sha256 -q ${WORKDIR}/${M}.new.pub`
899                gzip < ${WORKDIR}/${M}.new.pub                  \
900                    > ${STAGEDIR}/m/${H}.gz
901
902                # Search for old versions of the same metadata file...
903                grep "^${M}|" ${WORKDIR}/metadb |
904                    cut -f 2 -d '|' |
905                    while read OH; do
906                        # ... and build patches
907                        gunzip < ${WORKDIR}/oldmeta/${OH}.gz    \
908                            > ${TMPDIR}/${OH}
909
910                        # Identify the lines to remove.  Note the for any
911                        # "${C}|${SC}|${F}|", we either want to remove
912                        # all or none of the lines with that prefix; in
913                        # INDEX-ALL and INDEX-NEW, there is at most one
914                        # line with any such prefix, and in INDEX-OLD we
915                        # never remove lines.
916                        comm -23 ${TMPDIR}/${OH} ${WORKDIR}/${M}.new.pub |
917                            cut -f 1-3 -d '|' |
918                            lam -s '-' - -s '|' > ${TMPDIR}/${OH}-${H}
919
920                        # Identify the lines to add
921                        comm -13 ${TMPDIR}/${OH} ${WORKDIR}/${M}.new.pub |
922                            lam -s '+' - >> ${TMPDIR}/${OH}-${H}
923
924                        # Move to staging area and clean up.
925                        gzip < ${TMPDIR}/${OH}-${H}             \
926                            > ${STAGEDIR}/tp/${OH}-${H}.gz
927                        rm ${TMPDIR}/${OH} ${TMPDIR}/${OH}-${H}
928                    done
929        done
930
931        # Construct metadata index
932        log "Constructing metadata index and tag"
933        for M in INDEX-ALL INDEX-NEW INDEX-OLD; do
934                echo -n "${M}|"
935                sha256 -q ${WORKDIR}/${M}.new.pub
936        done > ${WORKDIR}/tINDEX.new
937
938        # Copy metadata index into the right place
939        TH=`sha256 -q ${WORKDIR}/tINDEX.new`
940        cp ${WORKDIR}/tINDEX.new ${STAGEDIR}/t/${TH}
941
942        # Remove published version of metadata files; we don't need
943        # them any more.
944        for M in INDEX-ALL INDEX-NEW INDEX-OLD; do
945                rm ${WORKDIR}/${M}.new.pub
946        done
947
948        # Construct tag of the form
949        # freebsd-update|i386|6.1-RELEASE|123|${TH}|${EOL}\n
950        # meaning "FreeBSD/i386 6.1-RELEASE-p123 is described by
951        # the tag file with hash ${TH}; and its EOL time is ${EOL}
952        # seconds after the epoch".  If the 4th field is "0", this
953        # is the original release.
954        echo -n 'freebsd-update|' > ${WORKDIR}/tag.new
955        echo -n "${TARGET}|${REL}|" >> ${WORKDIR}/tag.new
956        if [ -f ${WORKDIR}/patchnum.new ]; then
957                echo -n `cat ${WORKDIR}/patchnum.new` >> ${WORKDIR}/tag.new
958        else
959                echo -n "0" >> ${WORKDIR}/tag.new
960        fi
961        echo "|${TH}|${EOL}" >> ${WORKDIR}/tag.new
962}
963
964# Print a list of stamped files and filestamps located, in order to
965# allow the user to verify that everything is working properly.
966printstamps () {
967        echo
968        echo "Files found which include build stamps:"
969        cat ${WORKDIR}/stampedfiles.new
970        echo
971        echo "Values of build stamps, excluding library archive headers:"
972        cat ${WORKDIR}/stampvalues.new |
973            grep -vE '.*/[0-9]* *[0-9]{10}  0     0     (0|100644)'
974}
975
976# Print a list of new updates, in order to allow the user to
977# verify that everything is working properly.
978printupdates () {
979        echo
980        echo "New updates:"
981        comm -23 ${WORKDIR}/INDEX-NEW.new ${WORKDIR}/INDEX-NEW
982}
983
984# Record the patch number of the build being performed.
985patchnumber () {
986        # Read old patch number, if any
987        if [ -f ${WORKDIR}/patchnum ]; then
988                OPN=`cat ${WORKDIR}/patchnum`
989        else
990                OPN=0
991        fi
992
993        # If the patch number is not specified, add one to the previous
994        if [ -z "$1" ]; then
995                export PNUM=$((${OPN} + 1))
996        else
997                export PNUM="$1"
998        fi
999
1000        # Patch number must be greater than previous number unless zero
1001        if [ ${OPN} -gt 0 ] &&
1002            ! [ ${PNUM} -gt ${OPN} ]; then
1003                echo -n "Patch number (${PNUM}) is not greater"
1004                echo " than previous (${OPN})"
1005                exit 1
1006        fi
1007
1008        # Record the new patch number
1009        echo ${PNUM} > ${WORKDIR}/patchnum.new
1010        export RELP="${REL}-p${PNUM}"
1011}
1012
1013# Approve the build: Sign it and place the signature and public key
1014# into ${PUBDIR}; update all of our internal databases, and copy
1015# files from staging area into ${PUBDIR}.
1016approve () {
1017        # Check that we have a build ready for approval
1018        if ! [ -f ${WORKDIR}/tag.new ]; then
1019                cat <<-EOF
1020                        There is no release ready for approval.  You
1021                        probably want to run "init.sh" or "diff.sh".
1022                EOF
1023                exit 1
1024        fi
1025
1026        # Sign the build
1027        log "Signing build"
1028        openssl rsautl -inkey ${PRIVKEYDIR}/priv.ssl -sign      \
1029            < ${WORKDIR}/tag.new                                \
1030            > ${STAGEDIR}/latest.ssl
1031
1032        # Export the public key
1033        openssl rsa -in ${PRIVKEYDIR}/priv.ssl -pubout          \
1034            > ${STAGEDIR}/pub.ssl 2>/dev/null
1035
1036        # Copy files from staging area to old files directories
1037        log "Copying files to patch source directories"
1038        find ${STAGEDIR}/f/ -type f |
1039            xargs -J % cp % ${WORKDIR}/oldfiles
1040        find ${STAGEDIR}/m/ -type f |
1041            xargs -J % cp % ${WORKDIR}/oldmeta
1042
1043        # Copy files into upload staging area.
1044        log "Copying files to upload staging area"
1045        tar -cf - -C ${STAGEDIR} . |
1046            tar -xmf - -C ${PUBDIR}
1047
1048        # Back up some databases, just in case
1049        log "Updating databases"
1050        for X in INDEX-ALL INDEX-NEW INDEX-OLD metadb           \
1051            stampedfiles stampedfiles.tgz stamplocations stampvalues; do
1052                cp ${WORKDIR}/${X} ${WORKDIR}/${X}.bak
1053        done
1054
1055        # Update databases
1056        for X in INDEX-ALL INDEX-NEW INDEX-OLD stampedfiles     \
1057            stampedfiles.tgz stamplocations stampvalues; do
1058                mv ${WORKDIR}/${X}.new ${WORKDIR}/${X}
1059        done
1060        if [ -f ${WORKDIR}/patchnum.new ]; then
1061                mv ${WORKDIR}/patchnum.new ${WORKDIR}/patchnum
1062        fi
1063        cat ${WORKDIR}/tINDEX.new >> ${WORKDIR}/metadb
1064
1065        # Clean up stuff we no longer need
1066        log "Cleaning staging area"
1067        find ${STAGEDIR} -type f |
1068            xargs rm
1069        rm ${WORKDIR}/tag.new ${WORKDIR}/tINDEX.new
1070}
1071
1072# Edit $1-index to
1073# * Revert binaries to the versions in the most recent shipped build if
1074#   they have not changed aside from buildstamps, and
1075# * Revert binaries to the versions shipped in the release if they have
1076#   not changed since the first build.
1077unstamp () {
1078        log "Reverting changes due to build stamps"
1079        # Extract old versions of stamped files
1080        mkdir -p ${WORKDIR}/oldstamps
1081        tar -xzf ${WORKDIR}/stampedfiles.tgz -C ${WORKDIR}/oldstamps
1082
1083        # Compare old and newly built stamped files, and generate lists
1084        # of lines to be removed and added to $1-index
1085        : > ${WORKDIR}/$1-index-add
1086        : > ${WORKDIR}/$1-index-delete
1087        tr '|' ' ' < ${WORKDIR}/stampedfiles |
1088            while read C SC FP; do
1089                # Copy old file into TMPDIR
1090                cp ${WORKDIR}/oldstamps/${C}/${SC}/${FP} ${TMPDIR}/old
1091
1092                # Copy new file into TMPDIR
1093                cp ${WORKDIR}/$1/R/trees/${C}/${SC}/${FP} ${TMPDIR}/new
1094
1095                # Remove text stamps
1096                grep "${C}|${SC}|${FP}|t|" ${WORKDIR}/stamplocations |
1097                    cut -f 5 -d '|' |
1098                    ${BINDIR}/unstamp -t ${TMPDIR}/old ${TMPDIR}/old.1
1099                grep "${C}|${SC}|${FP}|t|" ${WORKDIR}/stamplocations |
1100                    cut -f 5 -d '|' |
1101                    ${BINDIR}/unstamp -t ${TMPDIR}/new ${TMPDIR}/new.1
1102
1103                # Remove binary stamps
1104                grep "${C}|${SC}|${FP}|b|" ${WORKDIR}/stamplocations |
1105                    cut -f 5- -d '|' |
1106                    ${BINDIR}/unstamp ${TMPDIR}/old.1 ${TMPDIR}/old.2
1107                grep "${C}|${SC}|${FP}|b|" ${WORKDIR}/stamplocations |
1108                    cut -f 5- -d '|' |
1109                    ${BINDIR}/unstamp ${TMPDIR}/new.1 ${TMPDIR}/new.2
1110
1111                # Compare unstamped files
1112                if cmp -s ${TMPDIR}/old.2 ${TMPDIR}/new.2; then
1113                        look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index |
1114                            cut -f 1-8 -d '|' |
1115                            tr -d '\n' >> ${WORKDIR}/$1-index-add
1116                        echo -n '|' >> ${WORKDIR}/$1-index-add
1117                        look "${C}|${SC}|${FP}|" ${WORKDIR}/INDEX-ALL |
1118                            cut -f 9 -d '|' |
1119                            tr -d '\n' >> ${WORKDIR}/$1-index-add
1120                        echo -n '|' >> ${WORKDIR}/$1-index-add
1121                        look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index |
1122                            cut -f 10 -d '|' >> ${WORKDIR}/$1-index-add
1123
1124                        look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index    \
1125                            >> ${WORKDIR}/$1-index-delete
1126                fi
1127            done
1128
1129        # Remove lines from $1-index and add new lines.
1130        sort ${WORKDIR}/$1-index-delete |
1131            comm -23 ${WORKDIR}/$1-index - |
1132            sort - ${WORKDIR}/$1-index-add > ${WORKDIR}/$1-index.tmp
1133        mv ${WORKDIR}/$1-index.tmp ${WORKDIR}/$1-index
1134
1135        # Repeat the process, only replacing the hashes of locally built
1136        # files with the hashes of the released files where appropriate.
1137        : > ${WORKDIR}/$1-index-add
1138        : > ${WORKDIR}/$1-index-delete
1139        tr '|' ' ' < ${WORKDIR}/release-filemap |
1140            while read C SC FP RH BH; do
1141                if look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index |
1142                    grep -q "|${BH}|"; then
1143                        look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index |
1144                            cut -f 1-8 -d '|' |
1145                            tr -d '\n' >> ${WORKDIR}/$1-index-add
1146                        echo -n "|${RH}|" >> ${WORKDIR}/$1-index-add
1147                        look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index |
1148                            cut -f 10 -d '|' >> ${WORKDIR}/$1-index-add
1149
1150                        look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index    \
1151                            >> ${WORKDIR}/$1-index-delete
1152                fi
1153            done
1154        sort ${WORKDIR}/$1-index-delete |
1155            comm -23 ${WORKDIR}/$1-index - |
1156            sort - ${WORKDIR}/$1-index-add > ${WORKDIR}/$1-index.tmp
1157        mv ${WORKDIR}/$1-index.tmp ${WORKDIR}/$1-index
1158
1159        # Clean up
1160        rm ${WORKDIR}/$1-index-add ${WORKDIR}/$1-index-delete
1161        rm ${TMPDIR}/old ${TMPDIR}/old.1 ${TMPDIR}/old.2
1162        rm ${TMPDIR}/new ${TMPDIR}/new.1 ${TMPDIR}/new.2
1163        nuke oldstamps
1164}
1165
1166# Upload files to server.  Note that we upload latest.ssl last.
1167upload () {
1168        log "Uploading files"
1169
1170        # Upload files, and make sure the upload was successful
1171        tar -cf - --exclude latest.ssl --exclude uploaded       \
1172            --newer-mtime-than ${PUBDIR}/uploaded               \
1173            -C ${BASEDIR}/pub ${REL}/${TARGET} |
1174            ssh -i ${SSHKEY} ${MASTERACCT}                      \
1175                tar -xmf - -C ${MASTERDIR}
1176        if [ $? -ne 0 ]; then
1177                exit 1
1178        fi
1179
1180        # Upload latest.ssl, and make sure the upload was successful
1181        tar -cf - -C ${BASEDIR}/pub ${REL}/${TARGET}/latest.ssl |
1182            ssh -i ${SSHKEY} ${MASTERACCT}                      \
1183                tar -xmf - -C ${MASTERDIR}
1184        if [ $? -ne 0 ]; then
1185                exit 1
1186        fi
1187
1188        # Mark all files prior to now as having been uploaded
1189        touch ${PUBDIR}/uploaded
1190}
Note: See TracBrowser for help on using the repository browser.