Added some new features to slice up the import from remote CVS via local CVS

[tinyos-utils.git] / tinyos-mirror.sh

#!/bin/bash
# tinyos-mirror.sh - Maintain a local git mirror of the tinyos CVS repository.
#
# This utility allows updating a local mirror of the TinyOS CVS modules as
# a set of three git repositories, one per notable CVS module:
#       tinyos-1.x -> tinyos-1.x.git
#       tinyos-2.x -> tinyos-2.x.git
#       tinyos-2.x-contrib -> tinyos-2.x-contrib.git
#
# The utility actually allows both for creation and update of the local
# repos, with no pre-setup work other than ensuring proper permissions.
#
# The directories used by tinyos-mirror.sh must already exist and be owned by
# the user who runs tinyos-mirror.sh, and the directories must have their group
# set to that user's primary group, like so:
# 
#       drwxr-sr-x 5 sysadmin repo  60 2009-04-08 14:14 /var/lib/cvs
#       drwxr-xr-x 7 sysadmin repo 112 2009-04-05 15:05 /var/lib/git
#
# Please note the permissions on the above directories.  Also please note that
# any user or program wanting to access this information must be a member of
# the group 'repo', or whatever group is used in your installation.
#
# Configuration:
# 
# 1. Decide on the user and group to use.  The user to use must have its
#    primary group set to the group selected.
#
# 2. Create /var/lib/cvs and /var/lib/git, owned by the selected group and
#    user, and with the above permissions.
#
# 3. Verify the contents of the variables below which define the location of
#    the upstream sources, the modules to mirror, etc.
#
# Usage:
#
# tinyos-mirror.sh [--cvslocal|--cvsdirect|--localonly|--rsynconly] [module [module ...]]
#    No --xxx option implies update from upstream git.
#    No module(s) implies update of all as defined in $MODULES.
#
# 1. To create or update the local git repos from the upstream git source,
#    which is BY FAR the fastest option, run tinyos-mirror.sh with no arguments.
#
# 2. To create or update the local git repos from the upstream CVS source,
#    which may be more definitive but much slower, run tinyos-mirror.sh with
#    the --cvslocal option.
#       --rsynconly does only the rsync update of the remote CVS to local CVS.
#       --localonly does everything of --cvslocal except the rsync.
#
# 3. To update (not create!) the local git repos from the upstream CVS source
#    directly via pserver (see the UPCVSROOT variable), then run the
#    tinyos-mirror.sh script with the --cvsdirect option.  This strategy is
#    perhaps faster than option #2 above if relatively few changesets have
#    been added upstream since the last invocation of tinyos-mirror.sh.
#
# It should be possible to invoke tinyos-mirror.sh using different update
# methods without creating problems.  We have tested creating and initially
# populating the local git repos using --cvslocal and then following with
# periodic invocations using --cvsdirect.  Other combinations have not been
# tested.
#
# By far the fastest and most resource friendly option is to use an upstream
# git server.

# Ensure these variables are correct for your installation

MODULES="tinyos-1.x tinyos-2.x tinyos-2.x-contrib"
GITBASE=/var/lib/git
UPGITBASE=git://hinrg.cs.hju.edu/git
CVSBASE=/var/lib/cvs
UPCVSROOT=:pserver:anonymous@tinyos.cvs.sourceforge.net:/cvsroot/tinyos
UPCVSRSYNC=rsync://tinyos.cvs.sourceforge.net/cvsroot/tinyos/

# There should be no need to change anything below this line

do_rsync()
{
    echo "Update local CVS mirror via rsync"
    links=""
    if [ -d $CVSBASE/tinyos ]; then
        if [ -d $CVSBASE/tinyos.backup ]; then
            if [ -d $CVSBASE/tinyos.backup2 ]; then
                echo "Mv $CVSBASE/tinyos.backup2 $CVSBASE/tinyos.backup3"
                mv $CVSBASE/tinyos.backup2 $CVSBASE/tinyos.backup3
            fi
            echo "Mv $CVSBASE/tinyos.backup $CVSBASE/tinyos.backup2"
            mv $CVSBASE/tinyos.backup $CVSBASE/tinyos.backup2
            links="--link-dest=$CVSBASE/tinyos.backup2 $links"
        fi
        echo "Mv $CVSBASE/tinyos -> $CVSBASE/tinyos.backup"
        mv $CVSBASE/tinyos $CVSBASE/tinyos.backup
        links="--link-dest=$CVSBASE/tinyos.backup $links"
        if [ -d $CVSBASE/tinyos.backup3 ]; then
            echo "Mv $CVSBASE/tinyos.backup3 -> $CVSBASE/tinyos"
            mv $CVSBASE/tinyos.backup3 $CVSBASE/tinyos
        fi
    else
        echo "Cp $CVSBASE/tinyos.backup3 $CVSBASE/tinyos.backup"
        cp -al $CVSBASE/tinyos.backup $CVS/tinyos # ok to fail
    fi
    echo "Rsync from sourceforge -> $CVSBASE/tinyos"
    echo "    links: $links"
    rsync -avz --delete --no-g --chmod=Dg+s,Du+w $links \
        --exclude "CVSROOT/CVS" \
        $UPCVSRSYNC $CVSBASE/tinyos/
    if [ $? -ne 0 ]; then
        echo "rsync failed"
        exit 1
    fi
}

locks_present()
{
    lines=$(find . -name "#cvs*" | wc -l)
    if [ $lines -gt 0 ]; then
        return 0 # locks are present
    else
        return 1 # no locks present
    fi
}

do_bare()
{
    # Ensure the bare repos are present
    cd $GITBASE
    for i in $MODULES; do
        if [ ! -d $i.git ]; then
            echo "Create shared repo $i.git"
            rm -rf $i.git
            mkdir $i.git
            git --git-dir=$i.git init --bare
            if [ $? -ne 0 ]; then
                echo "git init failed"
                exit 1
            fi
            echo "Mirror of the $i CVS module" > $i.git/description

            # This line allows us to do cvsimport directly into this git repo,
            # as git-cvsimport doesn't know how to work with bare repos.
            ln -s ../$i.git $i.git/.git
        fi
    done
}

do_import()
{
    # Now, do the imports.  We limit the number of commits to prevent memory
    # leaks from taking out the VE.  This command is incremental, just adding
    # new commits since the last run.
    if [ ! -d $GITBASE/cvsimports ]; then
        rm -rf $GITBASE/cvsimports
        mkdir $GITBASE/cvsimports
        chgrp repo $GITBASE/cvsimports
        chmod 755 $GITBASE/cvsimports
        chmod g+s $GITBASE/cvsimports
    fi
    cd $GITBASE/cvsimports
    for i in $MODULES; do
        echo "Import cvs module $i -> git"
        git cvsimport -v -o master -d $CVSBASE/tinyos -C $i-import -i -k $i
        if [ $? -ne 0 ]; then
            echo "cvsimport failed"
            exit 1
        fi
        if [ ! -f $i-import/.git/description ]; then
            echo "Mirror of the $i CVS module" > $i.git/description
        fi
    done
}

do_push()
{
    # Now we need to push the new commits in each of the import repos into the
    # bare repos which are the pseudo-centralized shared repos.
    cd $GITBASE/cvsimports
    for i in $MODULES; do
        echo "Push git import module cvsimports/$i-import -> $i.git"
        git --git-dir=$i-import/.git push --mirror ../$i.git/
        if [ $? -ne 0 ]; then
            echo "git push failed"
            exit 1
        fi
    done
}

do_import_cvsdirect()
{
    # Do the imports, but directly with the sf.net CVS server.  The
    # sysadmin user must have first done a cvs login, and this script must
    # run as the sysadmin user.  We limit the number of commits to prevent
    # memory  leaks from taking out the VE.  This command is incremental, just
    # adding new commits since the last run.

    cd $GITBASE
    for i in $MODULES; do
        echo "Update from CVS module DIRECT $i -> git"
        git cvsimport -v -o master -d $UPCVSROOT -C $i.git -i -k -L2000 $i
        if [ $? -ne 0 ]; then
            echo "cvsimport failed"
            exit 1
        fi
    done
}

do_import_localonly()
{
    # Do the cvs-gitimports from the local CVS.

    if locks_present; then
        echo "Locks present in the CVS upstream.  We won't import this time."
    else
        do_bare
        do_import
        do_push
    fi
}

do_import_cvslocal()
{
    # Do the imports by first using rsync to sync a local CVS from sf.net,
    # then do cvs-gitimports from the local CVS.

    do_rsync
    do_import_localonly
}

do_import_git()
{
    # Update from the git repos at hinrg.  As of 4/8/09, only the tinyos-2.x
    # repository seems to be being updated from the upstream sf.net CVS module.
    # tinyos-1.x is 14 months old when the last update was 3/17/09, and
    # tinyos-2.x-contrib.git is 5 months old when the last update was 4/7/09.

    fail=0
    cd $GITBASE
    for i in $MODULES; do
        if [ ! -d $i.git ]; then
            rm -rf $i.git
            echo "Clone gitupstream -> $i.git"
            git clone --bare $UPGITBASE/$i.git
            if [ $? -ne 0 ]; then
                echo "git clone failed"
                fail=$(($fail + 1))
            fi

            # This line allows us to do cvsimport directly into this git repo,
            # as git-cvsimport doesn't know how to work with bare repos.
            ln -s ../$i.git $i.git/.git
        else
            echo "Pull gitupstream -> $i.git"
            git --git-dir=$i.git fetch $UPGITBASE/$i.git master:master
            if [ $? -ne 0 ]; then
                echo "git fetch failed"
                fail=$(($fail + 1))
            fi
        fi
    done
    if [ $fail -gt 0 ]; then
        echo "git import from git failed"
        exit $fail
    fi
}

# MAIN

unset option
unset opt_modules
while [ -n "$1" ]; do
    if echo "$1" | grep -q -- "--"; then
        if [ "$1" = "--cvslocal" -o "$1" = "--cvsdirect" -o "$1" = "--localonly" -o "$1" = "--rsynconly" ]; then
            if [ -z "$option" ]; then
                option=$1
            fi
        else
            echo "$0: invalid option $1" >&2
            exit 1
        fi
    else
        opt_modules="$opt_modules $1"
    fi
    shift
done
if [ -n "$opt_modules" ]; then
    MODULES="$opt_modules"
fi
echo "Updating modules $MODULES"

if [ "$option" = "--cvsdirect" ]; then
    do_import_cvsdirect
elif [ "$option" = "--cvslocal" ]; then
    do_import_cvslocal
elif [ "$option" = "--localonly" ]; then
    do_import_localonly
elif [ "$option" = "--rsynconly" ]; then
    do_rsync
else
    do_import_git
fi
exit 0