view buildslave/tools/wget.cache.wrapper @ 15:72316564535a

wget.cache.wrapper: Fix racecondition and download a new copy if the URL changes. 2011-03-02 Xerxes Ranby <xerxes@zafena.se> * buildslave/tools/wget.cache.wrapper Fix racecondition when two or more builders are downloading the same file. Download a new copy if the URL changes.
author Xerxes R?nby <xerxes@zafena.se>
date Wed, 02 Mar 2011 11:45:22 +0100
parents 15bb449380ba
children 64749190d772
line wrap: on
line source

#!/bin/bash

# Copyright (C) 2011, Xerxes Ranby <xerxes@zafena.se>
# This wget cache wrapper is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
# http://www.gnu.org/copyleft/gpl.html

# This is a wrapper for wget that caches files fetched using
# wget foo -O files

# This wget cache wrapper are currently being run on the buildslaves 
# jaunty-ia32, natty-armv7l and squeeze-armv5tel connected to the
# http://builder.classpath.org/icedtea/buildbot/ buildmaster 
# to prevent the buildslaves to re-download all openjdk drops for each build

##### Please configure me to match your setup.

WGET=/usr/bin/wget.real
CACHEDIR=/home/wgetcache/

##### Please do not change anything below this line

TARGETDIR=$(pwd)
PID=$$

#remeber all arguments to wget
ALLARG=$*

#find out if the last argument are -O file
FILEINDEX=$#
shift $((${FILEINDEX}-3))
URL=`echo $1|tr -d '/?;'`
ARG=$2
FILE=$3

if [ ${ARG} == "-O" ] ; then
  if [ ! -e ${CACHEDIR}${URL}/${FILE} ]; then
    # file do not exist in cache
    # download the file to the cache using wget!

    # make sure subdirs in the cache exist
    CACHEDIRSUBPATH=$(dirname ${FILE})
    mkdir -p ${CACHEDIR}${PID}/${URL}/${CACHEDIRSUBPATH}
    mkdir -p ${CACHEDIR}${URL}/${CACHEDIRSUBPATH}

    # download the file to the current pids cache directory.
    cd ${CACHEDIR}${PID}/${URL}
    ${WGET} ${ALLARG}

    # atomically move the downloaded file to the main cache.
    mv ${CACHEDIR}${PID}/${URL}/${FILE} ${CACHEDIR}${URL}/
  fi

  if [ -e ${CACHEDIR}${URL}/${FILE} ] ; then
    # ok good, file exist now in cache
    # copy the file from cache to destination
    cd ${TARGETDIR}
    cp ${CACHEDIR}${URL}/${FILE} ${FILE}
    exit 0
  else
    #something got wrong
    exit 1
  fi
else
  # Last argument are something else than -O file
  # We do not know how to cache this.... sorry.
  # instead we simply run wget with all passed arguments
  # and pretend we dont exist
  ${WGET} ${ALLARG}
fi