CRAB tutorial (advanced)

By default jobs cannot run more than 24 hours on the Grid worker nodes. They are automatically killed if their runtime (called wall clock 
time) exceeds this limit. That's what happened to the jobs; they were killed, because they reached the wall clock limit.

from CRABClient.UserUtilities import config
config = config()

config.General.requestName = 'CRAB3_Advanced_Tutorial_May2015_Exercise1'

config.JobType.psetName = 'PSet.py'
config.JobType.pluginName = 'Analysis'

config.Data.inputDataset = '/VBF_HToTauTau_M-125_13TeV-powheg-pythia6/Phys14DR-PU20bx25_tsg_PHYS14_25_V1-v2/MINIAODSIM'
config.Data.splitting = 'FileBased'
config.Data.unitsPerJob = 1

config.Site.storageSite = <site where you have permission to write>

The estimated time per event is 3 seconds.

Looking at the first file in the dataset, it has in average ~100 events per lumi. One way to get this number is to use DAS to get the list of lumis in the file [1] (and count the number of lumis for example with python). Then divide this number by the number of events in the file (you can also find it in DAS):

$ python
>>> lumis = [[3967, 3969], [3973, 3973], ...]
>>> sum([y-x+1 for x, y in lumis])
431
>>> 42902 / 431
99

[1] https://cmsweb.cern.ch/das/request?input=lumi%20file%3D/store/mc/Phys14DR/VBF_HToTauTau_M-125_13TeV-powheg-pythia6/MINIAODSIM/PU20bx25_tsg_PHYS14_25_V1-v2/00000/147B369C-9F77-E411-B99D-00266CF9B184.root&instance=prod/global&idx=0&limit=10

eventsPerLumi = 100
timePerEv = 4
desiredTime = 8*60*60
config.Data.unitsPerJob = desiredTime / (eventsPerLumi*timePerEv)

import FWCore.ParameterSet.Config as cms

process = cms.Process('NoSplit')

process.source = cms.Source("PoolSource", fileNames = cms.untracked.vstring())
process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(10))
process.options = cms.untracked.PSet(wantSummary = cms.untracked.bool(True))
process.output = cms.OutputModule("PoolOutputModule",
    outputCommands = cms.untracked.vstring("drop *", "keep recoTracks_*_*_*"),
    fileName = cms.untracked.string('output.root'),
)
process.out = cms.EndPath(process.output)

from CRABClient.UserUtilities import config
config = config()

config.General.requestName = 'CRAB3_Advanced_Tutorial_June2015_Exercise2A'
config.General.transferOutputs = True
config.General.transferLogs = False

config.JobType.pluginName = 'Analysis'
config.JobType.psetName = 'pset_tutorial_analysis.py'

config.Data.inputDataset = '/SingleMu/Run2012B-13Jul2012-v1/AOD'
config.Data.splitting = 'FileBased'
config.Data.unitsPerJob = 1
config.Data.totalUnits = 5
config.Data.publication = True
config.Data.outputDatasetTag = config.General.requestName

config.Site.storageSite = <site where you have permission to write>

Look for example in the job log files linked from the monitoring pages. In each job log file, at the very beginning, it should say what 
were the input files analyzed by the corresponding job.

Answer:
/store/data/Run2012B/SingleMu/AOD/13Jul2012-v1/0000/008DBED0-86D3-E111-AEDF-20CF3019DF17.root
/store/data/Run2012B/SingleMu/AOD/13Jul2012-v1/0000/00F9715A-A1D3-E111-BE6F-E0CB4E29C4D1.root
/store/data/Run2012B/SingleMu/AOD/13Jul2012-v1/0000/00100164-41D4-E111-981B-20CF3027A5AF.root
/store/data/Run2012B/SingleMu/AOD/13Jul2012-v1/0000/0093BEF2-A4D3-E111-A6B9-E0CB4E19F979.root
/store/data/Run2012B/SingleMu/AOD/13Jul2012-v1/0000/0009F1CC-0DD4-E111-974D-20CF305B0584.root

Search for the input dataset /SingleMu/Run2012B-13Jul2012-v1/AOD in DAS.

Answer:
T2_BE_UCL, T2_IT_Legnaro, T2_RU_JINR, T1_US_FNAL, T1_IT_CNAF

from CRABClient.UserUtilities import config
config = config()

config.General.requestName = 'CRAB3_Advanced_Tutorial_June2015_Exercise2B'
config.General.transferOutputs = True
config.General.transferLogs = False

config.JobType.pluginName = 'Analysis'
config.JobType.psetName = 'pset_tutorial_analysis.py'

config.Data.outputPrimaryDataset = 'SingleMu'
config.Data.userInputFiles = open('CRAB3_Advanced_Tutorial_June2015_Exercise2B_inputFiles.txt').readlines()
config.Data.splitting = 'FileBased'
config.Data.unitsPerJob = 1
config.Data.totalUnits = 5
config.Data.publication = True
config.Data.outputDatasetTag = config.General.requestName

config.Site.storageSite = <site where you have permission to write>
config.Site.whitelist = ['T2_BE_UCL', 'T2_IT_Legnaro', 'T2_RU_JINR']

Use 'crab getoutput --dump'.

The task has not been submitted to the Grid, so there is nothing to resubmit (or to recover). One has to submit the task again.

Since the issue was a temporary problem not related to the jobs (which were otherwise finishing without problems), resubmitting the failed 
jobs would be the most reasonable choice.

In this case the problem was with the jobs themselves. Resubmitting without changing the requested job runtime would not help, as jobs will 
most probably fail again. Resubmitting requesting a higher job runtime may cause the jobs to be queued for a long time before they start 
running. Submitting a new task would be a waste of resources as ~80% of the task has already completed. The best choice is to submit a 
recovery task to analyze only the failed jobs, but it must be with a finer splitting.

# Auto generated configuration file
# using: 
# Revision: 1.19 
# Source: /local/reps/CMSSW/CMSSW/Configuration/Applications/python/ConfigBuilder.py,v 
# with command line options: MinBias_8TeV_cfi --conditions auto:startup -s GEN,SIM --datatier GEN-SIM -n 10 
# --relval 9000,300 --eventcontent RAWSIM --io MinBias.io --python MinBias.py --no_exec --fileout minbias.root

import FWCore.ParameterSet.Config as cms

process = cms.Process('SIM')

# Import of standard configurations
process.load('Configuration.StandardSequences.Services_cff')
process.load('SimGeneral.HepPDTESSource.pythiapdt_cfi')
process.load('FWCore.MessageService.MessageLogger_cfi')
process.load('Configuration.EventContent.EventContent_cff')
process.load('SimGeneral.MixingModule.mixNoPU_cfi')
process.load('Configuration.StandardSequences.GeometryRecoDB_cff')
process.load('Configuration.Geometry.GeometrySimDB_cff')
process.load('Configuration.StandardSequences.MagneticField_38T_cff')
process.load('Configuration.StandardSequences.Generator_cff')
process.load('IOMC.EventVertexGenerators.VtxSmearedRealistic8TeVCollision_cfi')
process.load('GeneratorInterface.Core.genFilterSummary_cff')
process.load('Configuration.StandardSequences.SimIdeal_cff')
process.load('Configuration.StandardSequences.EndOfProcess_cff')
process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff')

process.maxEvents = cms.untracked.PSet(
    input = cms.untracked.int32(10)
)

# Input source
process.source = cms.Source("EmptySource")

process.options = cms.untracked.PSet(

)

# Production Info
process.configurationMetadata = cms.untracked.PSet(
    version = cms.untracked.string('$Revision: 1.26 $'),
    annotation = cms.untracked.string('MinBias_8TeV_cfi nevts:10'),
    name = cms.untracked.string('Applications')
)

# Output definition
process.RAWSIMoutput = cms.OutputModule("PoolOutputModule",
    splitLevel = cms.untracked.int32(0),
    eventAutoFlushCompressedSize = cms.untracked.int32(5242880),
    outputCommands = process.RAWSIMEventContent.outputCommands,
    fileName = cms.untracked.string('minbias.root'),
    dataset = cms.untracked.PSet(
        filterName = cms.untracked.string(''),
        dataTier = cms.untracked.string('GEN-SIM')
    ),
    SelectEvents = cms.untracked.PSet(
        SelectEvents = cms.vstring('generation_step')
    )
)

# Additional output definition

# Other statements
process.genstepfilter.triggerConditions=cms.vstring("generation_step")
from Configuration.AlCa.GlobalTag import GlobalTag
process.GlobalTag = GlobalTag(process.GlobalTag, 'auto:startup', '')

process.generator = cms.EDFilter("Pythia6GeneratorFilter",
    pythiaPylistVerbosity = cms.untracked.int32(0),
    filterEfficiency = cms.untracked.double(1.0),
    pythiaHepMCVerbosity = cms.untracked.bool(False),
    comEnergy = cms.double(8000.0),
    maxEventsToPrint = cms.untracked.int32(0),
    PythiaParameters = cms.PSet(
        pythiaUESettings = cms.vstring('MSTU(21)=1     ! Check on possible errors during program execution', 
            'MSTJ(22)=2     ! Decay those unstable particles', 
            'PARJ(71)=10 .  ! for which ctau  10 mm', 
            'MSTP(33)=0     ! no K factors in hard cross sections', 
            'MSTP(2)=1      ! which order running alphaS', 
            'MSTP(51)=10042 ! structure function chosen (external PDF CTEQ6L1)', 
            'MSTP(52)=2     ! work with LHAPDF', 
            'PARP(82)=1.921 ! pt cutoff for multiparton interactions', 
            'PARP(89)=1800. ! sqrts for which PARP82 is set', 
            'PARP(90)=0.227 ! Multiple interactions: rescaling power', 
            'MSTP(95)=6     ! CR (color reconnection parameters)', 
            'PARP(77)=1.016 ! CR', 
            'PARP(78)=0.538 ! CR', 
            'PARP(80)=0.1   ! Prob. colored parton from BBR', 
            'PARP(83)=0.356 ! Multiple interactions: matter distribution parameter', 
            'PARP(84)=0.651 ! Multiple interactions: matter distribution parameter', 
            'PARP(62)=1.025 ! ISR cutoff', 
            'MSTP(91)=1     ! Gaussian primordial kT', 
            'PARP(93)=10.0  ! primordial kT-max', 
            'MSTP(81)=21    ! multiple parton interactions 1 is Pythia default', 
            'MSTP(82)=4     ! Defines the multi-parton model'),
        processParameters = cms.vstring('MSEL=0         ! User defined processes', 
            'MSUB(11)=1     ! Min bias process', 
            'MSUB(12)=1     ! Min bias process', 
            'MSUB(13)=1     ! Min bias process', 
            'MSUB(28)=1     ! Min bias process', 
            'MSUB(53)=1     ! Min bias process', 
            'MSUB(68)=1     ! Min bias process', 
            'MSUB(92)=1     ! Min bias process, single diffractive', 
            'MSUB(93)=1     ! Min bias process, single diffractive', 
            'MSUB(94)=1     ! Min bias process, double diffractive', 
            'MSUB(95)=1     ! Min bias process'),
        parameterSets = cms.vstring('pythiaUESettings', 
            'processParameters')
    )
)

# Path and EndPath definitions
process.generation_step = cms.Path(process.pgen)
process.simulation_step = cms.Path(process.psim)
process.genfiltersummary_step = cms.EndPath(process.genFilterSummary)
process.endjob_step = cms.EndPath(process.endOfProcess)
process.RAWSIMoutput_step = cms.EndPath(process.RAWSIMoutput)

# Schedule definition
process.schedule = cms.Schedule(
    process.generation_step,
    process.genfiltersummary_step,
    process.simulation_step,
    process.endjob_step,
    process.RAWSIMoutput_step
)

# Filter all path with the production filter sequence
for path in process.paths:
    getattr(process,path)._seq = process.generator * getattr(process,path)._seq

from CRABClient.UserUtilities import config
config = config()

config.General.requestName = 'CRAB3_Advanced_Tutorial_May2015_Exercise4A'

config.JobType.pluginName = 'PrivateMC'
config.JobType.psetName = 'pset_tutorial_MC_generation.py'
config.JobType.scriptExe = 'myscript.sh'

config.Data.outputPrimaryDataset = 'MinBias'
config.Data.splitting = 'EventBased'
config.Data.unitsPerJob = 10
config.Data.totalUnits = 30
config.Data.publication = True
config.Data.outputDatasetTag = config.General.requestName

config.Site.storageSite = <site where the user has permission to write>

echo "================= CMSRUN starting ===================="
cmsRun -j FrameworkJobReport.xml -p PSet.py
echo "================= CMSRUN finished ===================="

from CRABClient.UserUtilities import config
config = config()

config.General.requestName = 'CRAB3_Advanced_Tutorial_May2015_Exercise4B'

config.JobType.pluginName = 'PrivateMC'
config.JobType.psetName = 'pset_tutorial_MC_generation.py'
config.JobType.scriptExe = 'myscript.sh'
config.JobType.outputFiles = ['output.txt']

config.Data.outputPrimaryDataset = 'MinBias'
config.Data.splitting = 'EventBased'
config.Data.unitsPerJob = 10
config.Data.totalUnits = 30
config.Data.publication = True
config.Data.outputDatasetTag = config.General.requestName

config.Site.storageSite = <site where the user has permission to write>

echo "================= CMSRUN starting ====================" >> output.txt
cmsRun -j FrameworkJobReport.xml -p PSet.py
echo "================= CMSRUN finished ====================" >> output.txt

from CRABClient.UserUtilities import config
config = config()

config.General.requestName = 'CRAB3_Advanced_Tutorial_May2015_Exercise4B'

config.JobType.pluginName = 'PrivateMC'
config.JobType.psetName = 'pset_tutorial_MC_generation.py'
config.JobType.scriptExe = 'myscript.sh'
config.JobType.inputFiles = ['input.txt']

config.Data.outputPrimaryDataset = 'MinBias'
config.Data.splitting = 'EventBased'
config.Data.unitsPerJob = 10
config.Data.totalUnits = 30
config.Data.publication = True
config.Data.outputDatasetTag = config.General.requestName

config.Site.storageSite = <site where the user has permission to write>

while read line
do
    if [ "$line" == "Before" ]; then
        continue
    elif [ "$line" == "After" ]; then
        cmsRun -j FrameworkJobReport.xml -p PSet.py
    else
        echo $line
    fi
done < input.txt

Before
================= CMSRUN starting ====================
After
================= CMSRUN finished ====================

from CRABClient.UserUtilities import config
config = config()

config.General.requestName = 'CRAB3_Advanced_Tutorial_May2015_Exercise4B'

config.JobType.pluginName = 'PrivateMC'
config.JobType.psetName = 'pset_tutorial_MC_generation.py'
config.JobType.scriptExe = 'myscript.sh'
# Arguments have to be in the form param=value, without white spaces, quotation marks nor additional equal signs (=).
config.JobType.scriptArgs = ['Before=CMSRUN-starting', 'After=CMSRUN-finished']

config.Data.outputPrimaryDataset = 'MinBias'
config.Data.splitting = 'EventBased'
config.Data.unitsPerJob = 10
config.Data.totalUnits = 30
config.Data.publication = True
config.Data.outputDatasetTag = config.General.requestName

config.Site.storageSite = <site where the user has permission to write>

beforemsg=""
aftermsg=""
for i in "$@"
do
case $i in
    Before=*)
    beforemsg="${i#*=}"
    ;;
    After=*)
    aftermsg="${i#*=}"
    ;;
esac
done
echo "================= $beforemsg ===================="
cmsRun -j FrameworkJobReport.xml -p PSet.py
echo "================= $aftermsg ===================="

echo "================= CMSRUN starting ===================="
cmsRun -j FrameworkJobReport.xml -p PSet.py
echo "================= CMSRUN finished ===================="

exitCode=80500
exitMessage="This is a test to see if I can pass exit code 80500 to CRAB."
errorType=""

if [ -e FrameworkJobReport.xml ]
then
    cat << EOF > FrameworkJobReport.xml.tmp
<FrameworkJobReport>
<FrameworkError ExitStatus="$exitCode" Type="$errorType" >
$exitMessage
</FrameworkError>
EOF
    tail -n+2 FrameworkJobReport.xml >> FrameworkJobReport.xml.tmp
    mv FrameworkJobReport.xml.tmp FrameworkJobReport.xml
else
    cat << EOF > FrameworkJobReport.xml
<FrameworkJobReport>
<FrameworkError ExitStatus="$exitCode" Type="$errorType" >
$exitMessage
</FrameworkError>
</FrameworkJobReport>
EOF
fi

##### These are the lines you can add to your pset get the estimates in the FrameworkJobReport file
process.Timing = cms.Service("Timing",
    summaryOnly = cms.untracked.bool(True)
)

##### For your information CRAB3 also add the following two lines in addition to the previous three:
#process.CPU = cms.Service("CPU")

#process.SimpleMemoryCheck = cms.Service("SimpleMemoryCheck")

cmsRun -j FrameworkJobReport.xml -p pset_MC_generation_LHE.py

You can open the FrameworkJobReport.xml file now and look for the AvgEventTime, which is what CRAB3 currently looks for the estimation of the walltimes.

from CRABClient.UserUtilities import config
config = config()

config.General.requestName = 'CRAB3_Advanced_Tutorial_May2015_Exercise5A'

config.JobType.pluginName = 'PrivateMC'
config.JobType.generator = 'lhe'
config.JobType.psetName = 'pset_MC_generation_LHE.py'
config.JobType.inputFiles = ['dynlo.lhe']

config.Data.outputPrimaryDataset = 'TutorialMay2015Exercise5A'
config.Data.splitting = 'EventBased'
JOB_WALLTIME = 8*3600
TIME_PER_EVENT = 0.25
config.Data.unitsPerJob = int(JOB_WALLTIME / TIME_PER_EVENT)
NJOBS = 10
config.Data.totalUnits = config.Data.unitsPerJob * NJOBS
config.Data.publication = True
config.Data.outputDatasetTag = 'MC_generation_LHE'

config.Site.storageSite = <storage-site>

You are probably getting the following error, because passing the LHE file directly in the input sandbox causes the sandbox to go over the 
allowed limit of 100Mb:

Will use CRAB configuration file /afs/cern.ch/user/m/mmascher/tutorial/Exercise_5/crabConfig.py

Error contacting the server.
Server answered with: Invalid input parameter
Reason is: File is bigger then allowed limit of 10485760B

/afs/cern.ch/user/b/belforte/public/mybin/lfn2pfn.sh <site> [<LFN>]

#config.JobType.inputFiles = ['dynlo.lhe']
config.Site.whitelist = ['T2_CH_CERN']

process.source = cms.Source("LHESource",
    fileNames = cms.untracked.vstring('gsi
ftp://eoscmsftp.cern.ch//eos/cms/store/user/<username>/dynlo.lhe ')
)

#config.JobType.inputFiles = ['dynlo.lhe']
config.Site.whitelist = ['T2_CH*']

Create a python script named "multicrab" (with permissions 744) which you should be able to run in the following way:
     ./multicrab --crabCmd CMD [--workArea WAD --crabCmdOpts OPTS]
where CMD is the crab command, WAD is a work area directory with many CRAB project directories inside and OPTS are options for the crab 
command. 

#!/usr/bin/env python
"""
This is a small script that does the equivalent of multicrab.
"""
import os
from optparse import OptionParser


def getOptions():
    """
    Parse and return the arguments provided by the user.
    """
    usage = ("Usage: %prog --crabCmd CMD [--workArea WAD --crabCmdOpts OPTS]"
             "\nThe multicrab command executes 'crab CMD OPTS' for each project directory contained in WAD"
             "\nUse multicrab -h for help")

    parser = OptionParser(usage=usage)

    parser.add_option('-c', '--crabCmd',
                      dest = 'crabCmd',
                      default = '',
                      help = "crab command",
                      metavar = 'CMD')

    parser.add_option('-w', '--workArea',
                      dest = 'workArea',
                      default = '',
                      help = "work area directory (only if CMD != 'submit')",
                      metavar = 'WAD')

    parser.add_option('-o', '--crabCmdOpts',
                      dest = 'crabCmdOpts',
                      default = '',
                      help = "options for crab command CMD",
                      metavar = 'OPTS')

    (options, arguments) = parser.parse_args()

    if arguments:
        parser.error("Found positional argument(s): %s." % (arguments))
    if not options.crabCmd:
        parser.error("(-c CMD, --crabCmd=CMD) option not provided.")
    if options.crabCmd != 'submit':
        if not options.workArea:
            parser.error("(-w WAR, --workArea=WAR) option not provided.")
        if not os.path.isdir(options.workArea):
            parser.error("'%s' is not a valid directory." % (options.workArea))

    return options


def main():

    options = getOptions()

    # Do something. 

 
if __name__ == '__main__':
    main()

#!/usr/bin/env python
"""
This is a small script that does the equivalent of multicrab.
"""
import os
from optparse import OptionParser

import CRABClient
from CRABAPI.RawCommand import crabCommand
from CRABClient.ClientExceptions import ClientException
from httplib import HTTPException


def getOptions():
    """
    Parse and return the arguments provided by the user.
    """
    usage = ("Usage: %prog --crabCmd CMD [--workArea WAD --crabCmdOpts OPTS]"
             "\nThe multicrab command executes 'crab CMD OPTS' for each project directory contained in WAD"
             "\nUse multicrab -h for help")

    parser = OptionParser(usage=usage)

    parser.add_option('-c', '--crabCmd',
                      dest = 'crabCmd',
                      default = '',
                      help = "crab command",
                      metavar = 'CMD')

    parser.add_option('-w', '--workArea',
                      dest = 'workArea',
                      default = '',
                      help = "work area directory (only if CMD != 'submit')",
                      metavar = 'WAD')

    parser.add_option('-o', '--crabCmdOpts',
                      dest = 'crabCmdOpts',
                      default = '',
                      help = "options for crab command CMD",
                      metavar = 'OPTS')

    (options, arguments) = parser.parse_args()

    if arguments:
        parser.error("Found positional argument(s): %s." % (arguments))
    if not options.crabCmd:
        parser.error("(-c CMD, --crabCmd=CMD) option not provided.")
    if options.crabCmd != 'submit':
        if not options.workArea:
            parser.error("(-w WAR, --workArea=WAR) option not provided.")
        if not os.path.isdir(options.workArea):
            parser.error("'%s' is not a valid directory." % (options.workArea))

    return options


def main():

    options = getOptions()

    # The submit command needs special treatment.
    if options.crabCmd == 'submit':

        #--------------------------------------------------------
        # This is the base config:
        #--------------------------------------------------------
        from CRABClient.UserUtilities import config
        config = config()

        config.General.requestName = None
        config.General.workArea = 'CRAB3_Advanced_Tutorial_May2015_Exercise6'

        config.JobType.pluginName = 'Analysis'
        config.JobType.psetName = 'pset_tutorial_analysis.py'

        config.Data.inputDataset = None
        config.Data.splitting = 'LumiBased'
        config.Data.unitsPerJob = 10
        config.Data.totalUnits = 30
        config.Data.outputDatasetTag = None

        config.Site.storageSite = None # Choose your site. 
        #--------------------------------------------------------

        # Will submit one task for each of these input datasets.
        inputDatasets = [
                         '/DoubleMuParked/Run2012A-22Jan2013-v1/AOD',
                         '/DoubleMuParked/Run2012B-22Jan2013-v1/AOD',
                         '/DoubleMuParked/Run2012C-22Jan2013-v1/AOD',
                         '/DoubleMuParked/Run2012D-22Jan2013-v1/AOD',
                        ]

        for inDS in inputDatasets:
            # inDS is of the form /A/B/C. Since B is unique for each inDS, use this in the CRAB request name.
            config.General.requestName = inDS.split('/')[2]
            config.Data.inputDataset = inDS
            config.Data.outputDatasetTag = '%s_%s' % (config.General.workArea, config.General.requestName)
            # Submit.
            try:
                print "Submitting for input dataset %s" % (inDS)
                crabCommand(options.crabCmd, config = config, *options.crabCmdOpts.split())
            except HTTPException as hte:
                print "Submission for input dataset %s failed: %s" % (inDS, hte.headers)
            except ClientException as cle:
                print "Submission for input dataset %s failed: %s" % (inDS, cle)

    # All other commands can be simply executed.
    elif options.workArea:

        for dir in os.listdir(options.workArea):
            projDir = os.path.join(options.workArea, dir)
            if not os.path.isdir(projDir):
                continue
            # Execute the crab command.
            msg = "Executing (the equivalent of): crab %s --dir %s %s" % (options.crabCmd, projDir, options.crabCmdOpts)
            print "-"*len(msg)
            print msg
            print "-"*len(msg)
            try:
                crabCommand(options.crabCmd, dir = projDir, *options.crabCmdOpts.split())
            except HTTPException as hte:
                print "Failed executing command %s for task %s: %s" % (options.crabCmd, projDir, hte.headers)
            except ClientException as cle:
                print "Failed executing command %s for task %s: %s" % (options.crabCmd, projDir, cle)


if __name__ == '__main__':
    main()