-- ChengguangZhu - 01 Dec 2007 TESTED! for more tutorial look at http://freeos.com/guides/lsst/

use xterm

/usr/bin/xterm -fn 12*24 -ls -display $DISPLAY -sl 2000 -geometry 115x25

general linux Command

  • which : to find the directory where command is
  • trf_ls: list all the transform script, after athena env is setup
  • find_python_module.py : search where the python module is, syntax for module is e.g. AthenaCommon. GlobalFlags
  • for cvs at CERN:
    • CVSROOT=:kserver:atlas-sw.cern.ch:/atlascvs
    • cvs co -d xxxx -r AnalysisExamples-00-18-09 offline/PhysicsAnalysis/AnalysisCommon/AnalysisExamples check out AnalysisExamples of tag AnalysisExamples-00-18-09 and put its content into dir xxxx
    • cvs co -d JetRec -r JetRec-00-04-32 offline/Reconstruction/JetRec

command related to AFS system

Use awk AWK

use python

  • hex(integer) to convert interger to hex format

An Example with bash to do lot of work

#!/bin/bash

if [ $# -eq 0 ]; then
  exit
fi

rm nohup.out -f
#define verion
VERSION=12.0.6
MY_PACKAGE_DIR=`pwd`
#############################
##### working directory######
#############################

BIG_DISK=/raid0/zhucg/analysisResults/

  DATE=`date -Idate`
  HOUR=`date -Isecond | cut -d":" -f1 |cut -d"T" -f2 `
MINUTE=`date -Isecond | cut -d":" -f2  `
SECOND=`date -Isecond | cut -d":" -f3 |cut -d"+" -f1 `

WORKDIR=$BIG_DISK/analysis_zhucgcode_$VERSION-$DATE-$HOUR-$MINUTE-$SECOND
mkdir $WORKDIR
cd $WORKDIR

echo this is to run version $VERSION                    >> $WORKDIR/runlog
echo Start woking time is $DATE $HOUR:$MINUTE:$SECOND   >> $WORKDIR/runlog
echo Working dir is $WORKDIR                            >> $WORKDIR/runlog


#############################
##### run parameters 2#######
#############################


MY_PACKAGE=AnalysisExamples
DATA_SET=$1
INPUTFILE_DIR=`grep $DATA_SET /raid0/zhucg/myAnalysisCode/program/data_link |cut -d":" -f2`

while [ "$INPUTFILE_DIR" = "" ]
do
  echo look at what data we have ...
  cat /raid0/zhucg/myAnalysisCode/program/data_link
  read DATA_SET
  INPUTFILE_DIR=`grep $DATA_SET /raid0/zhucg/myAnalysisCode/program/data_link |cut -d":" -f2`
done

echo You are using pacake  $MY_PACKAGE_DIR/$MY_PACKAGE    >> $WORKDIR/runlog
echo You are using data in $INPUTFILE_DIR                 >> $WORKDIR/runlog


#########################################
########list files to be analyzed########
#########################################
INPUTFILE_LISTED_IN_FILE=$WORKDIR/$DATA_SET.py
echo Data files are  listed in $INPUTFILE_LISTED_IN_FILE  >> $WORKDIR/runlog
cat  $WORKDIR/runlog

rm -f $INPUTFILE_LISTED_IN_FILE
case  "$DATA_SET" in
  wjets)
    for dataset in 8240 8241 8242 8243 8244 8245 8246 8247 8248 8249 8250 8251
    do
      INPUTFILE_DIR=`grep $dataset /raid0/zhucg/myAnalysisCode/program/data_link |cut -d":" -f2`
      for file in `ls $INPUTFILE_DIR|grep root`
      do
        echo \"$INPUTFILE_DIR/$file\", >>$INPUTFILE_LISTED_IN_FILE
      done
    done
    ;;
  singlet)
    for dataset in 5500 5501 5502
    do
      INPUTFILE_DIR=`grep $dataset /raid0/zhucg/myAnalysisCode/program/data_link |cut -d":" -f2`
      for file in `ls $INPUTFILE_DIR|grep root`
      do
        echo \"$INPUTFILE_DIR/$file\", >>$INPUTFILE_LISTED_IN_FILE
      done
    done
    ;;
  *)
    for file in `ls $INPUTFILE_DIR|grep root`
    do
      echo \"$INPUTFILE_DIR/$file\", >>$INPUTFILE_LISTED_IN_FILE
    done
    ;;
esac


#########################################
###########Setup Athena env##############
#########################################
# setup of ATHENA
. $MY_PACKAGE_DIR/cmthome126/setup.sh -tag=${VERSION}
. $AtlasArea/AtlasOfflineRunTime/cmt/setup.sh ""
export CMTPATH=$WORKDIR:${CMTPATH}
. /raid0/SGE/default/common/settings.sh


cat >  $WORKDIR/setup.sh <<EOF
#!/bin/bash

# setup of ATHENA
. $MY_PACKAGE_DIR/cmthome126/setup.sh -tag=${VERSION}
. $AtlasArea/AtlasOfflineRunTime/cmt/setup.sh ""
export CMTPATH=$CMTPATH
. /raid0/SGE/default/common/settings.sh

EOF


#########################################
######  config analysis package##########
#########################################

cd $WORKDIR
cp -r $MY_PACKAGE_DIR/$MY_PACKAGE .

if [ ! -e $MY_PACKAGE ]
then
  echo my analysis package is not gotten
  exit
fi

# build joboptions from the packages just coped in
if [ "$DATA_SET" = "5200_bugfix" ]  # -o  "$DATA_SET" = "5200_pileup" ]
then
  echo "" >ttbarExample_jobOptions1.py
else
cat > ./ttbarExample_jobOptions1.py <<EOF
#do 1mm correction
doAODFix1mm=True
include( "ElectronPhotonAlgs/ElectronPhotonCorrector_jobOptions.py")
include( "JetMissingEtAlgs/ParticleJet1mmCorrection_jobOptions.py")
include( "JetMissingEtAlgs/MissingET1mmCorrection_jobOptions.py")
EOF
fi

cp $MY_PACKAGE/share/ttbarExample_jobOptions.py ./ttbarExample_jobOptions2.py

cat > ./ttbarExample_jobOptions3.py <<EOF
DoBJet.AODContainerName   = "Cone4TowerParticleJets"
DoBJet.OutputLevel        = DEBUG
ttbar.OutputLevel         = DEBUG
DoElectron.OutputLevel    = DEBUG
DoMuon.OutputLevel        = DEBUG
DoBJet.PrecaliSource      = "quark"
ttbar.DoInsituCalibration     = False
DoBJet.DoPrecalibration       = True
#DoBJet.BJetEtCut              = 30*GeV
MessageSvc.defaultLimit       = 3000
ttbar.EventRecoAlg            = 1
ttbar.MissingEtCut            = 20*GeV
DoElectron.ElectronEtCut      = 25.0*GeV
#DoElectron.EleInfoFromCali    = 1
#DoElectron.MaskEMCut          = 15
#DoBJet.BJetMiscalibration     = 1.1
EOF

cat ./ttbarExample_jobOptions1.py ./ttbarExample_jobOptions2.py ./ttbarExample_jobOptions3.py >./ttbarExample_jobOptions.py
rm -f ./ttbarExample_jobOptions1.py ./ttbarExample_jobOptions2.py ./ttbarExample_jobOptions3.py
if [ "$DATA_SET" = "5205" ]
then
  echo ttbar.MCGenerator = \"ACERMC\"  >> ./ttbarExample_jobOptions.py
fi



cd $MY_PACKAGE/cmt
cmt br rm -f setup.sh  clean.sh
cmt br cmt config
. setup.sh ""
cmt br gmake

cd $WORKDIR


#####################################
######produce datafile list##########
#####################################

rome=`cat $INPUTFILE_LISTED_IN_FILE |  wc -l`
if [ "$3" != "" ]; then rome=$3; fi
step=30
count=0
countend=0
for file in `cat $INPUTFILE_LISTED_IN_FILE `
do
  if [ $rome -gt 0 ]
  then
    echo $file >> $count
    countend=$(echo "$countend+1" | bc )
    close=no
    FILE=`echo $file|cut -d"\"" -f2`
    SIZE=`ls -s $FILE | cut -d" " -f1`
    if [ $SIZE -ge 30000 ]; then  step=30;  else step=30; fi
    if [ $SIZE -ge 100000 ]; then  step=30;  fi
    if [ $countend -ge $step ]; then close="yes"; fi
    if [ $countend -ge $rome ]; then close="yes"; fi
    if [ $close = "yes" ]
    then

cat > data_$count.py <<EOF
# Root Ntuple output
theApp.Dlls += [ "RootHistCnv" ]
theApp.HistogramPersistency = "ROOT"
NTupleSvc = Service( "NTupleSvc" )
HistogramPersistencySvc = Service( "HistogramPersistencySvc" )
theApp.EvtMax = -1

NTupleSvc.Output = [ "FILE1 DATAFILE='$WORKDIR/ana_result_$count.ntuple.root' OPT='NEW' "]
HistogramPersistencySvc.OutputFile = "$WORKDIR/ana_result_$count.hist.root";
EventSelector.InputCollections = [
EOF
    echo empty file > $WORKDIR/ana_result_$count.ntuple.root
    cat $count >> data_$count.py
    echo ] >>data_$count.py
    rm -f $count

    count=$(echo "$count+$countend" | bc )
    rome=$(echo "$rome-$countend" | bc )
    countend=0

    fi
  fi
done

###########################################
###########################################


if [ ! -e PDGTABLE.MeV ]
then
  get_files PDGTABLE.MeV
fi


#define 2 bash shell program for later usage
cat > Run_repair.sh <<EOF
#!/bin/bash
. $WORKDIR/setup.sh
export CMTPATH=$WORKDIR:$CMTPATH
. $WORKDIR/$MY_PACKAGE/cmt/setup.sh
export PATH=`echo $PATH |cut -d":" -f20-`

ls | grep ana >error
if [ -s error ]
then
  hadd -f ANA.ntuple.root ana*.ntuple.root 2>error
fi

while [ -s error ]
do
  # take out the name of root files, which have error.
  rm -f errorroot
  nstring=1;
  while [ "\$nstring" != "15" ]
  do
    cat error |cut -d" " -f\$nstring |grep ana_result >>errorroot
    nstring=\$(echo "\$nstring+1" | bc )
  done

  # take out the number in the file name
  RERUN=""
  TORERUN=\`cat  errorroot |cut -d"_" -f3|cut -d"." -f1\`
  #TORERUN="\`cat error| grep segmentation|cut -d"_" -f4|cut -d"." -f1\` \$TORERUN"

  # resend these jobs
  echo \$TORERUN should be redo
  for NUM in \$TORERUN
  do
    CHECK=""
    for i in \$RERUN
    do
      if [ "\$i" = "\$NUM" ]
      then
        CHECK="DONE"
      fi
    done
    if [ ! "\$CHECK" = "DONE" ]
    then
      RERUN="\$RERUN \$NUM"
      echo \$NUM is redoing
      rm -f $WORKDIR/logfile/data_\$NUM.py.log
      qsub -j y -o $WORKDIR/logfile/data_\$NUM.py.log $WORKDIR/jobfile/data_\$NUM.py.sh
    fi
  done

  echo
  echo resend these jobs to batch is...
  qstat -u zhucg  | grep zhucg > jobstatus
  cat jobstatus

  while [ -s jobstatus ]
  do
    sleep 200
    rm -f \$HOME/core.*
    qstat -u zhucg | grep zhucg > jobstatus
    echo now the status after a while of time
    cat jobstatus
    for i in \`cat jobstatus | grep Eqw |cut -d" " -f4\`
    do
      qdel \$i
    done
  done

  ls |grep ana >error
  if [ "\$TORERUN" != "" ]
  then
    hadd -f ANA.ntuple.root ana*.ntuple.root 2>error
  fi
done

hadd -f ANA.hist.root   *.hist.root

EOF



cat > Clean.sh <<EOF
mv logfile/data_0.py.log .
mv jobfile/data_0.py.sh  .
rm logfile/ DATAfile/ jobfile/ -rf
rm ana* -f
rm error errorroot InstallArea/ jobstatus PDGTABLE.MeV Run_repair.sh -fr
EOF


###########################################
#
#   send batch jobs
#
###########################################
mkdir -p $WORKDIR/logfile
mkdir -p $WORKDIR/jobfile
mkdir -p $WORKDIR/DATAfile

echo =======================env
#env
echo =======================env

echo because SGE6.0 has an artificial limit on PATH length
export PATH=`echo $PATH |cut -d":" -f20-`

ATHENADIR=/raid4/atlas/AtlasRelease/$VERSION/AtlasCore/*/InstallArea/share/bin/
for datafile in `ls data* `
do
  mv $datafile   $WORKDIR/DATAfile
  rm $datafile.sh -f
cat > $datafile.sh <<EOF
#!/bin/bash
echo "Starting at" `date` "on" `hostname`
#$ -S /bin/bash
echo . $WORKDIR/setup.sh
export CMTPATH=$WORKDIR:$CMTPATH
. $WORKDIR/setup.sh
echo $WORKDIR/$MY_PACKAGE/cmt/setup.sh
. $WORKDIR/$MY_PACKAGE/cmt/setup.sh
$ATHENADIR/athena.py $WORKDIR/ttbarExample_jobOptions.py  $WORKDIR/DATAfile/$datafile
EOF

  mv $datafile.sh $WORKDIR/jobfile
  echo qsub -o $WORKDIR/logfile/$datafile.log jobfile/$datafile.sh
  qsub -j y -o $WORKDIR/logfile/$datafile.log jobfile/$datafile.sh
done

echo
echo all jobs sent to batch is...
qstat -u zhucg  | grep zhucg > jobstatus
cat jobstatus

while [ -s jobstatus ]
do
  sleep 200
  rm -f $HOME/core.*
  qstat -u zhucg | grep zhucg > jobstatus
  echo now the status after a while of time
  cat jobstatus
  for i in `cat jobstatus | grep Eqw |cut -d" " -f4`
  do
    qdel $i
  done
done



. Run_repair.sh


cd 
echo " "
echo "##"
echo "##################################################################"
echo "End of job."
Edit | Attach | Watch | Print version | History: r8 < r7 < r6 < r5 < r4 | Backlinks | Raw View | WYSIWYG | More topic actions
Topic revision: r8 - 2010-10-15 - ChengguangZhu
 
    • Cern Search Icon Cern Search
    • TWiki Search Icon TWiki Search
    • Google Search Icon Google Search

    Main All webs login

This site is powered by the TWiki collaboration platform Powered by PerlCopyright & 2008-2019 by the contributing authors. All material on this collaboration platform is the property of the contributing authors.
Ideas, requests, problems regarding TWiki? Send feedback