Blobber stress script

Bash script to stress blobber.

Create blobber stress script

  1. Create blobberstress.sh script with below content inside .zcn dir.

    cd ~/.zcn

  2. Create blobberstress.sh script.

    touch blobberstress.sh

  3. Paste the below content to blobberstress.sh file.

    NETWORK=$1
PREFIX=$2
files=$3
kb=$4
base=$5

ZBOXBIN=zbox
ZWALBIN=zwallet

ramnode=http://ramnode.zus.network/ram/
cpunode=http://cpunode.zus.network/cpu/
chartupload=https://blob-chart.zus.network/upload.php

SIAMINPC=80
SIATIMEOUT=30
#SIAPW=""
#STJBKT=fred
CWD=$(pwd)

export CFG=$NETWORK".yaml"
echo "CFG: " $CFG
export WAL=$NETWORK".json"
echo "WAL: " $WAL
../$ZWALBIN getbalance --config $CFG --wallet $WAL

faucets=9
allocs=1
writes=1
reads=2
GB=9

databits=1
paritybits=1

FAUCET=0
ALLOC=0
READ=0
STORJ=0
SIA=0
AWS=1
ZCN=1
ZS3=0

UPLOAD=1
DOWNLOAD=1

ULFREQ=1
DLFREQ=1
LISTALLFREQ=0
RAMFREQ=0
CPUFREQ=0

if [ "$PREFIX" == "x" ]
then
        PREFIX="$NETWORK""_""$kb""x$files-EC$databits$paritybits"
fi

echo "PREFIX: $PREFIX"

if [ $files -lt 1000 ]
then
        ULFREQ=1
fi
if [ $files -ge 1000 ]
then
        ULFREQ=5
fi
if [ $files -ge 10000 ]
then
        ULFREQ=20
fi
if [ $files -ge 100000 ]
then
        ULFREQ=100
fi


while [ -n "$1" ]; do
        case "$1" in
        -f) FAUCET=1 ;;
        -a) ALLOC=1 ;;
        -u) UPLOAD=1 ;;
        -d) DOWNLOAD=1 ;;
        -r) READ=1 ;;
        -zcn) ZCN=1 ;;
        -nozcn) ZCN=0 ;;
        -stj) STORJ=1 ;;
        -sia) SIA=1 ;;
        -aws) AWS=1 ;;
        -zs3) ZS3=1 ;;
        --)
                shift
                break
                ;;
        *) if [[ $1 == "-"* ]] ; then echo "Option $1 not recognized" ; fi ;;
        esac
        shift
done

w=$(basename $PWD)

echo "Removing files from previous runs.."
rm $PREFIX-file*.dat
rm $PREFIX-file*.dat.zcn.dl
rm $PREFIX-file*.dat.stj.dl
rm $PREFIX-file*.dat.sia.dl
rm $PREFIX-file*.dat.aws.dl
rm $PREFIX-file*.dat.zs3.dl

if [ $ZCN -eq 1 ]
then

        echo "Initializing ZCN Wallet & Allocation.."

        echo "Using Wallet $w.json"

        echo "Creating config $w.yaml"

        rm ../$w.yaml
        echo "---" > ../$w.yaml
        if [ "$NETWORK" == "potato" ]
        then
                echo "block_worker: https://$NETWORK.devnet-0chain.net/dns" >> ../$w.yaml
        else
                echo "block_worker: https://$NETWORK.zus.network/dns" >> ../$w.yaml
        fi
        echo "signature_scheme: bls0chain" >> ../$w.yaml
        echo "min_submit: 50" >> ../$w.yaml
        echo "min_confirmation: 50" >> ../$w.yaml
        echo "confirmation_chain_length: 3" >> ../$w.yaml
        echo "max_txn_query: 5" >> ../$w.yaml
        echo "query_sleep_time: 5" >> ../$w.yaml

        if [ $FAUCET -eq 1 ]
        then
                echo "Fauceting tokens.."
                i=1 ; while [ $i -le $faucets ] ; do  ../$ZWALBIN faucet --methodName pour --input test --silent --wallet $w.json --config $w.yaml --tokens 9 ; ((i++)) ; done
        fi


        echo "Getting Balance.."
        ../$ZWALBIN getbalance --silent --wallet $w.json --config $w.yaml


        if [ $ALLOC -eq 1 ]
        then
                echo "Creating Allocations.."
                a=1
                echo "all-$w-$a.txt"
                while [ $a -le $allocs ]
                do
                ../$ZBOXBIN newallocation --size $GB"000000000" --lock 50 --data $databits --parity $paritybits --silent --wallet $w.json --config $w.yaml --allocationFileName all-$w-$a.txt
                ((a++))
                done
        fi


        echo "Getting Balance.."
        ../$ZWALBIN getbalance --silent --wallet $w.json --config $w.yaml


        if [ $READ -eq 1 ]
        then
                echo "Creating Read Pools.."
                a=1 ; while [ $a -le $allocs ] ; do ../$ZBOXBIN rp-lock --tokens 10 --silent --wallet $w.json --config $w.yaml ; ((a++)) ; done
        fi

        echo "Getting Balance.."
        ../zwallet getbalance --silent --wallet $w.json --config $w.yaml
fi


echo "Creating random files.."
h="01234567890abcdef"
x=$h$h$h$h$h$h$h$h$h$h$h$h$h$h$h${h:0:15}
k=$x$x$x$x

echo "H : "$h
echo "X : "$x
echo "K : "$k

f=0 ; while [ $f -le $files ] ; do d=$kb ; echo "TEST $PREFIX $f" > $PREFIX-file$f.dat ; while [ $d -gt 0 ] ; do echo "$k" >> $PREFIX-file$f.dat ; ((d--)) ; done ; ((f++)) ; done


echo "Writes.."

echo $(date) > $w-$PREFIX.start.txt
curl --http1.1 -F file=@$w-$PREFIX.start.txt $chartupload

echo "NETWORK: $NETWORK | SIZE: ""$kb""KB | FILES: $files | EC: $databits$paritybits" > $w-$PREFIX.desc.txt
curl --http1.1 -F file=@$w-$PREFIX.desc.txt $chartupload

echo -n "" > $w-$PREFIX.uploads.txt
echo -n "" > $w-$PREFIX.downloads.txt
echo -n "" > $w-$PREFIX.stjuploads.txt
echo -n "" > $w-$PREFIX.stjdownloads.txt
echo -n "" > $w-$PREFIX.siauploads.txt
echo -n "" > $w-$PREFIX.siadownloads.txt
echo -n "" > $w-$PREFIX.awsuploads.txt
echo -n "" > $w-$PREFIX.awsdownloads.txt
echo -n "" > $w-$PREFIX.zs3uploads.txt
echo -n "" > $w-$PREFIX.zs3downloads.txt
echo -n "" > $w-$PREFIX.listalls.txt
echo -n "" > $w-$PREFIX.cpus.txt
echo -n "" > $w-$PREFIX.rams.txt

a=1
#base=5
while [ $a -le $allocs ]
do
        t=1
        #while [ $t -le $writes ]
        while [ $t -le $files ]
        do
      path=""
      n=$t
      n=$((n / $base))  
      while [ $n -gt 0 ]
      do
        v=$(($n % $base))
        n=$((n / $base))  
        path=$v$path
      done
          remote=""
          i=0
          while [ $i -lt ${#path} ]
          do
            c=${path:$i:1}
            remote=$remote/$c$c$c$c$c$c$c$c$c$c
                ((i++))
          done
          remote=$remote/file$t.dat

if test -f "paws.txt" ; then echo "PAUSED" ; while ( test -f "paws.txt" ) do sleep 1 ; done ; fi

#UPLOADS          

        if [ $UPLOAD -eq 1 ]
    then
          
        #ZCNUPLOAD        
                if [ $ZCN -eq 1 ]
                then
                  a=$((1 + RANDOM % $allocs))
                  #f=$((1 + RANDOM % $files))
                  f=$t
                  sta=$(date +%s.%N)
                  ../$ZBOXBIN upload --wallet $w.json --config $w.yaml --allocation $(cat ../all-$w-$a.txt) --localpath $PREFIX-file$f.dat --remotepath /$PREFIX$remote --silent --chunknumber 500
                  fin=$(date +%s.%N)
                  
                  x=$((t%ULFREQ))
                  if [ $x -eq 0 ]
                  then    
                        echo $t,$sta,$fin >> $w-$PREFIX.uploads.txt
                        curl --http1.1 -F file=@$w-$PREFIX.uploads.txt $chartupload
                  fi
                fi

        #STJUPLOAD        
                if [ $STORJ -eq 1 ]
                then
                  a=$((1 + RANDOM % $allocs))
                  #f=$((1 + RANDOM % $files))
                  f=$t
                  sta=$(date +%s.%N)
                  uplink  cp $PREFIX-file$f.dat sj://fred/$PREFIX$remote
                  fin=$(date +%s.%N)
                  
                  x=$((t%ULFREQ))
                  if [ $x -eq 0 ]
                  then    
                        echo $t,$sta,$fin >> $w-$PREFIX.stjuploads.txt
                        curl --http1.1 -F file=@$w-$PREFIX.stjuploads.txt $chartupload
                  fi
                fi

        #SIAUPLOAD        
                if [ $SIA -eq 1 ]
                then
                  a=$((1 + RANDOM % $allocs))
                  #f=$((1 + RANDOM % $files))
                  f=$t
                  sta=$(date +%s.%N)
                  echo curl -A "Sia-Agent" -u "":$SIAPW --data "source=$CWD/$PREFIX-file$f.dat" "localhost:9980/renter/upload/$PREFIX$remote"
                  curl -A "Sia-Agent" -u "":$SIAPW --data "source=$CWD/$PREFIX-file$f.dat" "localhost:9980/renter/upload/$PREFIX$remote"
                  #uplink  cp $PREFIX-file$f.dat sj://$STJBKT/$PREFIX$remote

                  siaprog=0
                  siafail=0
                  #while [[ $siaprog -lt 100 ]]; do
                  while (( $(echo "$siaprog < $SIAMINPC" |bc -l) ))
                  do
                curl -s -A Sia-Agent -u :$SIAPW localhost:9980/renter/file/$PREFIX$remote | jq .file.uploadprogress > siaprog.txt
                sleep 0.1
                siaprog=$(cat siaprog.txt)
                echo -n $siaprog" "
                now=$(date +%s.%N)
                diff=$(echo "$now - $sta" | bc)
                if (( $(echo "$diff > $SIATIMEOUT" |bc -l) ))
                then
                  siafail=1
                  siaprog=$SIAMINPC
                fi
              done

                  fin=$(date +%s.%N)
                  
                  if [ $siafail -eq 1 ]
                  then
                    echo "FAIL - TIMEOUT"
                  else
                    echo "SUCCESS"
                  
                        x=$((t%ULFREQ))
                        if [ $x -eq 0 ]
                        then      
                          echo $t,$sta,$fin >> $w-$PREFIX.siauploads.txt
                          curl --http1.1 -F file=@$w-$PREFIX.siauploads.txt $chartupload
                        fi
                  fi
                fi


        #AWSUPLOAD        
                if [ $AWS -eq 1 ]
                then
                  a=$((1 + RANDOM % $allocs))
                  #f=$((1 + RANDOM % $files))
                  f=$t
                  sta=$(date +%s.%N)
                  aws s3 cp $PREFIX-file$f.dat s3://zus-loadtest-report/$PREFIX$remote
                  fin=$(date +%s.%N)
                  
                  x=$((t%ULFREQ))
                  if [ $x -eq 0 ]
                  then    
                        echo $t,$sta,$fin >> $w-$PREFIX.awsuploads.txt
                        curl --http1.1 -F file=@$w-$PREFIX.awsuploads.txt $chartupload
                  fi
                fi


        #ZS3UPLOAD        
                if [ $ZS3 -eq 1 ]
                then
                  a=$((1 + RANDOM % $allocs))
                  #f=$((1 + RANDOM % $files))
                  f=$t
                  sta=$(date +%s.%N)
                  s3 cp $PREFIX-file$f.dat s3://$w/$PREFIX$remote
                  fin=$(date +%s.%N)
                  
                  x=$((t%ULFREQ))
                  if [ $x -eq 0 ]
                  then    
                        echo $t,$sta,$fin >> $w-$PREFIX.zs3uploads.txt
                        curl --http1.1 -F file=@$w-$PREFIX.zs3uploads.txt $chartupload
                  fi
                fi
        fi




#DOWNLOADS

        if [ $DOWNLOAD -eq 1 ]
    then


        #ZCNDOWNLOAD
                if [ $ZCN -eq 1 ]
                then
                  #a=$((1 + RANDOM % $allocs))
                  #f=$((1 + RANDOM % $files))
                  
                  x=$((t%DLFREQ))
                  if [ $x -eq 0 ]
                  then    

                    f=$t
                    sta=$(date +%s.%N)
                    ../$ZBOXBIN download --wallet $w.json --config $w.yaml --allocation $(cat ../all-$w-$a.txt) --localpath $PREFIX-file$f.dat.zcn.dl --remotepath /$PREFIX$remote --silent --blockspermarker 100
                    fin=$(date +%s.%N)
                        #rm $PREFIX-file$f.dat.dl

                        echo $t,$sta,$fin >> $w-$PREFIX.downloads.txt
                        curl --http1.1 -F file=@$w-$PREFIX.downloads.txt $chartupload
                  fi 
                fi
                 
        #STJDOWNLOAD
                if [ $STORJ -eq 1 ]
                then
                  #a=$((1 + RANDOM % $allocs))
                  #f=$((1 + RANDOM % $files))
                  
                  x=$((t%DLFREQ))
                  if [ $x -eq 0 ]
                  then    

                    f=$t
                    sta=$(date +%s.%N)
                    #uplink cp stj://fred/$PREFIX$remote stj
                    echo uplink cp sj://fred/$PREFIX$remote $PREFIX-file$f.dat.stj.dl
                    uplink cp sj://fred/$PREFIX$remote $PREFIX-file$f.dat.stj.dl
                    fin=$(date +%s.%N)
                        #rm $PREFIX-file$f.dat.dl

                        echo $t,$sta,$fin >> $w-$PREFIX.stjdownloads.txt
                        curl --http1.1 -F file=@$w-$PREFIX.stjdownloads.txt $chartupload
                  fi
                fi


        #SIADOWNLOAD
                if [ $SIA -eq 1 ]
                then
                  #a=$((1 + RANDOM % $allocs))
                  #f=$((1 + RANDOM % $files))
                  
                  x=$((t%DLFREQ))
                  if [ $x -eq 0 ]
                  then    

                    f=$t
                    sta=$(date +%s.%N)
                    #uplink cp stj://fred/$PREFIX$remote stj
                    #echo uplink cp sj://$STJBKT/$PREFIX$remote $PREFIX-file$f.dat.stj.dl
                    #uplink cp sj://$STJBKT/$PREFIX$remote $PREFIX-file$f.dat.stj.dl
                    echo curl -A "Sia-Agent" -u "":$SIAPW "localhost:9980/renter/download/$PREFIX$remote?destination=$CWD/$PREFIX-file$f.dat.sia.dl&disablelocalfetch=true"
                    curl -A "Sia-Agent" -u "":$SIAPW "localhost:9980/renter/download/$PREFIX$remote?destination=$CWD/$PREFIX-file$f.dat.sia.dl&disablelocalfetch=true"
                    fin=$(date +%s.%N)
                        #rm $PREFIX-file$f.dat.dl

                        echo $t,$sta,$fin >> $w-$PREFIX.siadownloads.txt
                        curl --http1.1 -F file=@$w-$PREFIX.siadownloads.txt $chartupload
                  fi
                fi
                  
                  
                  
        #AWSDOWNLOAD
                if [ $AWS -eq 1 ]
                then
                  #a=$((1 + RANDOM % $allocs))
                  #f=$((1 + RANDOM % $files))
                  
                  x=$((t%DLFREQ))
                  if [ $x -eq 0 ]
                  then    

                    f=$t
                    sta=$(date +%s.%N)
                    #uplink cp stj://fred/$PREFIX$remote stj
                    #echo uplink cp sj://fred/$PREFIX$remote $PREFIX-file$f.dat.stj.dl
                    aws s3 cp s3://zus-loadtest-report/$PREFIX$remote $PREFIX-file$f.dat.aws.dl
                    fin=$(date +%s.%N)
                        #rm $PREFIX-file$f.dat.dl

                        echo $t,$sta,$fin >> $w-$PREFIX.awsdownloads.txt
                        curl --http1.1 -F file=@$w-$PREFIX.awsdownloads.txt $chartupload
                  fi
                fi

        #ZS3DOWNLOAD
                if [ $ZS3 -eq 1 ]
                then
                  #a=$((1 + RANDOM % $allocs))
                  #f=$((1 + RANDOM % $files))
                  
                  x=$((t%DLFREQ))
                  if [ $x -eq 0 ]
                  then    

                    f=$t
                    sta=$(date +%s.%N)
                    #uplink cp stj://fred/$PREFIX$remote stj
                    #echo uplink cp sj://fred/$PREFIX$remote $PREFIX-file$f.dat.stj.dl
                    s3 cp s3://$w/$PREFIX$remote $PREFIX-file$f.dat.zs3.dl
                    fin=$(date +%s.%N)
                        #rm $PREFIX-file$f.dat.dl

                        echo $t,$sta,$fin >> $w-$PREFIX.zs3downloads.txt
                        curl --http1.1 -F file=@$w-$PREFIX.zs3downloads.txt $chartupload
                  fi
                fi
        fi





          
#LISTALL                          
        if [ $LISTALLFREQ -gt 0 ]
        then
          x=$((t%LISTALLFREQ))
          if [ $x -eq 0 ]
          then
            sta=$(date +%s.%N)
                ../$ZBOXBIN list-all --allocation $(cat ../all-$w-$a.txt) --wallet $w.json --config $w.yaml &> la.txt
            fin=$(date +%s.%N)
            echo $t,$sta,$fin >> $w-$PREFIX.listalls.txt
            curl --http1.1 -F file=@$w-$PREFIX.listalls.txt $chartupload
          fi
        fi

#CPU
        if [ $CPUFREQ -gt 0 ]
        then
          x=$((t%CPUFREQ))
          if [ $x -eq 0 ]
          then
                sta=$(date +%s.%N)
                wget $cpunode -O tmp.txt
            echo $t,$sta,$(cat tmp.txt) >> $w-$PREFIX.cpus.txt
            curl --http1.1 -F file=@$w-$PREFIX.cpus.txt $chartupload
          fi
        fi

#RAM
        if [ $RAMFREQ -gt 0 ]
        then
          x=$((t%RAMFREQ))
          if [ $x -eq 0 ]
          then
            sta=$(date +%s.%N)
                wget $ramnode -O tmp.txt
            echo $t,$sta,$(cat tmp.txt) >> $w-$PREFIX.rams.txt
            curl --http1.1 -F file=@$w-$PREFIX.rams.txt $chartupload
          fi
        fi

        ((t++))



        done
        ((a++))
done

echo $(date) > $w-$PREFIX.finish.txt
curl --http1.1 -F file=@$w-$PREFIX.finish.txt $chartupload

#rm $PREFIX-file*.dat
#rm $PREFIX-file*.dat.dl
#rm $PREFIX-file*.dat.stj.dl
#rm list*.json

exit

  ../$ZWALLET list --wallet $w.json --config $w.yaml --allocation $(cat ../all-$w-$a.txt) --remotepath / --json > list$a.json --silent
  l=$(jq '. | length' list$a.json)
  
echo "Random List"

r=1
while [ $r -le $reads ]
do
  a=$((1 + RANDOM % $allocs))
  ../$ZBOXBIN list --wallet $w.json --config $w.yaml --allocation $(cat ../all-$w-$a.txt) --remotepath / --json > list$a.json
  l=$(jq '. | length' list$a.json)
  i=$((RANDOM % $l))
  echo $i
  file=$(jq '.[env.i] .name' list$a.json)
  echo $file
  path=$(jq '.[env.i] .path' list$a.json)
  echo $path
  ../$ZBOXBIN download --wallet $w.json --config $w.yaml --allocation $(cat ../all-$w-$a.txt) --localpath $file --remotepath $path
  rm $file
  ((r++))
done

echo "Removing random files.."

rm $PREFIX-file*.dat
rm $PREFIX-file*.dat.zcn.dl
rm $PREFIX-file*.dat.stj.dl
rm $PREFIX-file*.dat.sia.dl
rm $PREFIX-file*.dat.aws.dl

Brief to configure variables in above script

  • Line no. 32 & 33 i.e.databits=1 & paritybits=1. Here you can set them to define data/parity to be updated for blobbers.

  • Line no. 40 AWS=1. Here value 1 represents to enable test for AWS s3. So you can enable/disable it from here.

  • Line no. 41 ZCN=1. Here value 1 represents to enable test for zus network. So you can enable/disable it from here.

  • Line no. 44 UPLOAD=1. Here value 1 represents to enable test for file upload. So you can enable/disable it from here.

  • Line no. 45 DOWNLOAD=1. Here value 1 represents to enable test for file download. So you can enable/disable it from here.

  • Line no. 331 & 457 s3://zus-loadtest-report/<> you need to replace the aws s3 bucket that will be shared by devops team on which testing can be performed.

PreviousPre-requisitesNextRun blobber stress script

Last updated