#!/bin/bash
# % Last Change: Mon Oct 31 11:49:55 PM 2022 CDT
# Usage: bash ChIP-seq_run.sh input.fq chip.fq

date
pipe_version="target_190731"
host="zhanglab/chip-seq base"
# get the absolute path
pipe_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
md5=`md5sum $0 | awk '{print $1}'`
calling_method="Narrow"

# read parameters
while getopts m:t:g:o:p:r:i:c:h opts
do case "$opts" in
m) marker="$OPTARG";;    # default 'no_annotation'
t) threads="$OPTARG";;    # default 24
g) species="$OPTARG";;    # hg19, hg38, mm9, mm10, danRer10
i) input_file="$OPTARG";;    # input file or control file
c) chip_file="$OPTARG";;    # chip-seq file
q) methylQA_cutoff="$OPTARG";; # methylQA insertion cutoff
h) echo "
Analyzing ChIP-seq data, generating QC plot and json reports.

usage: bash path-to-pipe/pipe.sh -g <hg38/hg19/mm10/mm9/danRer10> -i <input_file> -c <chip_file>

Options:    -g      input species. Please notice that each docker/singularity image is designed for one species only.
            -i      input file(contrl file)
            -c      chipseq file
            -q      methylQA read length cutoff, default 38.
            -t      threads used for the pipe, mainly involved in cutadapt and bwa mem steps [24].
"
exit;;
[?]) echo "
Analyzing ChIP-seq data, generating QC plot and json reports.

usage: path-to-pipe/pipe.sh -g <hg38/hg19/mm10/mm9/danRer10> -r <PE/SE> -o <read_file1> -p <read_file2>

Options:    -g      input species. Please notice that each docker/singularity image is designed for one species only.
            -i      input file(contrl file)
            -c      chipseq file
            -q      methylQA read length cutoff, default 38.
            -t      threads used for the pipe, mainly involved in cutadapt and bwa mem steps [24].
"
esac
done
if [ -z "$threads" ]
    then
    threads=24
fi

if [ -z "$marker" ]
    then
    marker='no_annotation'
fi

if [ -z "$methylQA_cutoff" ]
    then
    methylQA_cutoff=38
fi

notrim="--no-trim"
source $pipe_path'/qc_source.sh' $species

echo "processing" $chip_file
IFS=, read -r chip_file1 chip_file2 <<< "$chip_file"
chip=`echo ${chip_file1%.fastq*}`
chip=`echo ${chip%.fq.gz}`
chip=`echo ${chip%_R1}`

echo "processing" $input_file
IFS=, read -r input_file1 input_file2 <<< "$input_file"
input=`echo ${input_file1%.fastq*}`
input=`echo ${input%.fq.gz}`
input=`echo ${input%_R1}`

mkdir Processed_${chip}
ln -s `pwd`/${input_file1} ./Processed_${chip}
ln -s `pwd`/${chip_file1} ./Processed_${chip}
if [[ -n $input_file2 ]]; then ln -s `pwd`/${input_file2} ./Processed_${chip}; fi
if [[ -n $chip_file2 ]]; then ln -s `pwd`/${chip_file2} ./Processed_${chip}; fi
cd Processed_${chip}
mkdir 'QC_data_collection_'${input}_input
mkdir 'QC_data_collection_'${chip}_chip
touch QC_pipe_processing.log

# start record
date >> QC_pipe_processing.log
echo "Target file is $input_file $chip_file" >> QC_pipe_processing.log
echo "Specified species is $peak_genome" >> QC_pipe_processing.log
echo " " >> QC_pipe_processing.log

### cutadapt
s1.1_cutadapt_input(){
   echo 'triming' $input_file

   if [[ -n $input_file2 ]]; then
     $cutadapt $notrim -j $threads -a $adapter_1 -A $adapter_2 --quality-cutoff=15,10 --minimum-length=25 -o step1.1_trimed_${input}_R1_input.fastq -p step1.1_trimed_${input}_R2_input.fastq $input_file1 $input_file2 > step1.1_${input}_cutadapt_input.trimlog
     input_temp=`grep "Total read pairs processed:" step1.1_*input.trimlog | awk '{print $5}'`
     input_raw_reads=`echo ${input_temp//,}`
     input_temp2=`grep "Pairs written" step1.1_*input.trimlog | awk '{print $5}'`
     input_written_reads=`echo ${input_temp2//,}`
   else
     $cutadapt $notrim -j $threads -a $adapter_1 --quality-cutoff=15,10 --minimum-length=25 -o step1.1_trimed_${input}_input.fastq $input_file1 > step1.1_${input}_cutadapt_input.trimlog
     input_temp=`grep "Total reads processed:" step1.1_*input.trimlog | awk '{print $4}'`
     input_raw_reads=`echo ${input_temp//,}`
     input_temp2=`grep "Reads written" step1.1_*input.trimlog | awk '{print $5}'`
     input_written_reads=`echo ${input_temp2//,}`
   fi

   if [ $? == 0 ]
        then
        echo "step1.1, input: cutadapt trimming done" >> QC_pipe_processing.log
    else
        echo "step1.1, input: cutadapt trimming fail" >> QC_pipe_processing.log
        #exit 1
    fi
}

s1.1_cutadapt_chip(){
   echo 'triming' $chip_file
   if [[ -n $chip_file2 ]]; then
     $cutadapt $notrim -j $threads -a $adapter_1 -A $adapter_2 --quality-cutoff=15,10 --minimum-length=25 -o step1.1_trimed_${chip}_R1_chip.fastq -p step1.1_trimed_${chip}_R2_chip.fastq $chip_file1 $chip_file2 > step1.1_${chip}_cutadapt_chip.trimlog
     chip_temp=`grep "Total read pairs processed:" step1.1_*chip.trimlog | awk '{print $5}'`
     chip_raw_reads=`echo ${chip_temp//,}`
     chip_temp2=`grep "Pairs written" step1.1_*chip.trimlog | awk '{print $5}'`
     chip_written_reads=`echo ${chip_temp2//,}`
     chip_temp3=`grep "Total written" step1.1_*chip.trimlog | awk '{print $4}'`
     chip_written_basepairs=`echo ${chip_temp3//,}`
     read_depth=`echo "scale=2; $chip_written_basepairs / $genome_size" | bc -l`
     chip_written_percentage=`grep "Pairs written" step1.1_*chip.trimlog | awk '{print $6}'|sed 's/(//'|sed 's/)//'`
   else
     $cutadapt $notrim -j $threads -a $adapter_1 --quality-cutoff=15,10 --minimum-length=25 -o step1.1_trimed_${chip}_chip.fastq $chip_file1 > step1.1_${chip}_cutadapt_chip.trimlog
     chip_temp=`grep "Total reads processed:" step1.1_*chip.trimlog | awk '{print $4}'`
     chip_raw_reads=`echo ${chip_temp//,}`
     chip_temp2=`grep "Reads written" step1.1_*chip.trimlog | awk '{print $5}'`
     chip_written_reads=`echo ${chip_temp2//,}`
     chip_temp3=`grep "Total written" step1.1_*chip.trimlog | awk '{print $4}'`
     chip_written_basepairs=`echo ${chip_temp3//,}`
     read_depth=`echo "scale=2; $chip_written_basepairs / $genome_size" | bc -l`
     chip_written_percentage=`grep "Reads written" step1.1_*chip.trimlog | awk '{print $6}'|sed 's/(//'|sed 's/)//'`
   fi

   if [ $? == 0 ]
        then
        echo "step1.1, chip: cutadapt trimming done" >> QC_pipe_processing.log
    else
        echo "step1.1, chip: cutadapt trimming fail" >> QC_pipe_processing.log
        # exit 1
    fi
}

### fastqc
#[ -f ` ls 'step1.1_trimed_'$input*'.fastq' | head -1` ] && $fastqc -t $threads 'step1.1_trimed_'$input*'.fastq' -o .

s1.2_fastqc_input(){
   echo 'fastqc is processing input fastq file ...'
   [ -f ` ls 'step1.1_trimed_'$input*'.fastq' | head -1` ] && $fastqc -t $threads 'step1.1_trimed_'$input*'.fastq' -o .
   if [ $? == 0 ]
        then
        echo "step1.2, input fastqc process done" >> QC_pipe_processing.log
    else
        echo "step1.2, input fastqc process fail" >> QC_pipe_processing.log
        exit 1
    fi

    for zip in `ls | grep input_fastqc.zip`
    do
      unzip -o $zip
 #     mv $zip 'step1.2_'$zip
    done
    # 1.3 fastqc data collection
    echo -e "filename\tdeduplication_percentage\tmarker" > 'step1.3_dedup_percentage_'$input'_input.result'
    for file in `ls -d *input*fastqc/`
    do
        cd $file
        temp=`echo ${file##step1.1_trimed_}`
        input_out_name=`echo ${temp%*_fastqc/}`
        input_out_value=`grep 'Total Deduplicated Percentage' fastqc_data.txt | awk '{print $4}'`
        echo -e "$input_out_name\t$input_out_value\t$marker" >> ../'step1.3_dedup_percentage_'$input'_input.result'
        echo -e "item\t$input_out_name\t$input_out_name" > 'step1.3_duplication_summary_'$input_out_name'_input.result'
        grep 'Sequence Duplication Levels' -A 15 fastqc_data.txt >> 'step1.3_duplication_summary_'$input_out_name'_input.result'
        mv 'step1.3_duplication_summary_'$input_out_name'_input.result' ../'QC_data_collection_'${input}_input
        echo -e "$input_out_name\tfastqc_test" > 'step1.3_fastqc_summary_'$input_out_name'_input.result'
        awk -F "\t" '{print $1,$2}' OFS='\t' summary.txt >> 'step1.3_fastqc_summary_'$input_out_name'_input.result'
        mv 'step1.3_fastqc_summary_'$input_out_name'_input.result' ../'QC_data_collection_'${input}_input
        cd ..
    done

    if [ $? == 0 ]
        then
        echo "step1.3, input fastqc data_collection process done" >> QC_pipe_processing.log
    else
        echo "step1.3, input fastqc data_collection process fail" >> QC_pipe_processing.log
    fi

    sed 1d step1.3_dedup_percentage_${input}_input'.result' | cut -f 2 > temp_dedup.txt \
        && input_before_dedup=$(python -c "print(`awk '{s+=$1}END{print s}' temp_dedup.txt` * 0.01 /`cat temp_dedup.txt | wc -l`)") \
        && input_before_dup=$(python -c "print(1-$input_before_dedup*1.0)") \
        && rm temp_dedup.txt
    mv 'step1.3_dedup_percentage_'$input'_input.result' ./'QC_data_collection_'${input}_input
    mv *input*fastqc* ./'QC_data_collection_'${input}_input

    if [ $? == 0 ]
        then
        echo "step1.4, input: calculate replicate difference process done" >> QC_pipe_processing.log
    else
        echo "step1.4, input: calculate replicate difference process fail" >> QC_pipe_processing.log
    fi
}

s1.2_fastqc_chip(){
   echo 'fastqc is processing chip fastq file ...'
   [ -f ` ls 'step1.1_trimed_'$chip*'.fastq' | head -1` ] && $fastqc -t $threads 'step1.1_trimed_'$chip*'.fastq' -o .
   if [ $? == 0 ]
        then
        echo "step1.2, chip fastqc process done" >> QC_pipe_processing.log
    else
        echo "step1.2, chip fastqc process fail" >> QC_pipe_processing.log
        exit 1
    fi

    for zip in `ls | grep chip_fastqc.zip`
    do
      unzip -o $zip
      mv $zip 'step1.2_'$zip
    done
    # 1.3 fastqc data collection
    echo -e "filename\tdeduplication_percentage\tmarker" > 'step1.3_dedup_percentage_'$chip'_chip.result'
    for file in `ls -d *chip*fastqc/`
    do
        cd $file
        temp=`echo ${file##step1.1_trimed_}`
        chip_out_name=`echo ${temp%*_fastqc/}`
        chip_out_value=`grep 'Total Deduplicated Percentage' fastqc_data.txt | awk '{print $4}'`
        echo -e "$chip_out_name\t$chip_out_value\t$marker" >> ../'step1.3_dedup_percentage_'$chip'_chip.result'
        echo -e "item\t$chip_out_name\t$chip_out_name" > 'step1.3_duplication_summary_'$chip_out_name'_chip.result'
        grep 'Sequence Duplication Levels' -A 15 fastqc_data.txt >> 'step1.3_duplication_summary_'$chip_out_name'_chip.result'
        mv 'step1.3_duplication_summary_'$chip_out_name'_chip.result' ../'QC_data_collection_'${chip}_chip
        echo -e "$chip_out_name\tfastqc_test" > 'step1.3_fastqc_summary_'$chip_out_name'_chip.result'
        awk -F "\t" '{print $1,$2}' OFS='\t' summary.txt >> 'step1.3_fastqc_summary_'$chip_out_name'_chip.result'
        mv 'step1.3_fastqc_summary_'$chip_out_name'_chip.result' ../'QC_data_collection_'${chip}_chip
        cd ..
    done
    if [ $? == 0 ]
        then
        echo "step1.3, chip fastqc data_collection process done" >> QC_pipe_processing.log
    else
        echo "step1.3, chip fastqc data_collection process fail" >> QC_pipe_processing.log
    fi

    sed 1d step1.3_dedup_percentage_${chip}_chip'.result' | cut -f 2 > temp_dedup.txt \
        && chip_before_dedup=$(python -c "print(`awk '{s+=$1}END{print s}' temp_dedup.txt` * 0.01 /`cat temp_dedup.txt | wc -l`)") \
        && chip_before_dup=$(python -c "print(1-$chip_before_dedup*1.0)") \
        && rm temp_dedup.txt
    mv 'step1.3_dedup_percentage_'$chip'_chip.result' ./'QC_data_collection_'${chip}_chip
    mv *chip*fastqc* ./'QC_data_collection_'${chip}_chip

    if [ $? == 0 ]
        then
        echo "step1.4, chip: calculate replicate difference process done" >> QC_pipe_processing.log
    else
        echo "step1.4, chip: calculate replicate difference process fail" >> QC_pipe_processing.log
    fi
}

# step2.0, files check
s2.0_ref () {
    # refine chrom_size file (remove random and Unknown record)
    awk '{ if ((length($1) < 6) && (length($1) > 1)) print $0}' OFS='\t' $chrom_size > refined_chrom_size.txt
    chrom_size=`pwd`"/refined_chrom_size.txt"
}

### mapping
s2.1_bwa_input () {
   echo "aligning" $input_file
   if [[ -n $input_file2 ]]; then
     $bwa mem -t $threads $index step1.1_trimed_${input}_R1_input.fastq step1.1_trimed_${input}_R2_input.fastq | samtools view -bS - | samtools sort - -O 'bam' -o step2.1_trimed_${input}_input.bam -T temp_aln
   else
     $bwa mem -t $threads $index step1.1_trimed_${input}_input.fastq | samtools view -bS - | samtools sort - -O 'bam' -o step2.1_trimed_${input}_input.bam -T temp_aln
   fi

   if [ $? == 0 ]
        then
        echo "step2.1, input bwa alignment process done" >> QC_pipe_processing.log

    else
        echo "step2.1, input bwa alignment process fail" >> QC_pipe_processing.log
        exit 1
    fi
}

s2.1_bwa_chip () {
   echo "aligning" $chip_file
   if [[ -n $chip_file2 ]]; then
     $bwa mem -t $threads $index step1.1_trimed_${chip}_R1_chip.fastq step1.1_trimed_${chip}_R2_chip.fastq | samtools view -bS - | samtools sort - -O 'bam' -o step2.1_trimed_${chip}_chip.bam -T temp_aln
   else
     $bwa mem -t $threads $index step1.1_trimed_${chip}_chip.fastq | samtools view -bS - | samtools sort - -O 'bam' -o step2.1_trimed_${chip}_chip.bam -T temp_aln
   fi

   if [ $? == 0 ]
        then
        echo "step2.1, chip bwa alignment process done" >> QC_pipe_processing.log

    else
        echo "step2.1, chip bwa alignment process fail" >> QC_pipe_processing.log
        exit 1
    fi
}

# step2.2, removing low mapQ reads and count reads distribution (mapQ=0 makes no sense here, because they are not reliable)
s2.2_distri_input () {
    $samtools view -h 'step2.1_trimed_'$input'_input.bam' > input.sam \
        && awk '$5>0' input.sam | sed '/^@/d' - | cat <(grep '^@' input.sam) - > output.sam \
        && cat output.sam | awk '{print $3}' | sort -k1,1V | uniq -c > count_no_mapq0.txt \
        && awk '! /random/ && ! /Un/ && /chr/ ' count_no_mapq0.txt | awk '{print $2, $1}' OFS="\t" | sort -k1,1 -V -s > temp2.2.txt
        mv temp2.2.txt 'step2.2_chrom_count_'$input'_input.txt'

    $methylQA density -S $chrom_size output.sam
    cut -f 1 output.extended.bed | uniq -c > count_unique.txt
    awk '{print $2, $1}' OFS="\t" count_unique.txt | sort -k1,1 -V -s > 'step2.2_chrom_count_unique_'$input'_input.txt'
#    rm temp*
    rm output*
    rm input.sam
    awk -F "\t" '{print $2}' 'step2.2_chrom_count_unique_'$input'_input.txt' | paste 'step2.2_chrom_count_'$input'_input.txt' - | awk -F "\t" -v marker=$marker '{print $1,$2+0,$3+0,marker}' OFS="\t" > ./'QC_data_collection_'${input}_input/'step2.2_chrom_count_'$input'_input.result'
    rm step2.2_chrom_count*txt
}

s2.2_distri_chip () {
    $samtools view -h 'step2.1_trimed_'$chip'_chip.bam' > input.sam \
        && awk '$5>0' input.sam | sed '/^@/d' - | cat <(grep '^@' input.sam) - > output.sam \
        && cat output.sam | awk '{print $3}' | sort -k1,1V | uniq -c > count_no_mapq0.txt \
        && awk '! /random/ && ! /Un/ && /chr/ ' count_no_mapq0.txt | awk '{print $2, $1}' OFS="\t" | sort -k1,1 -V -s > temp2.2.txt
        mv temp2.2.txt 'step2.2_chrom_count_'$chip'_chip.txt'

    $methylQA density -S $chrom_size output.sam
    cut -f 1 output.extended.bed | uniq -c > count_unique.txt
    awk '{print $2, $1}' OFS="\t" count_unique.txt | sort -k1,1 -V -s > 'step2.2_chrom_count_unique_'$chip'_chip.txt'
#    rm temp*
    rm output*
    rm input.sam
    awk -F "\t" '{print $2}' 'step2.2_chrom_count_unique_'$chip'_chip.txt' | paste 'step2.2_chrom_count_'$chip'_chip.txt' - | awk -F "\t" -v marker=$marker '{print $1,$2+0,$3+0,marker}' OFS="\t" > ./'QC_data_collection_'${chip}_chip/'step2.2_chrom_count_'$chip'_chip.result'
    rm step2.2_chrom_count*txt
}

### methylQA
s3.1_methylQA_input () {
    echo 'methylQA processing ...'
    grep -v ^chrM $chrom_size > nochrM_chrom_size.txt \
        && rm $chrom_size \
        && chrom_size=nochrM_chrom_size.txt
    echo "the chrom file for methylQA is $chrom_size" >> QC_pipe_processing.log

    $methylQA density -o step3.1_methylQA_${input}_input $chrom_size 'step2.1_trimed_'$input'_input.bam'
    if [ $? == 0 ]
        then
        echo "step3.1, input mathylQA process done" >> QC_pipe_processing.log
    else
        echo "step3.1, input mathylQA process fail" >> QC_pipe_processing.log
    fi

    # mapping status
    input_map_mapped=`grep 'mappable reads' step3.1_methylQA_$input'_input.report' | awk '{print $4}'`
    input_map_uniq=`grep '(mapQ >= 10)' step3.1_methylQA_$input'_input.report' | awk '{print $8}'`
    input_map_effect=`grep 'non-redundant' step3.1_methylQA_$input'_input.report' | awk '{print $6}'`
    input_mapped_ratio=`echo "scale=2; $input_map_mapped/$input_raw_reads" | bc -l`
    input_effect_ratio=`echo "scale=2; $input_map_effect/$input_raw_reads" | bc -l`

#    # unique chrM ratio from step2.2
#    input_unique_chrM_ratio=`python -c "print($input_unique_chrM*1.0 / ($input_unique_chrM+$input_map_uniq) )"`
#    echo -e "non_chrM_unique_mapped\tchrM\tunique_chrM_ratio" > 'step2.2_unique_chrM_ratio_'$input'_input.result'
#    echo -e "$input_map_uniq\t$input_unique_chrM\t$input_unique_chrM_ratio" >> 'step2.2_unique_chrM_ratio_'$input'_input.result'
#    mv 'step2.2_unique_chrM_ratio_'$input'_input.result' ./'QC_data_collection_'${input}_input

    input_nodup_ratio=`echo "scale=3; $input_map_effect/$input_map_uniq" | bc -l`
    input_after_dup=$(python -c "print(1-$input_nodup_ratio*1.0)")

    input_useful=`grep 'non-redundant' step3.1_methylQA_${input}'_input.report' | awk '{print $6}'`
    input_single_end=`wc -l step3.1_methylQA_${input}'_input.extended.bed' | awk '{print $1}'`
    input_uf_ratio=`echo "scale=3; $input_useful / $input_raw_reads" | bc -l`
    echo -e "file\ttotal\tuseful\tuseful_ratio\tsingle_end" > 'step3.1_useful_reads_'${input}_input.result
    echo -e "$input\t$input_raw_reads\t$input_useful\t$input_uf_ratio\t$input_single_end" >> 'step3.1_useful_reads_'${input}_input.result
    mv 'step3.1_useful_reads_'${input}_input.result ./'QC_data_collection_'${input}_input
  #  sort -n 'step3.1_methylQA_'$input'_input.insertdistro' | uniq -c | awk '{print $2,$1}' > 'step3.1_insertion_distri_'$input'_input.result'
    mv step3.1_*_input*.pdf ./'QC_data_collection_'${input}_input
#    mv step3.1_methylQA*_input* ./'QC_data_collection_'${input}_input
   # rm step3.1_methylQA_*bigWig
}

s3.1_methylQA_chip () {
    $methylQA density -o step3.1_methylQA_${chip}_chip $chrom_size 'step2.1_trimed_'$chip'_chip.bam'
    if [ $? == 0 ]
        then
        echo "step3.1, chip mathylQA process done" >> QC_pipe_processing.log
    else
        echo "step3.1, chip mathylQA process fail" >> QC_pipe_processing.log
    fi

    # mapping status
    chip_map_mapped=`grep 'mappable reads' step3.1_methylQA_$chip'_chip.report' | awk '{print $4}'`
    chip_map_uniq=`grep '(mapQ >= 10)' step3.1_methylQA_$chip'_chip.report' | awk '{print $8}'`
    chip_map_effect=`grep 'non-redundant' step3.1_methylQA_$chip'_chip.report' | awk '{print $6}'`
    chip_mapped_ratio=`echo "scale=2; $chip_map_mapped/$chip_raw_reads" | bc -l`
    chip_effect_ratio=`echo "scale=2; $chip_map_effect/$chip_raw_reads" | bc -l`

#    # unique chrM ratio from step2.2
#    chip_unique_chrM_ratio=`python -c "print($chip_unique_chrM*1.0 / ($chip_unique_chrM+$chip_map_uniq) )"`
#    echo -e "non_chrM_unique_mapped\tchrM\tunique_chrM_ratio" > 'step2.2_unique_chrM_ratio_'$chip'_chip.result'
#    echo -e "$chip_map_uniq\t$chip_unique_chrM\t$chip_unique_chrM_ratio" >> 'step2.2_unique_chrM_ratio_'$chip'_chip.result'
#    mv 'step2.2_unique_chrM_ratio_'$chip'_chip.result' ./'QC_data_collection_'${chip}_chip

    chip_nodup_ratio=`echo "scale=3; $chip_map_effect/$chip_map_uniq" | bc -l`
    chip_after_dup=$(python -c "print(1-$chip_nodup_ratio*1.0)")

    chip_useful=`grep 'non-redundant' step3.1_methylQA_${chip}'_chip.report' | awk '{print $6}'`
    chip_single_end=`wc -l step3.1_methylQA_${chip}'_chip.extended.bed' | awk '{print $1}'`
    chip_uf_ratio=`echo "scale=3; $chip_useful / $chip_raw_reads" | bc -l`

    chip_map_total_reads=`grep 'total reads' step3.1_methylQA_${chip}'_chip.report' | awk '{print $4}'`
    NRF=`echo "scale=3; $chip_useful / $chip_map_mapped" | bc -l`

    echo -e "file\ttotal\tuseful\tuseful_ratio\tsingle_end\tNon-Redundant Fraction (NRF)" > 'step3.1_NRF_useful_reads_'${chip}_chip.result
    echo -e "$chip\t$chip_raw_reads\t$chip_useful\t$chip_uf_ratio\t$chip_single_end\t$NRF" >> 'step3.1_NRF_useful_reads_'${chip}_chip.result
    mv 'step3.1_NRF_useful_reads_'${chip}_chip.result ./'QC_data_collection_'${chip}_chip
   # sort -n 'step3.1_methylQA_'$chip'_chip.insertdistro' | uniq -c | awk '{print $2,$1}' > 'step3.1_insertion_distri_'$chip'_chip.result'
    mv step3.1_*_chip*.pdf ./'QC_data_collection_'${chip}_chip
   # rm step3.1_methylQA_*bigWig

   # PBC
   perl -ane 'chomp;my $l=$F[2]-$F[1];print $_."\t".$l."\n";' step3.1_methylQA_${chip}_chip.extended.bedGraph > step3.1_methylQA_${chip}_chip.extended.bedGraph.PBC
   #M1: number of genomic locations where exactly one read maps uniquely
   #M2: number of genomic locations where two reads map uniquely
   M1=$(awk '$4==1' step3.1_methylQA_${chip}_chip.extended.bedGraph.PBC|awk '$5>100'|perl -e 'my $sum;foreach(<>){chomp;my $t=(split/\t/,$_)[4];$sum=$sum+$t;} print $sum;')
   M2=$(awk '$4==2' step3.1_methylQA_${chip}_chip.extended.bedGraph.PBC|awk '$5>100'|perl -e 'my $sum;foreach(<>){chomp;my $t=(split/\t/,$_)[4];$sum=$sum+$t;} print $sum;')
   PBC1=`echo "scale=3; $chip_useful / $chip_map_uniq" | bc -l`
   PBC2=`echo "scale=3; $M1 / $M2" | bc -l`

}

## peakcall parameters:
s3.2_peakcall () {
    echo 'peak calling ...'
   if [[ -n $input_file2 ]]; then
     $macs2 callpeak -t "step3.1_methylQA_"$chip"_chip.extended.bed" -c "step3.1_methylQA_"$input"_input.extended.bed" -f BEDPE -g $peak_genome -n step3.2_peakcall_${chip} -q 0.05 --nomodel --shift -100 --extsize 200
   else
     $macs2 callpeak -t "step3.1_methylQA_"$chip"_chip.extended.bed" -c "step3.1_methylQA_"$input"_input.extended.bed" -g $peak_genome -n step3.2_peakcall_${chip} -q 0.05 --nomodel --shift -100 --extsize 200
   fi

   if [ $? == 0 ]
        then
        echo "step3.2, macs2 peak calling process done" >> QC_pipe_processing.log
    else
        echo "step3.2, macs2 peak calling process fail" >> QC_pipe_processing.log
    fi
   # mv step3.1_methylQA_${chip}_chip* 'QC_data_collection_'${chip}_chip
    awk '{print $3-$2+1}' 'step3.2_peakcall_'${chip}'_peaks.narrowPeak' | sort -n | uniq -c | awk '{print $2,$1}' > 'step3.2_peak_length_distri_'$chip'.result'
    mv 'step3.2_peak_length_distri_'$chip'.result' ./'QC_data_collection_'${chip}_chip
}

# step4.0, set variable
s4.0_set () {
    peak='step3.2_peakcall_'$chip'_peaks.narrowPeak'
    bed='step3.1_methylQA_'$chip'_chip.extended.bed'
}

# 4.1, RUP and insertion site
s4.1_rup () {
    total=`wc -l $bed | awk '{print $1}'`
    sum=`intersectBed -iobuf 200M -a $bed -b $peak -f 0.5 -u | wc -l`
    ratio=`echo "scale=2; $sum*100/$total" | bc -l`
    peak_number=`wc -l $peak | awk '{print $1}'`
    if [ $? == 0 ]
        then
        echo "step4.1, reads under peak ratio calculation process done" >> QC_pipe_processing.log
    else
        echo "step4.1, reads under peak ratio calculation process fail" >> QC_pipe_processing.log
    fi
}

s4.6_visualization () {
    #summarize results
    echo -e "file\ttotal\twritten_reads\tmapped\tmapped_ratio\tuniq_mapped\tnon_redundant_uniq_mapped\teffect_ratio\tfastqc_dup\tafter_align_dup\tnumber_of_reads_under_peak\tRUP\tNon_Redundant_Fraction(NRF)\tPCR_Bottlenecking_Coefficient_1(PBC1)\tPCR_Bottlenecking_Coefficient_2_(PBC2)\ttotal_reads_for_map\twritten_percentage\tPeak_number" > 'QC_data_collection_'$chip'.result'
    echo -e "$chip\t$chip_raw_reads\t$chip_written_reads\t$chip_map_mapped\t$chip_mapped_ratio\t$chip_map_uniq\t$chip_map_effect\t$chip_effect_ratio\t$chip_before_dup\t$chip_after_dup\t$sum\t$ratio\t$NRF\t$PBC1\t$PBC2\t$chip_map_total_reads\t$chip_written_percentage\t$peak_number" >> 'QC_data_collection_'$chip'.result'
    mv 'QC_data_collection_'$chip'.result' ./'QC_data_collection_'${chip}_chip

    #input
    echo -e "file\ttotal\twritten_reads\tmapped\tmapped_ratio\tuniq_mapped\tnon_redundant_uniq_mapped\teffect_ratio\tfastqc_dup\tafter_align_dup" > 'QC_data_collection_'$input'.result'
    echo -e "$input\t$input_raw_reads\t$input_written_reads\t$input_map_mapped\t$input_mapped_ratio\t$input_map_uniq\t$input_map_effect\t$input_effect_ratio\t$input_before_dup\t$input_after_dup" >> 'QC_data_collection_'$input'.result'
    mv 'QC_data_collection_'$input'.result' ./'QC_data_collection_'${input}_input

    if [ -z $chip ] || [ -z $chip_raw_reads ] || [ -z $chip_map_mapped ] || [ -z $chip_mapped_ratio ] || [ -z $chip_map_uniq ] || [ -z $chip_written_reads ] || [ -z $chip_map_effect ] || [ -z $chip_effect_ratio ] || [ -z $chip_nodup_ratio ] || [ -z $sum ]|| [ -z $ratio ] || [ -z $chip_before_dedup ] || [ -z $NRF ] || [ -z $PBC1 ] || [ -z $PBC2 ]
    then
        echo "step4.6, sumarizing result process fail" >> QC_pipe_processing.log
    else
        echo "step4.6, sumarizing result process done" >> QC_pipe_processing.log
    fi

    # plot and json
    time=`head -1 QC_pipe_processing.log | sed 's/ /_/g'`
    image_id=`bash $pipe_path'/find_image_ID_digest.sh' $host 2> /dev/null | awk '{print $2}'`
    if [ -z "$image_id" ]
    then
    image_id="failed_to_get_id"
    fi
    # clean result
    find . -name "*.result" | xargs sed -i 's/^-e //'

    cd ./'QC_data_collection_'${chip}_chip
    # calculate score
    ### 1) useful reads
    cal_score () {
        score=0
        echo -e "iterm\tnumber\tscore" > step4.6_score_calculation_${chip}.result

        _usable_fra=`cut -f 7 QC_data_collection_*.result | sed '1d'` \
            && if (( $_usable_fra >= 40000000 )); then _change=2; elif (( $_usable_fra >= 20000000 )) && (( $_usable_fra < 40000000 )); then _change=1; else _change=0; fi \
            && let "score+=$_change"
            echo -e "Number_NR_U_M\t$_usable_fra\t$_change" >> step4.6_score_calculation_${chip}.result \
            && unset _change

        _nrf=`cut -f 6 step3.1_NRF_useful_reads_*.result | sed '1d'` \
            && if (( $(echo "$_nrf>=0.8" | bc -l) )); then _change=2; elif (( $(echo "$_nrf>=0.5" | bc -l) )) && (( $(echo "$_nrf<0.8" | bc -l) )); then _change=1; else _change=0; fi \
            && let "score+=$_change"
            echo -e "NRF\t$_nrf\t$_change" >> step4.6_score_calculation_${chip}.result \
            && unset _change

        _pbc1=`cut -f 14 QC_data_collection_*.result | sed '1d'` \
            && if (( $(echo "$_pbc1>=0.8" | bc -l) )); then _change=2; elif (( $(echo "$_pbc1<0.8" | bc -l) )) && (( $(echo "$_pbc1>=0.5" | bc -l) )); then _change=1; else _change=0; fi \
            && let "score+=$_change"
            echo -e "PBC1\t$_pbc1\t$_change" >> step4.6_score_calculation_${chip}.result \
            && unset _change

        _pbc2=`cut -f 15 QC_data_collection_*.result | sed '1d'` \
            && if (( $(echo "$_pbc2>=3" | bc -l) )); then _change=2; elif (( $(echo "$_pbc2<3" | bc -l) )) && (( $(echo "$_pbc2>=1" | bc -l) )); then _change=1; else _change=0; fi \
            && let "score+=$_change"
            echo -e "PBC2\t$_pbc2\t$_change" >> step4.6_score_calculation_${chip}.result \
            && unset _change

        _rup=`cut -f 12 QC_data_collection_*.result | sed '1d'` \
            && if (( $(echo "$_rup>=20" | bc -l) )); then _change=2; elif (( $(echo "$_rup>=12" | bc -l) )) && (( $(echo "$_rup<20" | bc -l) )); then _change=1; else _change=0; fi \
            && let "score+=$_change"
            echo -e "RUP\t$_rup\t$_change" >> step4.6_score_calculation_${chip}.result \
            && unset _change

    }
    cal_score

    Rscript $pipe_path'/visualization.R' $chip $species $chip_written_reads $pipe_version $time $image_id $calling_method
    if [ $? == 0 ]
        then
        echo "step4.6, plot process done" >> ../QC_pipe_processing.log
    else
        echo "step4.6, plot process fail" >> ../QC_pipe_processing.log
    fi

    sed 's/\[/{/g' ${chip}_report.json | sed '/    {/d' | sed '/\]/d' | sed 's/    }/  },/g' | sed 's/"!/{/g' | sed 's/!"/}/g' | sed 's/"?/[/g' | sed 's/?"/]/g' | sed 's/@/"/g' | tac | sed '2s/},/}/g' | tac | sed 's/\\r//g' | sed "s/MD5ToBeChange/$md5/g" > QC_$chip'.json'
#    rm $chip'_report.json'
    mv QC_$chip'.json' ../
    paste <(cut -f 1-8 QC_data_collection_${chip}.result) <(cut -f 5 step3.1_NRF_useful_reads_${chip}_chip.result) <(cut -f 9- QC_data_collection_${chip}.result) > QC_table_${chip}.result

    mkdir 'plots_collection_'$chip
    mv *png 'plots_collection_'$chip
 #   rm $chip'_report.txt'
    cd ..

    rm pesudo_bl.txt 2> /dev/null
#    rm refined_chrom_size.txt nochrM_chrom_size.txt
    find -type l -delete

#    /usr/local/bin/multiqc .
#    rename 's/multiqc/step4.6_multiqc/' multiqc*
}

# step-by-step
s1.1_cutadapt_input
s1.1_cutadapt_chip
s1.2_fastqc_input
s1.2_fastqc_chip
s2.0_ref
s2.1_bwa_input
s2.1_bwa_chip
s2.2_distri_input
s2.2_distri_chip
s3.1_methylQA_input
s3.1_methylQA_chip
s3.2_peakcall
s4.0_set
s4.1_rup
s4.6_visualization