#!/usr/bin/perl -w
# Needed moudles
# In order to understand the code, it's best to read the DEBUG BUS USER MANUAL first
use strict;
use English;
use File::Path;
use File::stat;
use Cwd;
use POSIX qw(ceil floor);
use warnings;
use Net::TcpDumpLog;

# constants
use constant BUFFER => 0;
use constant NETWORK => 1;
use constant PCI =>2;


# Consatns for the Kernel Debug ability
my $general_info_size = 4;
my $fill_regs_size = 4;
my $end_read_regs_size = 7;

# The script perforce version
my $VERSION = '$Revision: #24 $';
# The chip version {E1 E1H ..}
my $CHIP_NUM;
# The chip for which the meta data is needed (Useful when working on E1H in E1 mode).
my $CHIP_META;
# a regualer expression which holds known chips (The scripts required a csv folder CSV$CHIP_NUM for each version
my $CHIPS='(E1|E1H|E2|E3)';
my %CHIPS_P4_NAME=('E1' => 'everest1', 'E1H' => 'everest1h', 'E2' => 'everest2', 'E3' => 'everest2');

# Tools base smc_path
my $TOOLS_BASE="..\\..\\..\\..\\common\\";
# Tools needed by the script location
my $MC_SCRIPTS_DIR="microcode\\scripts\\";
# The location of tee (tee is activated only when using piping so from there comes the need for |
my $TEE="|".$TOOLS_BASE.$MC_SCRIPTS_DIR."tee_win.exe ";
# The location of merge
my $MERGE=$TOOLS_BASE.$MC_SCRIPTS_DIR."merge.pl";
# The location of gawk
my $GAWK=$TOOLS_BASE.$MC_SCRIPTS_DIR."gawk.exe";
# The location of formatMsg
my $formatMsg=$TOOLS_BASE.$MC_SCRIPTS_DIR."formatMsg.awk";
# The location of splitter
my $SPLITTER=$TOOLS_BASE.$MC_SCRIPTS_DIR."splitter.awk";
# The location of gawk
my $PARSE_PRINTF=$TOOLS_BASE.$MC_SCRIPTS_DIR."parse_dbus_printf.pl";

# Structed data's location under SMC
my $DATA_BASE="microcode\\binary\\";

# Allowed working modes and formats
my %MODES = ('TDM32' => 0, 'TDM64' => 3, '64BIT' => 1, '24BIT' => 2);
my %FORMATS = ('NORMAL'=>0, 'FMT' => 1, 'STORE' =>2, 'SEM' =>3, 'PRINTF'=>4, 'STORM'=>4);
my %STORE_STORM =('PBF' => 'X', 'UPB' => 'U');

# The framing mode in which the parsing is done  
my $MODE;
# The format to parse according to
my $FORMAT;
# If working in STORE FORMAT the INTERFACE address is needed
# STORE FORMAT pass the output to the merge script which work upon the INTERFACE ADDRESS in the SDM
my $STOREADDRESS;

# If working in TDM mode there are 2 options: TDM32 & TDM64
my $TDM64;

# If working in FMT format, will count the number of messages
my $FMTCOUNTER=0;

# if working in other modes, will count the number of messages
my $CHUNKCOUNTER=0;

# This variables are used if limit for the output file was requested
# WRITTEN count the amount of bytes written by now and limit holds the threshold
# LIMIT_BY_SPLIT is used to notify state when the split can only be done on the fmt output
# (cases when this script can tell how to seprated the messages, which are:
# a profile which require the merge script (STORE format or SEM format in some cases - see below in
# set64BIT) and in one of the modes of the sem script - when the frame comes 2 cycles too late.)
my $WRITTEN=0;
my $LIMIT;
my $LIMIT_BY_SPLIT=0;

# The number of allowed warnings in the fmt
# default is 0 (can be change by using -warn <number>)
my $WARNINGS;

# Variables holding information about files.
my @INPUT_FILES;
my @OUTPUT_FILE;
my $ERROR_FILE;
my $FMT_OUTPUT;
my $OUTPUT_FOLDER;
my $PROFILE_FILE;

# a template for output files names
my $OUTFILE_TEMPLATE;
# a template for fmt files names
my $FMT_TEMPLATE;
my $PROFILE;
my $PARSE_MODE;
# an object for the network data
my $NETDATA;
# if NOFIRST not equals 0 the first message will be dropped. 
my $NOFIRST=0;
# useful for dropping the first message in TDM mode
my %NOFIRSTDM;
# holds the data per each client until reaching the frame
my %BUFFER_PER_CLIENT;
# the header number in the profiles file
my $HEADERNUM=0;
# The header in binary format
my $HEADERBIN=""; 
# The packet currently being parsed
my $PACKETCOUNTER=0;
# The packet currently being parsed (Global number)
my $CURRENT_PACKET_NUM = 0;
# Global number of the first packet to handle
my $START_PACKET_NUM = 0;
# holds the files opened for the blocks in TDM mode
my %FILES;
my @DATA_FORMAT;
# an array which holds the parsing information foreach calendar slot. (if working in 64 bit only slot 0 is used,
# if working in 24 bit only 0 1 2 are used)
# Each slot in the array is an hash table build as follows:
# { "Size" - the size of the total relevant data in the slot,
#   "block_name" - the block that outputted the data in that slot
#   "line_name" - the name of the line accroding to which the blcok outputted the data
#   "groups" - an array which hold the data about the group which the line are build from, see below }
# The groups array have a slot for each group in the line
# The slot is a hash table build as follows:
# {"Size" - the group size (8/16/24/32/64},
#  "group_name" - the name of the group,
#  "Data" - an array where each slot is an hash representing a field in the group,
#           The fields are sorted from lsb to msb where each field has:
#           "Comment" - Comments about the field from the CSV file
#           "Name" - the field name with the relevant bits in that interface inside brackets (e.g. debug_bus_cfc[3,1]) }


# The fmt.gp file to be use in formatMsg
my $FMT_FILE;
# The fmt.gp file to be use in formatMsg for printf
my $PFMT_FILE;
# The smc path to use fmt.gp files according too
my $SMC_PATH;
# The varible used to create a pipeline process.
# It will hold the pipe command line is piping is used.
# Otherwise will hold the value "NOT"
my $PIPELINE="SET";
# If the user doesn't want intermediate files NOTEE will be set to 1
my $NOTEE=0;
# represent the name of the current file to write to if tee is need.
my $CURRTEE;
# represnt the sem mode, need to know the working mode to use and to know how to limit the files.
my $SEM_MODE=0;
my $STORM_MODE=0;
# If the user doesn't want files per block will be set to 1 (have meaning only in TDM32 mode).
my $NOSEP=0;

###### set profile data #####################################################################
# reads the profiles input file
# puts the relevant profile into an array 
# checks the working mode and format
# uses setProfile(TDM|24BIT|64BIT) to set DATA_FORMAT
# uses parse(TDM|24BIT|64BIT) to parse the data (doesn't activate outter programs like fmt)
#############################################################################################
sub setProfile
{
    my ($profile_file,$profile_name)=@_;
    my ($line, $line_number);
    my (@tmpArray);
    my $pace='';
    die "Can't find profiles' file $profile_file. Please make sure the file exists.\n"
        unless (open(PROFILE, $profile_file));
    
    # if -filter was uses find the relevant header in the profiles file        
    if ($HEADERNUM!=0) {
        do{
            $line=<PROFILE>;
            if ($line=~/^HEADER$HEADERNUM\s+([\dA-Fa-f]{28})/) {
                $HEADERBIN=pack("H28",$1);
                $line="";
            }
        } while ($line ne "" && !(eof PROFILE));
        die "The requested header doesn't exists, recheck profiles file\n." unless ($HEADERBIN ne "");
    }
    # finding the requested profile in the file
    do{
        $line=<PROFILE>;
    }while (!(eof PROFILE)&&!($line=~/PROFILE: $profile_name($|\s)/));
        
    die "Can't find profile: $profile_name in $profile_file. Recheck profile's name and profiles' file.\n"
        unless (!(eof PROFILE)); 
    
    # Inserting the profile lines into a array excluding comments
    do{
        $line=<PROFILE>;
        if ($line!~/^\s*#.+/){
            $line=~s/(#.+)/ /;
            @tmpArray=(@tmpArray,$line);
        }
    } while (!(eof PROFILE)&&!($line eq "\n")&&!($line=~/^ENDPROFILE/));
    
    # Exitting if the profile syntax is not well defined
    die "Can't find end profile: $profile_name in $profile_file. Recheck profile's name and profiles' file.\n"
        unless ($line=~/ENDPROFILE/); 
    close PROFILE;
    # getInput opens the relevant input file
    getInput();
    
    # Setting the mode 
    if ($tmpArray[0]=~/MODE:\s+(.+)/){
        $MODE=$MODES{$1};
    } else { die "Profile is not properly defined, failed to find active mode.\n"}

    shift(@tmpArray);
    
    # Getting the calendar pace
    if ($tmpArray[0]=~/PACE:\s+(\d+)/) {
        $pace="The calendar pace was $1 cycles.\n";
        shift(@tmpArray);
    }
    # Setting the format
    if ($tmpArray[0]=~/FORMAT:\s+(.+)/) {
        $FORMAT=$FORMATS{$1};
        shift(@tmpArray);
    }
    
    # Ensuring STORE FORMAT has STOREADDRESS define
    if ($FORMAT==$FORMATS{'STORE'}) {
        if ($tmpArray[0]=~/^STOREADDRESS:\s+(\d{16})$/) {
            $STOREADDRESS=$1;
            shift(@tmpArray);
        }
        else { die "FORMAT Store required STOREADDRESS in the line after it. Recheck Profile.\n" }
    }
    # Ensuring that the mode and the format relevant together
    # and that limit is defined only with relevant modes and formats
    die "Tdm32/Tdm64 can be parsed only into NORMAL,RH or PRINTF format.\n" unless (($MODE!=$MODES{'TDM32'} && $MODE!=$MODES{'TDM64'})||$FORMAT==$FORMATS{'STORM'}||$FORMAT==$FORMATS{'NORMAL'});
    die "64Bit can't be parsed into printf foramt.\n" unless ($MODE!=$MODES{'64BIT'}||$FORMAT!=$FORMATS{'STORM'});
    die "24BIT mode can be parsed only in NORMAL format.\n" unless ($MODE!=$MODES{'24BIT'} || $FORMAT==$FORMATS{'NORMAL'});
    
    # fix comment!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    # setting the output for the file
    # since the script now works with piping it has to know the which scripts are going to be activated
    # before running the sem script, so 64BIT requires special treatment
    
    # Set the porfile and parsing the data according to mode
    if ($MODE==$MODES{'TDM32'}) {
        $STORM_MODE=setTDM32(@tmpArray);
        setOUT();
        printf OUT $pace;
        parseTDM();
    } elsif ($MODE==$MODES{'64BIT'}) {
        $SEM_MODE=set64BIT(@tmpArray);
        setOUT();
        printf OUT $pace;
        parse64BIT($FORMAT);
        # only 64 bit mode has more than one allowed format (which effect the output)
    } elsif ($MODE==$MODES{'24BIT'}) {
        set24BIT(@tmpArray);
        setOUT();
        printf OUT $pace;
        parse24BIT();
    } elsif ($MODE==$MODES{'TDM64'}) {
        $STORM_MODE=setTDM64(@tmpArray);
        setOUT();
        printf OUT $pace;
        parseTDM();
    } else {die "UNKNOWN MODE\n." }
} 
    
###### set profile for 64 bit mode #############################
# receive an array of the profile data
# set Data Format to include the relevant block information
# (only one in 64BIT mode, so only slot 0 in @DATA_FORMAT is set)
################################################################
sub set64BIT {
    my (@tmpArray)=@_;
    my ($block_name,$line_name);
    #reading the blocks configuration
    # 64Bit block syntax is:
    # <block_name> <line_name>
    # Only 1 block is allowed
    $tmpArray[1]=~/(.+)\s+(.+)/;
    ($block_name,$line_name)=($1,$2);
    setDataFormat(0,$block_name,$line_name,0);
    # When activating the sem script with -c it return 3 is the this line require the merge script
    # if the result isn't 0, the file can't be splitted by this script and only the fmt output is
    # is splitted by using splitter
    if ($FORMAT==$FORMATS{'SEM'}) {
        return system("perl -w sem.pl -c $line_name");
    } else {
        return 0;
    }
}

###### set profile for 24 bit mode #############################
# receive an array of the profile data
# set Data Format to include the relevant block information
# Each slot in the DATA_FORMAT array represent a byte
# Only the 3 first Slots are used.
################################################################
sub set24BIT {
    my (@tmpArray)=@_;
    my ($block_name,$line_name);
    my $line_type=0;
    my ($i)=1;
    
    #reading the blocks configuration
    # 24Bit block syntax is:
    # <block_name> <line_name> <byte_enable 3 binary digits> <shift (0-7) | optional>
    while($tmpArray[$i]!~/ENDBLOCKS/&&$i<$#tmpArray){
        $tmpArray[$i]=~/^([\w\d_]+)\s+([\w\d_]+)\s+([01]{3})\s*([0-7]{0,1})\s*$/;
        my $shift=$4;
        if ($shift eq "") {$shift=0;}
        my $byteEnable=$3;
        ($block_name,$line_name)=($1,$2);
        
        # A variable for converting byte enable to decimal number
        my $byteEnable_dec=0;
        # Setting the entire line in the relevant byte, alignment will be done later
        for (my $j=0;$j<3;$j++) {
            if (substr($byteEnable,2-$j,1)) {
                setDataFormat(($j-$shift+8)%8,$block_name,$line_name,0);
                $byteEnable_dec+=1<<$j;
            }
            
        }
        $byteEnable=$byteEnable_dec;
        
        # aligning each slot to hold only the fields it possess
        # Based on the the line structure options available in each byte enable
        # If a byte is standalone (not in the same group as the other byte or in the same group but with a gap)
        # then it will have his fields as group 0
        # Otherwise, each byte in the group will have his position in the group and the number of bytes in the group
        # When the data is parsed, only when all parts of the group have arrived it will be parsed 
        if ($byteEnable==7) {
            if ($DATA_FORMAT[(8-$shift)%8]{'groups'}[0]{'Size'}==24) {
                $DATA_FORMAT[(8-$shift)%8]{'Pos'}=0;
                $DATA_FORMAT[(9-$shift)%8]{'Pos'}=1;
                $DATA_FORMAT[(10-$shift)%8]{'Pos'}=2;
                $DATA_FORMAT[(8-$shift)%8]{'ByteSize'}=3;
                $DATA_FORMAT[(9-$shift)%8]{'ByteSize'}=3;
                $DATA_FORMAT[(10-$shift)%8]{'ByteSize'}=3;
            } elsif ($DATA_FORMAT[(8-$shift)%8]{'groups'}[0]{'Size'}==16) {
                align($DATA_FORMAT[(10-$shift)%8]{'groups'},16,8);
                $byteEnable=3;
            } elsif ($DATA_FORMAT[(8-$shift)%8]{'groups'}[1]{'Size'}==16) {
                $byteEnable=6;
            } else {
                align($DATA_FORMAT[(9-$shift)%8]{'groups'},8,8);
                $byteEnable=5;
            }
        }
        if ($byteEnable==3) {
            if ($DATA_FORMAT[(8-$shift)%8]{'groups'}[0]{'Size'}==8) {
                $byteEnable=2;       
            } else {
                $DATA_FORMAT[(8-$shift)%8]{'Pos'}=0;
                $DATA_FORMAT[(9-$shift)%8]{'Pos'}=1;

                $DATA_FORMAT[(8-$shift)%8]{'ByteSize'}=2;
                $DATA_FORMAT[(9-$shift)%8]{'ByteSize'}=2;

                align($DATA_FORMAT[(9-$shift)%8]{'groups'},0,16);
            }
        }
        if ($byteEnable==6) {
            if ($DATA_FORMAT[(9-$shift)%8]{'groups'}[0]{'Size'}!=24&&$DATA_FORMAT[(9-$shift)%8]{'groups'}[1]{'Size'}!=16) {
                align($DATA_FORMAT[(9-$shift)%8]{'groups'},8,8);
                align($DATA_FORMAT[(10-$shift)%8]{'groups'},16,8);
            } else {
                $DATA_FORMAT[(9-$shift)%8]{'Pos'}=0;
                $DATA_FORMAT[(10-$shift)%8]{'Pos'}=1;
                
                $DATA_FORMAT[(9-$shift)%8]{'ByteSize'}=2;
                $DATA_FORMAT[(10-$shift)%8]{'ByteSize'}=2;
                
                align($DATA_FORMAT[(10-$shift)%8]{'groups'},8,16);
            }
        }
        if ($byteEnable==1||$byteEnable==5) {
            align($DATA_FORMAT[(8-$shift)%8]{'groups'},0,8);
        }
        if ($byteEnable==2) {
            align($DATA_FORMAT[(9-$shift)%8]{'groups'},8,8);
        }
        if ($byteEnable==4||$byteEnable==5) {
            align($DATA_FORMAT[(10-$shift)%8]{'groups'},16,8);
        }
        
        # Moving to the next line
        $i++;
    }
}

# 24 bit uses base 3 for compress of the client number of each data chunk
# this function recieve a number in binary and translate it to a 6 slots array
# where each slot represnt a digit in base 3
sub BinaryToBase3 {
    my ($base3)=@_;
    my @clientNum;
    
    $base3=unpack("n",pack("B16", "0"x6 .$base3));
    for (my $j=0;$j<6;$j++){
        @clientNum=(@clientNum,$base3%3);
        $base3=($base3-$base3%3)/3;
    }
    return @clientNum;
    
}

# This function is used for 24 bit mode for adjusting and aligning The line data
# to the relevant block
# It recieved a reference to the groups of that block, an offset in bits from the beginning of the groups
# and the size of relevant data (So each byte in a group will hold the entire group data)
sub align {
    my ($groups,$offset,$size)=@_;
    my $pass=0;
    my $curr=2;
    # getting to the current offset
    while ($pass+$$groups[0]{"Size"}<=$offset) {
        $curr++;
        $pass+=$$groups[0]{"Size"};
        shift(@$groups);
    }
    
    my $arrRef=$$groups[0]{"Data"};
    # If the offset cut a group in the middle, The field is splitted into 2 fields
    if ($pass!=$offset) {
        while ($pass+$$groups[0]{"Data"}[2]{"Size"}<=$offset){
            $pass+=$$groups[0]{"Data"}[2]{"Size"};
            shift(@$arrRef);
        }
        # If the offset cut a field in the middle, The field is adjust only to the relevant part into 2 fields
        # e.g. field[2,0] might be cut into field[2,1]
        if ($pass!=$offset) {
            $$arrRef[2]{'Name'}=~/,(\d+)]/;
            my $diff=$1+$offset-$pass;
            $$arrRef[2]{'Name'}=~s/,(\d+)]/$diff/;
            $$arrRef[2]{'Size'}+=$offset-$pass;
        }
        
    }
    $pass=0;
    # Saving only the relevant size
    while ($curr<=$#$arrRef && $pass+$$arrRef[$curr]{'Size'}<=$size) {
        $pass+=$$arrRef[$curr]{'Size'};
        $curr++;
    }
    # Adjusting the field (the samething as in offset)
    if ($pass!=$size) {
        $$arrRef[$curr]{'Name'}=~/\[(\d+),/;
        my $diff=$1+$size-$pass;
        $$arrRef[2]{'Name'}=~s/\[(\d+),/$diff/;
        $$arrRef[2]{'Size'}+=$size-$pass;    
    }
    # Deleting non-relevant groups
    if ($curr-1!=$#$arrRef) {
        splice (@$arrRef,$curr);
    }
    
}

###### set profile in TDM32 mode ############################
# receive an array of the profile data
# set Data Format to include the relevant blocks' information
# sorted according to their position in the calendar
#############################################################

sub setTDM32 {
    my (@tmpArray)=@_;
    my (%tmpHASH);
    my ($i)=1;
    my $storm_mode=0;
    # getting the blocks, lines, and shift if exist and saving them in tmpHASH
    while($tmpArray[$i]!~/ENDBLOCKS/&&$i<$#tmpArray){
        $tmpArray[$i]=~/([\w\d_]+)\s+([\w\d_]+)(\s+(\d*))?/;
        my $shift;
        if (defined($3)) {
            $shift=$3;
            chomp($shift);
        } else {
            $shift=0;
        }
        if ($shift eq "") {$shift=0;}
        $tmpHASH{$1}={"line" => $2, "shift" => $shift};
        $i++;
    }
    
    # ensuring all needed data exist, since there must be 8 slots + 4 syntax mandtory line (CALENDAR:, ENDCALENDAR and ENDPROFILE)
    # NOTE: TDM32 must have excatly 8 slots while TDM64 must have excatly 7 slots
    die "Problem with calendar configuration in the profile. Recheck Profi1le.\n"
    unless ($i+11==$#tmpArray);
    # skipping ENDBLOCKS and CALENDAR
    $i+=2;
    
    # Setting each slot with the relevant block's information
    # Skipping EMPTY slots
    for (my ($j) =0; $j<8;$j++) {
        if ($tmpArray[$i+$j]=~/SLOT\s+([\w\d_]+)\s*/) {
            my $block_name=$1;
            if ($block_name eq "EMPTY") {
                next;
            }
            setDataFormat($j,$block_name,$tmpHASH{$block_name}{'line'},$tmpHASH{$block_name}{'shift'});
            $storm_mode|=system("perl -w storm.pl -c $tmpHASH{$block_name}{'line'}");
        } else { die "The calnedar is not defined correctly. Recheck Profile.\n"; }
    }
    return $storm_mode;
}

###### set profile in TDM64 mode ###################
# receive an array of the profile data
# set Data Format to include the relevant blocks' information
# sorted according to their position in the calendar
####################################################
sub setTDM64 {
    my (@tmpArray)=@_;
    my (%tmpHASH);
    my ($i)=1;
    my $storm_mode;
    
    # represent whether the current block is using the upper 32 bit
    my $UpBlock=0;
    
    # getting the blocks, lines, and shift if exist
    while($tmpArray[$i]!~/ENDBLOCKS/&&$i<$#tmpArray){
        $UpBlock=0;
        # Checking if the current block is using the upper 32 bit
        # There are 2 bareword for TDM64 which are 64: and UPPER:
        # 64: is when the entire line is going to be sampled (UpBlock =1)
        # While UPPER: is used when only the 32 upper bits would be sampled (UpBlock =2)
        if ($tmpArray[$i]=~s/^(64:|UPPER:)//){
            if ($1 eq "UPPER:") {
                $UpBlock++;
            }
            $UpBlock++;
        }
        
        $tmpArray[$i]=~/([\w\d_]+)\s+([\w\d_]+)\s+?(\d*)/;
        my $shift=$3;
        if ($shift eq "") {$shift=0;}

        if ($UpBlock) {
            $tmpHASH{"UP"}={"line"=>$2,"shift"=> $shift,"block"=>$1};
        }
        # in 64: the lower 32 bit should be treated same as in TDM32
        if ($UpBlock<2) {
            $tmpHASH{$1}={"line" => $2, "shift" => $shift};
        }
        $i++;
        
    }
    
    # ensuring all needed data exist, since there must be 7 slots + 4 syntax mandtory line (CALENDAR:, ENDCALENDAR and ENDPROFILE)
    # NOTE: TDM32 must have excatly 8 slots while TDM64 must have excatly 7 slots
    die "Problem with calendar configuration in the profile. Recheck Profile.\n"
    unless ($i+10==$#tmpArray);
    $i+=2;
    
    # Setting each slot with the relevant block's information
    # Skipping EMPTY slots
    for (my ($j) =0; $j<7;$j++) {
        if ($tmpArray[$i+$j]=~/SLOT\s+([\w\d_]+)\s*/) {
            my $block_name=$1;
            if ($block_name eq "EMPTY") {
                next;
            }
            setDataFormat($j,$block_name,$tmpHASH{$block_name}{'line'},$tmpHASH{$block_name}{'shift'});
            $storm_mode|=system("perl -w storm.pl -c $tmpHASH{$block_name}{'line'}");
        } else { die "The calnedar is not defined correctly. Recheck Profile.\n"; }
    }
    # The upper 32 bit always arrived on slot 7
    # Settingslot 7 to the Upper part of the line
    setDataFormat(7,$tmpHASH{"UP"}{"block"},$tmpHASH{"UP"}{'line'},($tmpHASH{"UP"}{"shift"}+4)%8);
    return $storm_mode;
}

###### set data format ##################################################
# Recieve a slot number in the array to fill
# a block and a line name to fill according to
# And a shift if any
# This function is general so 24 bit mode needed also the align function
# Set Data Format to include the relevant block in the require structure.
# The function read the block's CSV file which has all the data about available outputs
# The DATA_FORMAT array structure is explained in the beginning of the script
#########################################################################

sub setDataFormat {
    my($slot,$block_name,$line_name,$shift)=@_;
    my ($blockFile);
    my $line_type=0;
    # getGroupFromFile has all the groups save by their names
    my(%groupHash)=getGroupFromFile($block_name);
    my ($line,$group);
    my (@groups);
    
    $DATA_FORMAT[$slot] = {"block_name"=>$block_name ,"line_name"=>$line_name};
    # In tdm mode each block needs its own file
    if ($MODE%3==0 && !$NOSEP) {
        if (!defined($FILES{$block_name})) {
            die "Can't open output file for $block_name" unless open($blockFile,">".$OUTPUT_FOLDER . '/' ."$block_name.txt");
            $FILES{$block_name}=$blockFile;
        }
    }
    
    # Finding relevant line in the csv
    # and using getGroupFromFile to set Data Format
    # Some of the CSV has special lines defined after the regular lines
    # Special lines are used by the tcl script and hold a reference to a regular line + pair of register name/address with
    # a desired value
    # If a profile is defined with a sp(special) line, this script should regard the line as if it was the regular line refernced
    # sp_lines is used to seperate between the regular lines parsing to the special lines
    my ($sp_lines)=0;
    my (%lines);
    my ($tmp_name)=$line_name;
    while (<CSV>) {
        $line=$_;
        # regard sp_lines
        # Syntax: <sp_line_name>,<regular line>,<reg name/address>,<value>.....
        # when finding the relevant sp_lines changing line to the regular line referenced will cause the rest of
        # loop to set the data normally
        if ($sp_lines&&$line=~/^$line_name,([\w_\d]+),?/){
            $line=$lines{$1};
            $line_name=$1;
        }
        # regard regular lines
        # Syntax: <regular line>,<line size 64/24>,<msb group>,<middle group>,<lsb group>
        # For supporting the special lines all the lines are being save in a hash
        if ($line=~/^([\w_\d]+),(\d+),([^,]+),([^,]+),([^,\n]+)/) {
            $lines{$1}=$line;
        }
        if ($line=~/^Sp_Lines/) {
            $sp_lines++;
        }
        # regard regular lines
        # Syntax: <regular line>,<line size 64/24>,<msb group>,<middle group>,<lsb group>
        # When finding the relevant line, sorting the groups in the right place.
        # line_type represent the structure of a line (0 - 64 bit line with 1 64 bit group)
        # (1 - 64 bit line with 1 32 bit group) (2 - 64 bit line with 2 32 bit group)
        # (3 - 24 bit line)
        if ($line=~/^$line_name,(\d+),([^,]+),([^,]+),([^,]+)/){
            my (@tmpArray,@tmpGroups);
            if ($shift==4 && ($3 ne "none")) {
                @tmpGroups=($4,$2,$3);
            } else { @tmpGroups=($4,$3,$2);}
            my $lSize=0;
            foreach $group (@tmpGroups) { 
                if (!($group eq "none")) {
                    @tmpArray=(@tmpArray,$groupHash{$group});
                    if ($groupHash{$group}{"Size"}==32) {$line_type++;}
                    $lSize+=$groupHash{$group}{"Size"};
                }
                
            }
            if ($1==24) {$line_type=3;}
            $DATA_FORMAT[$slot]{"groups"}=\@tmpArray;
            $DATA_FORMAT[$slot]{"Size"}=$lSize;
            last;
        }
        
    }
    # If reached up to here without defining the slot, There was some problem with the requested line
    die "Unknown line $line_name. Recheck profile and csv file."
        unless (defined($DATA_FORMAT[$slot]{"groups"}));
    
    if ($sp_lines) {
        $line_name=$tmp_name;
    }
    close CSV;
    # formatting the data accroding to the shift
    # and ensuring group size won't exceed the data size
    # relevant only to TDM modes
    # e.g. a 64 bit line will be splitted to two 32 bit line when working in TDM32
    # A 24 bit line will only have one group when working in TDM mode
    # (due to validity problem, more information in the user manual).
    
    if ($MODE%3==0) {
        if ($shift==4&&$line_type==2) {
            my $tmp=$DATA_FORMAT[$slot]{"groups"}[0];
            $DATA_FORMAT[$slot]{"groups"}[0]=$DATA_FORMAT[$slot]{"groups"}[1];
            $DATA_FORMAT[$slot]{"groups"}[1]=$tmp;
        }
        elsif ($line_type==3) {
            if ($shift==1) {
                my $tmp=$DATA_FORMAT[$slot]{"groups"}[0];
                $DATA_FORMAT[$slot]{"groups"}[0]=$DATA_FORMAT[$slot]{"groups"}[1];
                $DATA_FORMAT[$slot]{"groups"}[1]=$tmp;
            }
            elsif ($shift==2) {
                my $tmp=$DATA_FORMAT[$slot]{"groups"}[0];
                $DATA_FORMAT[$slot]{"groups"}[0]=$DATA_FORMAT[$slot]{"groups"}[2];
                $DATA_FORMAT[$slot]{"groups"}[2]=$tmp;
            }
            my $arrRef=$DATA_FORMAT[$slot]{"groups"};
            splice(@$arrRef,1);
            my $left=31-$$arrRef[0]{"Size"};
            $arrRef=$$arrRef[0]{"Data"};
            my %alignGr=("Name"=>"none0[".$left.",0]","Size"=>$left+1,"Comment"=>"");
            @$arrRef=(@$arrRef,\%alignGr);
        }
        elsif ($shift!=0) {
            if ($line_type!=0) {die "Illegel shifts of line "+$line_name+". Recheck profile.\n"  }
            
            my $i=2;
            my $size=0;
            my $relGroup=$DATA_FORMAT[$slot]{"groups"}[0]{"Data"};
            while ($size<$shift*8){
                $size+=$$relGroup[$i]{"Size"};
                $i++; 
            }
            
            if ($size>$shift*8) {
                # splitting the group that's exceed into two groups
                my $probLine=$$relGroup[$i-1];
                my $diff=($size-$shift*8);
                my $name=$$probLine{"Name"};
                $name=~/\[(\d+),(\d+)\]/;
                my $new=$1-($diff);                 
                $name=~s/\[(\d+),/\[$new,/;
                $$probLine{"Name"}=$name;
                $$probLine{"Size"}=$$probLine{"Size"}-($diff);
                $new=$new+1;
                my $end=$new+($diff)-1;
                $name=~s/\[(\d+),(\d+)\]/\[$end,$new\]/;
                my %addGr=("Name"=>$name,"Size"=>$diff,"Comment"=>$$probLine{"Comment"});
                splice(@$relGroup,$i,0,\%addGr);
            }
            
            if ($size>=$shift*8) {
                my @tmpArr=splice(@$relGroup,2,$i-2);
                @$relGroup=(@$relGroup,@tmpArr);
            }
        }
        # Ensuring the last bits in the group won't exceed 32 bit
        if ($line_type==0) {
            my $i=2;
            my $size=0;
            my $relGroup=$DATA_FORMAT[$slot]{"groups"}[0]{"Data"};
            while ($size<32){
               $size+=$$relGroup[$i]{"Size"};
                $i++;
            }
            
            if ($size>32) {
                # splitting the group that's exceed into two groups
                my $probLine=$$relGroup[$i-1];
                my $diff=($size-32);
                my $name=$$probLine{"Name"};
                $name=~/\[(\d+),(\d+)\]/;
                my $new=$1-($diff);                 
                $name=~s/\[(\d+),/\[$new,/;
                $$probLine{"Name"}=$name;
                $$probLine{"Size"}=$$probLine{"Size"}-($diff);
                $new=$new+1;
                my $end=$new+($diff)-1;
                $name=~s/\[(\d+),(\d+)\]/\[$end,$new\]/;
                my %addGr=("Name"=>$name,"Size"=>$diff,"Comment"=>$$probLine{"Comment"});
                splice(@$relGroup,$i,0,\%addGr);
            }
        }    
    }
}

# TODO: CHECK bus configuration;

###### get groups from file ####################################
# receives a block name and return an HashMap of all his groups
# and their data. (The groups structure is explained in the beginning of the script)
################################################################
sub getGroupFromFile {
    my ($block_name)=@_;
    my ($i);
    my $line;
    my $isFirst=1;
    my ($cur_group);
    my (%groupHash);
    my ($bitName);
    my (@tmpArray,%tmpHash);
    my ($file_name)=$block_name;
    my $csv_folder="CSV$CHIP_NUM";
    my $bus;
    my $bus_size;
    if ($block_name=~/(SEM|STORM)$/) {
        # The SEMIS and the STORMS each has 1 file for all 4.
        # The other blocks have a file for each block
        $file_name=$1;
        die "You are using old version of the csv files with a new version of the script.\nPlease update the csv folder.\n"
            unless -e "$csv_folder/$file_name".".csv";
    }
    # opening the file
    die "Cannot read CSV  file: \\$csv_folder\\$block_name".".csv"
	unless open(CSV, "$csv_folder/$file_name".".csv");
    
    # reaching the groups area
    for ($i=0; $i<5; $i++)
    {	
	$line=<CSV>;
    }
        
    while (<CSV> ) {
        $line=$_;
        # Searching for the end of groups, Upon reaching saving the last group
        # and leaving the loop
        if ($line=~/#End_Of_Groups/){
            if (defined($bus) && $bus_size) {
                my $name=$$bus{'Name'};
                $name=~/([^\[]+)\[(\d+),/;
                @tmpArray=(@tmpArray,{'Name'=>$1.'['.$2.','.($2-$bus_size+1).']','Size'=>$bus_size,'Comment'=>$$bus{'Comment'}});
            }
            undef($bus);
            my (@arr)=@tmpArray;
            @tmpArray=();
            $groupHash{$cur_group}{'Data'}=\@arr;
            last;
        }
        # Searching for new groups, a group always begin with a line: #group_name,size-1,0
        # When there is a new group which is not the first one, saving the last group and starting to
        # collect the fields information about the new group
        if ($line=~/#(.+),(\d+),(\d+)/) { 
            if (defined($bus) && $bus_size) {
                my $name=$$bus{'Name'};
                $name=~/([^\[]+)\[(\d+),/;
                @tmpArray=(@tmpArray,{'Name'=>$1.'['.$2.','.($2-$bus_size+1).']','Size'=>$bus_size,'Comment'=>$$bus{'Comment'}});
            }
            undef($bus);
            if (!$isFirst) {
                my (@arr)=@tmpArray;
                @tmpArray=();
                $groupHash{$cur_group}{'Data'}=\@arr;
            } else { $isFirst=0};
            $cur_group=$1;
            $groupHash{$cur_group}{"group_name"}=$cur_group;
            $groupHash{$cur_group}{'Size'} = $2-$3+1;
        }
        elsif ($line=~/,,([{\w\d]+),(\d+),(\d+)($|,(.*))/) {
        # fields' lines are build out of field(interface) name, msb bit inside that interface, lsb bit inside that interface, comment optional
        # If the current line is a field, that its data is added to its' group
              if (substr($1,0,1) eq '{') {
                unless (defined($bus)) {
                    $bus=pop(@tmpArray); 
                    $bus_size=$$bus{'Size'};
                }
                $bus_size-=($2-$3+1);
              } elsif (defined($bus)) {
                if (!$bus_size) {
                    my $name=$$bus{'Name'};
                    $name=~/([^\[]+)\[(\d+),/;
                @tmpArray=(@tmpArray,{'Name'=>$1.'['.$2.','.($2-$bus_size+1).']','Size'=>$bus_size,'Comment'=>$$bus{'Comment'}});
                }
                undef($bus);
              }
            
              $bitName="$1\[$2,$3\]";
              @tmpArray=(@tmpArray,{'Name'=>$bitName,'Size'=>$2-$3+1,'Comment'=>$5});
              if (!defined($5)) {
                $tmpArray[$#tmpArray]{'Comment'}="";
              }
        }
    }
    return %groupHash;   
}

# subroutine which open the input file (a buffer/Pci file or a network file)
# If the input is a pcap file then opening the file into NETDATA using the tcpdumplog module
sub getInput {
    my @data;
    if ($PARSE_MODE==BUFFER||$PARSE_MODE==PCI){
        die "Can't open data file $INPUT_FILES[0]" unless (open (INPUT,$INPUT_FILES[0]));
    } elsif ($PARSE_MODE==NETWORK) {
        $NETDATA = Net::TcpDumpLog->new();
        $NETDATA->read($INPUT_FILES[0]);
    }
}

# subroutine which return a packet (without the header) as an array, which every cell has 128 bits in bianry
# so it will fit the file recieved from the buffer
sub getPacket {
    
    my @packet;
    my $data;
    
    # Opening the next file in the input files array when the packetcounter reached the last packet in the current file
    if ($PACKETCOUNTER==$NETDATA->{"count"}) {
            shift(@INPUT_FILES);
            $NETDATA->clear($NETDATA);
            $NETDATA->read($INPUT_FILES[0]);
            $PACKETCOUNTER=0;
    }
    
    # getting the next packet if -filter was used (HEADERNUM must be different from 0) will return only valid debug packets
    # the filtering is done according to the ethernet header (the first 14 bytes in the pcaket)
    do{
        $data=$NETDATA->data($PACKETCOUNTER);
        # also, incread current packet number
    } while ($HEADERNUM!=0 && (substr($data,0,14) ne $HEADERBIN) && (++$PACKETCOUNTER < $NETDATA->{"count"}) && (++$CURRENT_PACKET_NUM >= 0));
    
    # if reached the last packet, and it wasn't a debug packet and it was the last file will return an empty array
    # otherwise will recall this function so it will open the next file
    if ($PACKETCOUNTER==$NETDATA->{"count"}) {
        if ($#INPUT_FILES==0) {
            return @packet;
        } else {
            return getPacket();
        }    
    }
    if ($CURRENT_PACKET_NUM < $START_PACKET_NUM) {
        $CURRENT_PACKET_NUM++;
        $PACKETCOUNTER++;
        return getPacket();
    }
    # if filter wasn't used, The script will exit upon packets with a size different then 1038 (the usual debug packet) or (1042 - the usual debug packet + 4 crc bytes
    # the smartbits tend to save). with execption given to the last packet which might me smaller (created by a flush event)
    die "The pcap file($INPUT_FILES[0]) contains non-debug packets (packet $PACKETCOUNTER), please filter it and try again or use -filter h<num>\n"
        unless ($HEADERNUM!=0 || length $data==1038 || length $data == 1042 || $PACKETCOUNTER==$NETDATA->{"count"}-1);
    
    my $size=length $data;
    # saving each 16 bytes as a line in binary (128 chars)
    # ignoring the last 4 bytes of crc if exists
    for (my $i=14; $i<$size -(($size-14)%16);$i+=16){
        my $line=substr($data,$i,16);
        @packet=(@packet,unpack("B128",$line));
    }
    
    return @packet;
}

# a subroutine relevant for all modes
# If the output target was network will return the next packet
# if the output target was the internal buffer will return the entire buffer
# and if the output target was the pci will return 128 lines at a time
# The input files from the buffer and from the ram are built as follows:
# <line_number> <data>
# ....
# ....
sub getNextLines {
    
    if ($PARSE_MODE==NETWORK) {
        return getPacket();
    }
    my @data;
    if ($PARSE_MODE==BUFFER) {
        @data=<INPUT>;
    } else {
        for (my $i=0; $i<128 && !eof INPUT; $i++) {
            my $line=<INPUT>;
            @data=(@data,$line);
        }
    }
    for (my $i=0;$i<=$#data;$i++){
        my @line=split(' ', $data[$i]);
        $data[$i]=$line[1];
    }
    return @data;
}

###### parsing the input file in TDM32 mode #######
# Uses getNextLines to recieve lines to process.
# read it according to TDM32 trailer and
# uses printOutPut to puts the data into the file
# Parsing is done only for NORMAL format
########################################
# The TDM trailer
# When working in TDM mode the debug bus output a trailer line after every 7 lines of data.
# Each line data is build out of 4 DWORD(32 bit)
# The DWORDs are sorted from the lower number to the higher numeber
# For example. the first line is:
# [127:96] - DWORD 0 , [95:64] - DWORD 1 , [63:32] - DWORD 2 , [31:0] - DWORD 3
# The trailer holds the information about from which slot each dword arrived and a frame (an extra bit) about the DWORD
# the trailer (128 bit) structure is [127:118] - none0, [117:6] - information about the slot number and frame of
# each DWORD (3 bit of slot number and 1 bit of frame for up to 28 words, I.E. [117:115] - slot number DWORD 0,
# [114] - frame for DWORD 1, [113:111] - slot number for DWORD 1 and so on...), [5:1]- the number of valid DWORDS - usually
# 28 but in flush event can be lower and [0] - error bit - will be one if data was lost.
sub parseTDM {
    
    my @lines;
    my $wasErrorSomewhere = 0;
    do{
        @lines=getNextLines();
         
        

        # every 8th line is the trailer which associate the data chunks to relevant slot
        # The loop run on each trailer and tranfering the chunk and the slot number to
        # printOutput
         for (my $i=0;$i<=$#lines;$i+=8){
            my $trailer=$lines[$i+7];
            my $validWords= ord(pack('B8',"000". substr($trailer, 122, 5)));
            
            my $error=ord(pack('B8',"000". substr($trailer, 127, 1)));
            
            if ($error) {
                $wasErrorSomewhere = 1;
            }
            # Seperating each DWORD together with its frame and slot number and passing it to print output
            for (my $word=0;$word<$validWords;$word++) {
                my ($client_number)= ord(pack('B8', "00000". substr($trailer, 10+4*$word,3)));
                my ($frame)= ord(pack('B8', '0000000' . substr($trailer, 10+4*$word+3,1))); 
                my ($data)=substr($lines[int($word/4)+$i],($word%4)*32,32);
                if ($FORMAT==$FORMATS{'STORM'} && $CHIP_NUM ne 'E1') {
                    printRH($data,$client_number,$frame,$error);
                } else {
                    printOutput($data,$frame,$client_number,$error);
                }
                $error=0;
            }
            
        }
        $PACKETCOUNTER++;
        
    } while (($PARSE_MODE==PCI && !eof INPUT) || ($PARSE_MODE==NETWORK && ($PACKETCOUNTER<$NETDATA->{"count"} || $#INPUT_FILES>0)));
	if ($wasErrorSomewhere) {
		print "An error appeared in one of the chunks, so not all data was dumped.\n";
		}
}

###### parsing the input file in 24BIT mode #######
# Uses getNextLines to recieve lines to process.
# read it according to 24BIT trailer and
# uses printOutPut to puts the data into the file
# Parsing is done only for NORMAL format
########################################
# The 24BIT trailer
# When working in 24BIT mode the debug bus output line (128 bit) which is built of 12 BYTES of data and
# 4 BYTES of trailer
# The BYTESs are sorted from the lower number to the higher numeber
# For example. The first four bytes (from the msb to the lsb:
# [127:120] - DATA 0 , [119:112] - DATA 1 , [111:104] - DATA 2 and so on
# The trailer holds the information about from which byte (2, 1, 0) each chunk arrived and a frame (an extra bit)
# about the chunk.
# For saving resources the 6 client number encoding to 10 bits is as follows:
# Each byte's client number will be represented by a number between 0-2. This representation will be marked by
# b0, b1, b2, b3, b4, b5 (b0 marks the first valid byte from the input data0, b1 for data1, b5 for data5 where data0 is the higher byte data). 
# Calculate: ClntNum = 3^5 * b5 + 3^4 * b4 + 3^3 * b3 + 3^2 * b2 + 3 * b1 + b0
# NOTE: the values of ClntNum are in the range 0 - 728
# so the last 4 BYTES are built as follows:
# [31:22] - ClntNum_0_5, [21:12] - ClntNum_6_11, [11:6] - Frames_0_5, [5:0] - Frames_6_11
# In case of an error, instead of  ClntNum_6_11 ([21:12] there will be all ones (higher than 728). and there will be no data in that line.
# In case of a flush event the last line won't have data in it but in [21:12] the first 4 bits will be ones and the last 6 bits
# will represent the number of valid word in the preceeding line
# Since in 24Bit mode a group might be shiftted, each byte which is part of group will be saved in partWord until the other
# parts of the group arrived
sub parse24BIT {
    
    my @lines;
    
    my @partWord=("","","");
    my $partCount=0;
    my $lastPacket=0; # represent flush events
    
    
     
    my $file_size=stat ("INPUT");
    do{
        @lines=getNextLines();
        
        # Since the line preceeding the last line has dependency on the last line
        # The script ensures the last two lines will be parsed together
        if (($PARSE_MODE==NETWORK && $PACKETCOUNTER==$NETDATA->{"count"}-2 && length $NETDATA->data($PACKETCOUNTER+1)==30) ||
            ($PARSE_MODE==PCI && (tell INPUT) + 138==$$file_size[7])) {
                @lines=(@lines,getNextLines($PACKETCOUNTER++));
        }
        # Checking if the flush event occured
        if (($#lines>-1) && (($PARSE_MODE==PCI && eof INPUT) || ($PARSE_MODE==NETWORK && $PACKETCOUNTER==$NETDATA->{"count"}-1)
            || $PARSE_MODE==BUFFER)) {
                my $flushCheck=unpack("n",pack("B16","0"x6 .substr($lines[$#lines],106,10)));
                if ($flushCheck>728 && $flushCheck<973) {
                    $lastPacket=1;
                }
        }
        
        # every line has a 4 bytes trailer which associate the data chunks to relevant byte
        # The loop run on each trailer and transfering the chunk and the slot number to
        # printOutput
        for (my $i=0;$i<=$#lines-$lastPacket;$i++){
            my $error=0;
            if (unpack("n",pack("B16","0"x6 .substr($lines[$i],106,10)))>992) {
                $error=1;
            }
            # getting the client number for the bytes
            my @clientNum=BinaryToBase3(substr($lines[$i],96,10));
            @clientNum=(@clientNum,BinaryToBase3(substr($lines[$i],106,10)));
            
            
                
            my $validWords=12;
            # If needed calculating the number of valid words
            if ($lastPacket && $i==$#lines-$lastPacket) {
                $validWords=unpack("n",pack("B16","0"x12 .substr($lines[$#lines],112,4)));
            }
            
            # In case of an error notify the user the moving to the next line
            if ($error==1) {
                print "Data lost before $CHUNKCOUNTER\n";
                print OUT "# error occurred between the previous packet and this packet\n\n";
                printf ERROR "# Error occurred before packet $CHUNKCOUNTER\n\n";
                next;
            }
            # Seperating each DATA together with its frame and slot number and passing it to print output
            # if the DATA is part of a group then only when all members of the group arrived it will be passed
            # forward
            for (my $word=0;$word<$validWords;$word++) {
                my ($frame)= substr($lines[$i], 116+$word,1); 
                my ($data)=substr($lines[$i],$word*8,8);
                
                if (defined($DATA_FORMAT[$clientNum[$word]]{'Pos'})) {
                    die "Something is wrong with the debug data, chunks aren't sorted in the right order"
                    unless ($partWord[$DATA_FORMAT[$clientNum[$word]]{'Pos'}] eq "");
                
                    $partWord[$DATA_FORMAT[$clientNum[$word]]{'Pos'}]=$data;
                    $partCount++;
                    if ($DATA_FORMAT[$clientNum[$word]]{'ByteSize'}!=$partCount) {
                        next;
                    } else {
                        $data=$partWord[2].$partWord[1].$partWord[0];
                        @partWord=("","","");
                        $partCount=0;
                    }
                }
                printOutput($data,$frame,$clientNum[$word],$error);
            }
            
        }
        $PACKETCOUNTER++;
        
    } while (($PARSE_MODE==PCI && !eof INPUT) || ($PARSE_MODE==NETWORK && ($PACKETCOUNTER<$NETDATA->{"count"} || $#INPUT_FILES>0)));
}

###### parsing the input file in 64BIT mode #######
# recieve format
# Uses getNextLines to recieve lines to process.
# read it according to 64BIT trailer and
# uses printOutPut|printSEM|printFMT|printSTORE to write the data into the file
# depending on the profile format
########################################
# The 64BIT trailer
# When working in 64BIT mode the debug bus output a trailer line after every 15 lines of data.
# Each line of data is build out of 2 QWORD(64 bit)
# The QWORDs are sorted from the lower number to the higher numeber
# For example. the first line is:
# [127:64] - QWORD 0 , [63:0] - QWORD 1 
# The trailer holds the information about the frame (an extra bit) about each QWORD, the number of valid words and errors
# the trailer ( the lower 64 bit of every 16th line, the upper 64 bit is the last QWORD of that segement) structure is
# [63:37] - none0, [36:6] - information about the frame of each QWORD (In descending order [36] - frame 0, [35] - frame 35 and so on...)
# [5:1]- the number of valid QWORDS - usually
# 31 but in flush event can be lower and [0] - error bit - will be one if data was lost.

sub parse64BIT {
    my ($fmt)=@_;
    my @lines;
    my $new_msg=1;

    
    do{
        @lines=getNextLines();
        
        # every 16th line is the trailer which has the frame data for each data chunk
        # The loop run on each trailer and tranfering the chunk and the frame to the relevant function
        # depending on the format:
        # if format is normal, printOutput (the slot in 64BIT is always 0) 
        # if format is FMT, printFMT
        # if format is SEM, printSEM
        # if format is STORE, printSTORE
        # There is an option for dropping the first message since if might be partial and unparsable by succeeding tools 
        for (my $i=0;$i<=$#lines;$i+=16){
            my $trailer=substr($lines[$i+15],64,64);
            my $error=0;
            if (substr($trailer,63,1)==1) {
                print "Data lost before $FMTCOUNTER\n";
                $error=1;
            }
            my $validWords= ord(pack('B8',"000". substr($trailer, 58, 5)));
            for (my $word=0;$word<$validWords;$word++) {
                my ($frame)= ord(pack('B8', '0000000' . substr($trailer, 27+$word,1))); 
                my ($data)=substr($lines[$i+int($word/2)],($word%2)*64,64);
                
                # Dropping the first message 
                # relevant for FORMATs FMT and SEM
                # (will exit on other formats
                # in format FMT the frame represent the end of message, so the second message start in the word right
                # after the frame was 1. while in SEM mode the frame represent the beginning of a message
                # so the second message start with the first frame.
                if ($NOFIRST) {
                    die "There is no reason to drop the first message, excpet when the format is either FMT, STORE or SEM"
                        unless ( $fmt==$FORMATS{'SEM'} || $fmt==$FORMATS{'FMT'} || $fmt==$FORMATS{'STORE'});
                        
                    if ($frame) {
                        $NOFIRST=0;
                    }
                    
                    if ($NOFIRST || !($fmt==$FORMATS{'SEM'} && ($CHIP_NUM eq 'E1') )) {
                        next;
                    }
                }
                
                if ($fmt==$FORMATS{'FMT'} || ($fmt==$FORMATS{'SEM'} && $CHIP_NUM ne 'E1')) {
                    printFMT($data,$new_msg,0,$frame,$error);
                    $new_msg=$frame;
                } elsif ($fmt==$FORMATS{'STORE'}) {
                    printSTORE($data,0,$error);
                } elsif ($fmt==$FORMATS{'SEM'}) {
                    printSEM($data,$new_msg||$frame,0,$error);
                    $new_msg=0;
                } else { 
                    printOutput($data,$frame,0,$error);
                }
                $error=0;
            }
        }
        $PACKETCOUNTER++;
    } while (($PARSE_MODE==PCI && ![eof INPUT]) || ($PARSE_MODE==NETWORK && ($PACKETCOUNTER<$NETDATA->{"count"} || $#INPUT_FILES>0)));
}

# Subroutine for format STORE
# Useful for interfaces which need the merge script
# output each QWORD together with the store address given in the profile
# Will usually be used for the PBF and the UPB
sub printSTORE {
    
    my ($data,$client_number,$error)=@_;
    my $block_name=$DATA_FORMAT[$client_number]{"block_name"};
    my $line_name=$DATA_FORMAT[$client_number]{"line_name"};
    
    if ($error) {
        printf OUT "# Error occurred between the previous packet and this packet\n\n";
        printf ERROR "# Error occurred before packet $FMTCOUNTER\n\n";
    }
    die "This block ($block_name) doesn't fit STORE Format." unless (defined($STORE_STORM{$block_name}));
    printf OUT "DATA ".$STORE_STORM{$block_name}."STORM_WR_BUS # $FMTCOUNTER\n";
    $FMTCOUNTER++;
    
    printf OUT $STOREADDRESS."\t$data\t".("1"x8)."\n";
    printf OUT "\n";
    
}

# subroutine for printing in fmt mode
# print to a file for the formatMessage
# the user should create links between this script names to the relavant struct in the
# the fmt file
# The names will always be BLOCK_NAME:LINE_NAME
sub printFMT {
    my ($data,$first,$client_number,$last,$error)=@_;
    my $block_name=$DATA_FORMAT[$client_number]{"block_name"};
    my $line_name=$DATA_FORMAT[$client_number]{"line_name"};
    my $tmp=tell OUT;
    if ($error) {
        printf OUT "# Error occurred between the previous packet and this packet\n\n";
        printf ERROR "# Error occurred before packet $FMTCOUNTER\n\n";
    }
    if ($first) {
        # if limit is used, the script will create few files with limited size instead of one large file.
        # except in cases the limit can only by done by the outter splitter script 
        if (defined($LIMIT) && $WRITTEN>=$LIMIT && !$LIMIT_BY_SPLIT) {
            setOUT();
            $tmp=0;
            $WRITTEN=0;
        }
        printf OUT "DATA $block_name:$line_name # $FMTCOUNTER\n";
        $FMTCOUNTER++;
    }
    printf OUT "$data\n";
    if ($last) {
        printf OUT "\n";
    }
    $WRITTEN+=(tell OUT)-$tmp;
    
}

# subroutine for printing in rh or printf modes
# collect the data per client (by name) and print it upon recieving a frame.
# the user should create links between this script names to the relavant struct in the
# the fmt file
# The names will always be BLOCK_NAME:LINE_NAME
sub printRH {
    my ($data,$client_number,$last,$error)=@_;
    my $block_name=$DATA_FORMAT[$client_number]{"block_name"};
    my $line_name=$DATA_FORMAT[$client_number]{"line_name"};
	
    if ($NOFIRST) {
        if (!defined($NOFIRSTDM{$block_name})) {
            if ($last || ($block_name !~ "STORM")) {
                $NOFIRSTDM{$block_name}=1;
            }
            return $last;
        }
    }
	
    if ($error) {
        printf OUT "# Error occurred between the previous packet and this packet\n\n";
        printf ERROR "# Error occurred before packet $FMTCOUNTER\n\n";
    }
    if (!defined($BUFFER_PER_CLIENT{$block_name})) {
        # if limit is used, the script will create few files with limited size instead of one large file.
        # except in cases the limit can only by done by the outter splitter script 
        $BUFFER_PER_CLIENT{$block_name}="DATA $block_name:$line_name # $FMTCOUNTER pkt: $PACKETCOUNTER\n";
        $FMTCOUNTER++;
    }
    $BUFFER_PER_CLIENT{$block_name}.="$data\n";
    if ($last || ($block_name !~ "STORM")) {
        printf OUT $BUFFER_PER_CLIENT{$block_name}."\n";
        undef $BUFFER_PER_CLIENT{$block_name};
    }
    
    
}

# subroutine for printing in fmt mode
# print to a file for the formatMessage
# the user should create links between this script names to the relavant struct in the
# the fmt file
sub printSEM {
    my ($data,$frame,$client_number,$error)=@_;
    my $block_name=$DATA_FORMAT[$client_number]{"block_name"};
    my $line_name=$DATA_FORMAT[$client_number]{"line_name"};
    my $tmp=tell OUT;
    if ($error) {
        printf OUT "# Error occurred between the previous packet and this packet\n\n";
        printf ERROR "# Error occurred before packet $FMTCOUNTER\n\n";
    }
    
    if ($frame) {
        printf OUT "\n";
        # if limit is used, the script will create few files with limited size instead of one large file.
        # except in cases the limit can only by done by the outter splitter script 
        if (defined($LIMIT) && $WRITTEN>=$LIMIT && !$LIMIT_BY_SPLIT) {
            setOUT();
            $tmp=0;
            $WRITTEN=0;
        }
        printf OUT "DATA $block_name:$line_name # $FMTCOUNTER\n";
        $FMTCOUNTER++;
    }
    printf OUT "$data\n";
    $WRITTEN+=(tell OUT)-$tmp;
    
}
###### printing the output to the file #######
# receives an data chunk, frame and slot number (in the data format array).
# print the parsed data into the output file/s
# if working in tdm32, every block is outputed to a file and also one file for all the messages
# (NORMAL FORMAT)
########################################
sub printOutput {
    my ($data,$frame,$client_number,$error)=@_;
    
    my $block_name=$DATA_FORMAT[$client_number]{"block_name"};
    my $line_name=$DATA_FORMAT[$client_number]{"line_name"};
    my $groupsRef=$DATA_FORMAT[$client_number]{"groups"};
    my $blockFile;
    if (!$NOSEP) {
        $blockFile=$FILES{$block_name};
    }
    my $bits=0;
    my $dataSize=length $data;
    my $output="";
    if ($error==1) {
        $output="# error occurred between the previous packet and this packet\n\n";
        print "# Error occurred before packet $CHUNKCOUNTER\n";
        printf ERROR "# Error occurred before packet $CHUNKCOUNTER\n\n";
    }
    $output.= "$block_name:$line_name".'{'."\t# Chunk no. $CHUNKCOUNTER \n";
    $CHUNKCOUNTER++;
    my $groupNum=0;
    
    while ($bits<$dataSize){
        
        my ($groupRef)=$$groupsRef[$groupNum];
        $output.= "\t".$$groupRef{"group_name"}.'{'."\n";
        
        if ($$groupRef{"Data"}[0]{"Name"}!~"^none0")
            {$output.="\t\tFrame:".$$groupRef{"Data"}[0]{"Name"}."=$frame\n";}
        
        if ($$groupRef{"Data"}[1]{"Name"}!~"^none1")
            {$output.="\t\tValid:".$$groupRef{"Data"}[1]{"Name"}."=1\n";}
        
        my $tmpArray=$$groupRef{"Data"};
        for (my $i=2;$i<=$#$tmpArray&&$bits<$dataSize;$i++){
            my $size=$$groupRef{"Data"}[$i]{"Size"};
            if ($size>$dataSize-$bits) {$size=$dataSize-$bits};
            if ($$groupRef{"Data"}[$i]{"Name"}!~"^(none0|none1)") {
                my $res=unpack("H".ceil($size/4),pack("B".(ceil($size/4)*4),"0"x((4-$size%4)%4).substr($data, $dataSize-$bits-$size,$size)));
                my $comment=$$groupRef{"Data"}[$i]{"Comment"};
                if ($comment ne "") {$comment ="\t# $comment";}
                $output.="\t\t".$$groupRef{"Data"}[$i]{"Name"}."=0x".$res.$comment."\n";
            }
            $bits+=$size;
        }
        $output.="\t".'}'."\n";
        $output.='}'."\n\n";
        
        $groupNum++;
    }
    my $tmp=tell OUT;
    printf OUT $output;
    $WRITTEN+=(tell OUT)-$tmp;
    if ($MODE%3==0 && !$NOSEP) {
        printf $blockFile $output;
    }
    # if limit is used, the script will create few files with limited size instead of one large file.
    # except in cases the limit can only by done by the outter splitter script         
    if (defined($LIMIT) && $WRITTEN>=$LIMIT && !$LIMIT_BY_SPLIT) {
            setOUT();
            $WRITTEN=0;
    }
}

# 64 bits substitution - only when the result is 32 bits
sub sub64Bits {
	my ($a_lsb,$a_msb,$b_lsb,$b_msb) = @_;
	
        # is msb are the same, only sub the lsb
	if ($a_msb == $b_msb) {
		return ($a_lsb - $b_lsb);
	}
	
        # is diff between msb's is 1
	if ($a_msb - $b_msb == 1) {
		if ($a_lsb >= $b_lsb) {
			die "Sub result is too big."
		}
                # calculate sub but add 2^32
		my $result = 4294967295 - $b_lsb;
		$result = $result + 1;
		$result = $result + $a_lsb;
		return $result;
	}
}
# read a 4 byte register from a file handle
sub readReg {
	my ($fh) = @_;
	my $tmp;
	read ($fh, $tmp, 4);
	return unpack("l",$tmp);
}
# write a 4 byte register to a byte handle
sub writeBinaryReg {
	my ($fh,$reg) = @_;
	my $tmp_reg = pack("l",$reg);
	print $fh $tmp_reg;
}


# convert an array of registers to binary text
sub binRegsToText {
	my (@in_args) = @_;
	my $result = "";
	my $tmp_result;
        # go over each register
	while (defined $in_args[0]) {
                # turn it to binary text
		$tmp_result = unpack("B32",pack("N",$in_args[0]));
		$result = "$tmp_result" . "$result";
		shift @in_args;
	}

	return $result;
}
#write binary data to text file
# format is:
# <line index in 8 Hex digits>\t<128 bits>
# is_big_endian - in this case, replace 64 msb with 64 lsb
sub writeBinaryDataAsText {
	my ($file_name,$data_holder,$is_big_endian) = @_;
        my $out_file_string = ">"."$file_name";
	open(FILE_HD,$out_file_string);
	my $i = 0;
        # go over the data
	while (defined $$data_holder[0]) {
		my $str_i = unpack("H8",pack("N",$i));
		my @curr_arr;
                # prepare next binary string
		for (my $j=0;$j<4;$j++) {
			if (defined $$data_holder[0]) {
				$curr_arr[$j] = $$data_holder[0];
			} else {
				$curr_arr[$j] = 0;
			}
			shift @$data_holder;
		}
                # replacing 64 msb with 64 lsb
		if ($is_big_endian) {
			reorderCyclicArray(\@curr_arr,2,1);
		}
		my $bin_data = binRegsToText(@curr_arr);
		my $curr_line = "$str_i\t"."$bin_data\n";
		$i++;
		print FILE_HD $curr_line;
	}
	close FILE_HD;
}
# according to Kernel Debug
sub getChipName {
	my ($chip_name_id) = @_;
	if ($chip_name_id == 0) {return "E1";}
	if ($chip_name_id == 1) {return "E1H";}
	if ($chip_name_id == 2) {return "E2";}
	die "unknown chip";
}
# This function opens the text meta data file.
# The first string in the file is the timestamp
# and the next lines are:
# E<version>,<profile number> profile_name
sub getProfileName {
	my ($meta_filename, $timestamp, $chip_string, $profile_num) = @_;
	my $metaf = "<"."$meta_filename";
	open(META_FILE,$metaf);
	my $line = <META_FILE>;
        # First line must be the timestamp
	if ($line =~ /^TIMESTAMP (.*)$/) {
		die "Supplied Meta Data ($timestamp) doesn't fit meta data in dump file ($1)" unless ($1 == $timestamp);
	} else {
		die "Meta data file does not start with TIMESTAMP";
	}
	# looking for the profile name
	my $search_string = "$chip_string".","."$profile_num";
	while (<META_FILE>) {
		my $curr_line = $_;
		die "Wrong syntax for Meta file" unless ($curr_line =~ /^(.*,.*)\s(.+)$/);
		if ($1 eq $search_string) {
			close META_FILE;
			return $2;
		}
	}
	
	close META_FILE;
	die "Cannot find appropriate profile.";
}
#assumes that the given array represents an array that may be cyclic
# given a new start, the array is rearranged in order
sub reorderCyclicArray {
	
	my ($array_ref, $new_start_index,$is_cyclic) = @_;
	my @tmp_arr = @$array_ref;
	my @result_array;
	@$array_ref = @result_array;
	my $j = 0;
	for (my $i=$new_start_index;$i<=$#tmp_arr;$i++)
	{
		$$array_ref[$j] = $tmp_arr[$i];
		$j++;
	}
	if ($is_cyclic) {
		for (my $i=0;$i<$new_start_index;$i++)
		{
			$$array_ref[$j] = $tmp_arr[$i];
		$j++;
		}
	}
}
# read a number of registers from a file
sub readBlock {
	my ($fd,$num_regs,$ref_arr) = @_;
	for (my $i = 0; $i < $num_regs; $i++) {
		$$ref_arr[$i] = readReg($fd);
	}
}

######## Parsing the Header File ###########
# This is for the case of using the kernel #
#Debug abilities. The feature outputs a    #
#header file with all the needed info      #
#for parsing.                              #
############################################
sub parseHeaderFile {
	my ($file_name,$ref_start,$ref_is_cyclic,$ref_nump,$ref_chipname,$ref_tstamp) = @_;
	my $fhd;
        #open the file        
	my $file_string = "<".$file_name;
	open($fhd,$file_string);
	binmode($fhd);
        #read the arrays
	my @general_info;
	my @fill_regs;
	my @end_read_regs;
	#my $header_length = readReg($fhd);
	
	readBlock($fhd,$general_info_size,\@general_info);
        if ($general_info[0] < 500)
        {
            # get length in reg num
            my $header_length = $general_info[0]/4 - 1;
            shift(@general_info);
            # add last field, to the end
            $general_info[$general_info_size - 1] = readReg($fhd);
        }
        
        
        
        
	readBlock($fhd,$fill_regs_size,\@fill_regs);
	readBlock($fhd,$end_read_regs_size,\@end_read_regs);

	my $is_block_on = $end_read_regs[0];
	if ($is_block_on == 0) {
		return 0;
	}
	$$ref_tstamp = $general_info[0];
	$$ref_chipname = getChipName($general_info[1]);
	$$ref_nump = $general_info[2];
	my $intr_buffer_read = $end_read_regs[1];
	my $ext_buffer_start_lsb = $fill_regs[0];
	my $ext_buffer_start_msb = $fill_regs[1];
	my $ext_buffer_ptr_lsb = $end_read_regs[3];
	my $ext_buffer_ptr_msb = $end_read_regs[4];
	my $ext_buffer_wrap = $end_read_regs[6];
	# $diff is the difference between the start of the buffer and the current ptr
	my $diff = sub64Bits($ext_buffer_ptr_lsb, $ext_buffer_ptr_msb, $ext_buffer_start_lsb, $ext_buffer_start_msb);
        # but we must also take into accound the last line of the internal buffer (lines are 16 bytes)
	my $remaining_bytes = ($intr_buffer_read % 16) * 16;
	my $new_start_bytes = $diff - $remaining_bytes;
	$$ref_start = $new_start_bytes / 4;
	$$ref_is_cyclic = $ext_buffer_wrap;
	close($fhd);
	return 1;
}



###### parsing the cmdline arguments ########
# read the command line arguments and set the
# working mode appropriately
# for all the allowed options see the user manual
# NOTE: -warn is an unpublished option!
# It help defining the amount of allowed warnings
# for the fmt before stopping
#############################################

sub checkArgs {
    my $rel_dir="";
    my @emptyArgs;
    while ($#ARGV>-1) {
        if ($ARGV[0]=~/^-/) {
            $ARGV[0]=lc($ARGV[0]);
            
            # Check for kernel dump
            if ($ARGV[0] eq "-kernel_dump") {
                shift @ARGV;
                my $bin_file = $ARGV[0];
                shift @ARGV;
                my $hdr_file = $ARGV[0];
                shift @ARGV;
                my $meta_file = $ARGV[0];
                shift @ARGV;
                my $is_big_endian = $ARGV[0];
                if ($is_big_endian eq "BE") {
                    $is_big_endian = 1;
                } else {
                    die "Wrong argument for endianity $is_big_endian" unless ($is_big_endian eq "LE");
                    $is_big_endian = 0;
                }
                shift @ARGV;
                $ARGV[0] = lc($ARGV[0]);
                die "Wrong arguments: expecting -o" unless ($ARGV[0] eq "-o");
                shift @ARGV;
                my $output_dir = $ARGV[0];
                if (-e $output_dir){
                        my @out_contents=<$output_dir/*>;
                        die "The output directory isn't empty. Please choose a different output directory.\n"
                                 unless ($#out_contents==-1);
                }
                rmtree ($output_dir, 0, 0);
                die "Cannot create new output folder: $output_dir"
                unless mkdir($output_dir);
                
                #Parsing and preparing the file for regular parsing
                my $is_cyclic = 1;
                my $new_start;
                my $num_profile;
                my $chip_name;
                my $timestamp;
                
                my $block_on = parseHeaderFile($hdr_file,\$new_start,\$is_cyclic,\$num_profile,\$chip_name,\$timestamp);
                die "Debug Block was OFF!" unless ($block_on);
                my $chosen_profile = getProfileName($meta_file,$timestamp,$chip_name,$num_profile);
                print "Reading Binary File . . .\n";
                my $bin_string = "<".$bin_file;
                my $fhd;
                open($fhd,$bin_string);
                binmode($fhd);
                my @bin_arr;
                my $i = 0;
                while (!eof($fhd)) {
                    $bin_arr[$i] = readReg($fhd);
                    $i++;
                }
                close $fhd;
                reorderCyclicArray(\@bin_arr,$new_start,$is_cyclic);
                print "Creating temporary file . . .\n";
                my $temp_string = "$output_dir"."\\tempDbg.pci";
                writeBinaryDataAsText($temp_string,\@bin_arr,$is_big_endian);
                
                
                @ARGV = @emptyArgs;
                print "Building new arguemnt list . . .\n";
                $ARGV[0] = "-f";
                $ARGV[1] = $temp_string;
                $ARGV[2] = "-o";
                $ARGV[3] = "$output_dir"."\\dbgData";
                $ARGV[4] = "-c";
                $ARGV[5] = $chip_name;
                $ARGV[6] = "-p";
                $ARGV[7] = $chosen_profile;
                $ARGV[8] = "-dropfirst";
            }
            if ($ARGV[0] eq "-f") {
                # getting the data file
                shift(@ARGV);
                while (($#ARGV+1)&& ($ARGV[0]!~/^-/)){
                    die "The data file extension must be .buf, .pcap or .pci" 
                        unless (lc($ARGV[0])=~/\.(buf|pcap|pci|enc)/);
                    @INPUT_FILES=(@INPUT_FILES,$rel_dir.lc($ARGV[0]));
                    shift(@ARGV);
                    die "You can enter multiple files only when working with pcap files.\n"
                        unless ($#INPUT_FILES==0 || $1 eq "pcap");    
                }
                $INPUT_FILES[0]=~/\.(buf|pcap|pci|enc)/;
                if ($1 eq "buf") {
                    $PARSE_MODE=BUFFER;
                } elsif ($1 eq "pcap" || $1 eq "enc") {
                        $PARSE_MODE=NETWORK;
                } else { $PARSE_MODE=PCI;}
                
                die "You must give at least 1 input file\n" unless ($#INPUT_FILES>-1);
                
            } elsif ($ARGV[0] eq "-o") {
                # gets the output folder
                shift(@ARGV);
                $OUTPUT_FOLDER=$rel_dir.$ARGV[0];
                shift(@ARGV);
                #create new output folders
                if (-e $OUTPUT_FOLDER){
                        my @output_contents=<$OUTPUT_FOLDER/*>;
                        die "The output directory isn't empty. Please choose a different output directory.\n"
                                 unless ($#output_contents==-1);
                }
                rmtree ($OUTPUT_FOLDER, 0, 0);
            } elsif ($ARGV[0] eq "-oo") {
                # gets the output folder
                shift(@ARGV);
                $OUTPUT_FOLDER=$rel_dir.$ARGV[0];
                shift(@ARGV);
                #create new output folders
                rmtree ($OUTPUT_FOLDER, 0, 0);
            } elsif ($ARGV[0] eq "-pf") {
                # gets the profiles' file
                shift(@ARGV);
                $PROFILE_FILE=$ARGV[0];
                shift(@ARGV);
            } elsif ($ARGV[0] eq "-p") {
                # gets the profile
                shift(@ARGV);
                $PROFILE=$ARGV[0];
                shift(@ARGV);
            } elsif ($ARGV[0] eq "-start_packet") {
                # gets the profile
                shift(@ARGV);
                $START_PACKET_NUM=$ARGV[0];
                shift(@ARGV);
            } elsif ($ARGV[0] eq "-c") {
                # gets the chip_ver
                shift(@ARGV);
                die "The chip version must be a known version such as $CHIPS. $ARGV[0] is not allowed.\n"
                unless ($ARGV[0]=~/^$CHIPS$/);
                $CHIP_NUM=$ARGV[0];
                shift(@ARGV);
            } elsif ($ARGV[0] eq "-meta") {
                # gets the chip_ver
                shift(@ARGV);
                die "The chip firmware version must be a known version such as $CHIPS. $ARGV[0] is not allowed.\n"
                unless ($ARGV[0]=~/^$CHIPS$/);
                $CHIP_META=$ARGV[0];
                shift(@ARGV);
            } elsif ($ARGV[0] eq "-dropfirst") {
                # Is the first message relevant
                shift(@ARGV);
                $NOFIRST=1;
            } elsif ($ARGV[0] eq "-filter") {
                # filter the pcap file 
                shift(@ARGV);
                if ($ARGV[0]=~/^h(\d+)$/) {
                    $HEADERNUM=$1;
                    shift(@ARGV);
                } else {
                    die "After -filter you must give the proper header to use <h<num>>.\nE.g. -filter h4.\n";
                }
            } elsif ($ARGV[0] eq "-smc") {
                shift(@ARGV);
                $SMC_PATH=$ARGV[0];           
                if ($SMC_PATH!~/\\$/) {
                    $SMC_PATH.="\\";
                }
                shift(@ARGV);
            } elsif ($ARGV[0] eq "-fmt") {
              shift(@ARGV);
              $FMT_FILE=$ARGV[0];
              shift(@ARGV);
            } elsif ($ARGV[0] eq "-pfmt") {
              shift(@ARGV);
              $PFMT_FILE=$ARGV[0];
              shift(@ARGV);
            } elsif ($ARGV[0] eq "-limit") {
                shift(@ARGV);
                if ($ARGV[0]=~/^(\d+)$/) {
                    $LIMIT=$1*1048576;
                } else {
                    die "After -limit you must give a numeric value in MB.\nE.g. -limit 2.\n";
                }
                shift(@ARGV);
            } elsif ($ARGV[0] eq "-warn") {
                shift(@ARGV);
                if ($ARGV[0]=~/^(-?\d+)$/) {
                    $WARNINGS=$1;
                } else {
                    die "After -warn you must give a numeric value for fmt warnings allowed.\nE.g. -warn 1.\n";
                }
                shift(@ARGV);
            } elsif ($ARGV[0] eq "-notee") {
                shift(@ARGV);
                # Not creating intermedate files
                $NOTEE=1;
            } elsif ($ARGV[0] eq "-nopipe") {
                shift(@ARGV);
                # Not creating intermedate files
                $PIPELINE="NOT";
            } elsif ($ARGV[0] eq "-nosep") {
                shift(@ARGV);
                # Not creating files for the blocks in TDM
                $NOSEP=1;
            } elsif ($ARGV[0] eq "-ver") {
                print "parseDbg.pl Version $VERSION\n";
                shift(@ARGV);
            } elsif ($ARGV[0] eq "-cfg") {
                shift(@ARGV);
                $rel_dir=$ARGV[0];
                $rel_dir=~s/([^\/\\]+)$//;
                open(CFG,$ARGV[0]) || die "Can't open the requested configuration file.\n";
                shift(@ARGV);
                my $line=<CFG>;
                chomp($line);
                @ARGV=(split(/ /,$line),@ARGV);
                close CFG;
            } else {
                die "Unknown option: $ARGV[0]\nSyntax: perl $PROGRAM_NAME (Mandatory) <-f debug data file (buffer.txt)> <-o output folder> <-p profile name>\n (Optional) <-pf profiles file> <-smc smc path without an ending \\> <-start_packet firstPacketNum> <-fmt memories file (fmt.gp)> <-dropfirst>  <-filter h<num>>  <limit <size in MB>>\n";
            }   
            
        } else {
            die "Syntax: perl $PROGRAM_NAME (Mandatory:) <-f debug data file (buffer.txt)> <-o output folder> <-p profile name>\n(Optional) <-pf profiles file> <-smc smc path without an ending \\> <-start_packet firstPacketNum> <-fmt memories file (fmt.gp)> <-dropfirst>  <-filter h<num>> <limit <size in MB>>\n";
        }
    }
    die "You must defined the input file, the output folder, the chip version and the relevant profile\n".
    "Syntax: perl $PROGRAM_NAME (Mandatory:) <-f debug data file (buffer.txt)> <-o output folder> <-p profile name> <-c chipnum - can be $CHIPS>\n(Optional) <-pf profiles file> <-smc smc path without an ending \\> <-start_packet firstPacketNum> <-fmt memories file (fmt.gp)> <-dropfirst>  <-filter h<num>> <limit <size in MB>>\n" unless (defined($INPUT_FILES[0]) && defined($OUTPUT_FOLDER) && defined($PROFILE) && defined($CHIP_NUM));
}

# opening the relevant file
# setting file and folder references
sub prepare {
    
    $INPUT_FILES[0]=~/([^\\\/]+)\.(pcap|buf|pci|enc)$/;
    $OUTFILE_TEMPLATE="parsed".$1;
	if ($2 eq "enc") {
		system("encToPcap.bat $INPUT_FILES[0]");
		$INPUT_FILES[0]=~s/\.enc$/\.pcap/;
	}
    #$OUTPUT_FILE[0]=$OUTPUT_FOLDER.'/'.$OUTFILE_TEMPLATE.".txt";
    $ERROR_FILE=$OUTPUT_FOLDER. '/' ."errors.txt";
    $FMT_TEMPLATE=$OUTPUT_FOLDER. '/' ."fmt".$1;
    $FMT_OUTPUT=$FMT_TEMPLATE.".txt";
    die "Cannot create new output folder: $OUTPUT_FOLDER"
	unless mkdir($OUTPUT_FOLDER);
    
    # opening output file
    #die "Can't open output file $OUTPUT_FILE[0]" unless open (OUT,">$OUTPUT_FILE[0]");
    die "Can't open error file $ERROR_FILE" unless open (ERROR,">$ERROR_FILE");
     # Setting default paramters
    if (!defined($PROFILE_FILE)){
        $PROFILE_FILE="profiles.dbg";
        print "Using default profiles' file: profiles.dbg\n";
    }
    if (!defined($CHIP_META)) {
        $CHIP_META=$CHIP_NUM;
    }
    if (!defined($SMC_PATH)) {
        $SMC_PATH="..\\..\\..\\..\\$CHIPS_P4_NAME{$CHIP_META}\\";
        print "Using default smc path: $SMC_PATH\n";
    }
    if (!defined($FMT_FILE)) {
        $FMT_FILE="everest.fmt.gp";
        print "Using default fmt files: $FMT_FILE for FMT format\n";
    }
    if (!defined($PFMT_FILE)) {
        $PFMT_FILE="printf.fmt.gp";
        print "Using default fmt file fo printf: printf.fmt.gp for PRINTF format\n";
    }
    $FMT_FILE=$SMC_PATH.$DATA_BASE.$FMT_FILE;
    $PFMT_FILE=$SMC_PATH.$DATA_BASE.$PFMT_FILE;
    
    if (!defined($WARNINGS)) {
        $WARNINGS=0;
        # print "Using default warnings allowed value in the fmt (0).\n";
    }

}

# closing all files
sub clean {
    
    if ($MODE%3==0 && !$NOSEP) {
        my $block;
        foreach $block (keys %FILES){
            close $FILES{$block};
        }
    }
    close OUT;
    close OUT;
    close ERROR;
}

# this sub open the output, if a file was open ($#OUTPUT_FILE >-1), it will be closed \
# before opening the next file.
# the pipe is build using setPipe
sub setOUT {
    my $index=$#OUTPUT_FILE+2;
    if ($index==1) {
        $index="";
    }
    @OUTPUT_FILE=(@OUTPUT_FILE,$OUTPUT_FOLDER.'/'.$OUTFILE_TEMPLATE.$index.'.txt');
    if ($index eq "") {
        $index=1;
    } else {
        close OUT;
    }
    if ($LIMIT && ($SEM_MODE || $FORMAT==$FORMATS{'STORM'} || $FORMAT==$FORMATS{'STORE'})) {
        $LIMIT_BY_SPLIT=1;
    }
    if ($PIPELINE eq "NOT") {
        die "Can't open output file $OUTPUT_FILE[$#OUTPUT_FILE]" unless open (OUT,">$OUTPUT_FILE[$#OUTPUT_FILE]");
    } else {
        setPipe($index,$SEM_MODE|$STORM_MODE);
        if ($LIMIT_BY_SPLIT) {
            die "Can't open output file $OUTPUT_FILE[0]" unless open (OUT,$PIPELINE."$CURRTEE");
        } else {
            die "Can't open output file $OUTPUT_FILE[0]" unless open (OUT,$PIPELINE."> $CURRTEE");
        }
    }
}
# this sub build the pipeline according to the format and to the errorlevel return by the sem script
# if relevant.
sub setPipe {
    my ($index,$merge)=@_;
    my $tempFormat=$FORMAT;
    if ($FORMAT==$FORMATS{'NORMAL'}) {
        $CURRTEE=$OUTPUT_FILE[$index-1];
    } else {
        $CURRTEE="\"".$OUTPUT_FILE[$index-1]."\"";
    }
    if ($index>1) {
        close (OUT);
    } else {
        $index="";
    }
    $PIPELINE="";
    if ($FORMAT==$FORMATS{'SEM'}) {
        die "Can't find sem.pl script.\n" unless (-e "sem.pl");
        if ($index eq "") {
            print "Adding sem.pl...\n";
        }
        if ($NOTEE) {
            $PIPELINE.="|perl -w sem.pl -cn E$CHIP_NUM -f \"<&STDIN\"";
        } else {
            $PIPELINE.=$TEE.$CURRTEE." |perl -w sem.pl -cn E$CHIP_NUM -f \"<&STDIN\"";
        }
        $CURRTEE="\"".$OUTPUT_FOLDER."\\afterSemScript".$index.".txt"."\"";
        if ($merge==0) {
            $FORMAT=$FORMATS{'FMT'};
        } else {
            $FORMAT=$FORMATS{'STORE'};
        }
    }
    if ($FORMAT==$FORMATS{'STORE'}) {
        die "Either wrong smc path or merge script doesn't exists in the version you specified.\n" unless (-e $MERGE);
        if ($index eq "") {
            print "Adding merge.pl...\n";
        }
        if ($NOTEE) {
            $PIPELINE.="|perl -w ".$MERGE." \"<&STDIN\" -w";
        } else {
            $PIPELINE.=$TEE.$CURRTEE."|perl -w ".$MERGE." \"<&STDIN\" -w";
        }
        $CURRTEE="\"".$OUTPUT_FOLDER."\\mergeFile".$index.".txt\""; 
        $FORMAT=$FORMATS{'FMT'};
    }
    
    if ($FORMAT==$FORMATS{'STORM'}) {
        if ($CHIP_NUM eq 'E1') {
            if ($index eq "") {
                print "Adding parse_dbus_printf.pl...\n";
            }
            if ($NOTEE) {
                $PIPELINE.="|perl -w ".$PARSE_PRINTF." \"<&STDIN\"";
            } else {
                $PIPELINE.=$TEE.$CURRTEE."|perl -w ".$PARSE_PRINTF." \"<&STDIN\"";
            }
            $CURRTEE="\"".$OUTPUT_FOLDER."\\printfFile".$index.".txt"."\"";
            die "Either wrong smc path or the request memories file doesn't exists in the version you specified.\n" unless (-e $FMT_FILE);
            print "Adding formatMsg.awk...\n";
            if ($NOTEE) {
                $PIPELINE.="|".$GAWK." -f ".$formatMsg." -v warnings=$WARNINGS -v optionalUnionSelection=1 \"".$FMT_FILE."\" \"".$PFMT_FILE."\" /dev/stdin";
            } else {
                $PIPELINE.=$TEE.$CURRTEE."|".$GAWK." -f ".$formatMsg." -v warnings=$WARNINGS -v optionalUnionSelection=1 \"".$FMT_FILE."\" \"".$PFMT_FILE."\" /dev/stdin";
            }
            if ($LIMIT_BY_SPLIT) {
                die "Can't find splitter.awk script...\n" unless (-e $SPLITTER);
                $FMT_TEMPLATE=~s/\\/\//g;
                $CURRTEE="| ".$GAWK." -f ".$SPLITTER." -v len=$LIMIT -v delimiter=^DATA -v filename=\"".$FMT_TEMPLATE.".txt\"";
            } else {
                $CURRTEE="\"".$FMT_TEMPLATE.$index.".txt\"";
            }
        } else {
            print "Adding storm.pl...\n";
            
            if ($NOTEE) {
                $PIPELINE.="|perl -w storm.pl \"<&STDIN\" output.rh";
            } else {
                $PIPELINE.=$TEE.$CURRTEE."|perl -w  storm.pl \"<&STDIN\" \"$OUTPUT_FOLDER\\output.rh\"";
            }
            $CURRTEE="\"".$OUTPUT_FOLDER."\\stormFile".$index.".txt\"";
            
            die "Either wrong smc path or the request memories file doesn't exists in the version you specified.\n" unless (-e $FMT_FILE);
            if ($merge) {
                print "Adding merge.pl...\n";
                if ($NOTEE) {
                    $PIPELINE.="|perl -w ".$MERGE." \"<&STDIN\" -w";
                } else {
                    $PIPELINE.=$TEE.$CURRTEE."|perl -w ".$MERGE." \"<&STDIN\" -w";
                }
                $CURRTEE="\"".$OUTPUT_FOLDER."\\mergeFile".$index.".txt\"";
            }
            print "Adding formatMsg.awk...\n";
            if ($NOTEE) {
                $PIPELINE.="|".$GAWK." -f ".$formatMsg." -v warnings=$WARNINGS -v optionalUnionSelection=1 \"".$FMT_FILE."\" \"".$PFMT_FILE."\" /dev/stdin";            
            } else {
                $PIPELINE.=$TEE.$CURRTEE."|".$GAWK." -f ".$formatMsg." -v warnings=$WARNINGS -v optionalUnionSelection=1 \"".$FMT_FILE."\" \"".$PFMT_FILE."\" /dev/stdin";        
            }
            if ($LIMIT_BY_SPLIT) {
                die "Can't find splitter.awk script...\n" unless (-e $SPLITTER);
                $FMT_TEMPLATE=~s/\\/\//g;
                $CURRTEE="| ".$GAWK." -f ".$SPLITTER." -v len=$LIMIT -v delimiter=^DATA -v filename=\"".$FMT_TEMPLATE.".txt\"";
            } else {
                $CURRTEE="\"".$FMT_TEMPLATE.$index.".txt\"";
            }
        }
    }
    if ($FORMAT==$FORMATS{'FMT'}){
        die "Either wrong smc path or the request memories file doesn't exists in the version you specified.\n" unless (-e $FMT_FILE);        
        if ($index eq "") {
            print "Adding formatMsg.awk...\n";
        }
        if ($NOTEE) {
            $PIPELINE.="|".$GAWK." -f ".$formatMsg." -v warnings=$WARNINGS \"".$FMT_FILE."\" /dev/stdin";        
        } else {
            $PIPELINE.=$TEE.$CURRTEE."|".$GAWK." -f ".$formatMsg." -v warnings=$WARNINGS \"".$FMT_FILE."\" /dev/stdin";
        }
        if ($LIMIT_BY_SPLIT) {
            die "Can't find splitter.awk script...\n" unless (-e $SPLITTER);
            $FMT_TEMPLATE=~s/\\/\//g;
            $CURRTEE="| ".$GAWK." -f ".$SPLITTER." -v len=$LIMIT -v delimiter=^DATA -v filename=\"".$FMT_TEMPLATE.".txt\"";
        } else {
            $CURRTEE="\"".$FMT_TEMPLATE.$index.".txt\"";
        }
    }
    
    $FORMAT=$tempFormat;
    
    

}

# passing the relevant file to succeeding scripts/tools depending on the profile FORMAT
# used only when working without a pipeline
sub parseFiles {
    my $filemode="";
    if ($FORMAT==$FORMATS{'SEM'}) {
        die "Can't find sem.pl script.\n" unless (-e "sem.pl");
        print "Activating sem.pl...\n";
        
        my $suffix="";
        
        for(my $i=0; $i<=$#OUTPUT_FILE; $i++) {
            system("perl -w sem.pl -cn E$CHIP_NUM -f \"$OUTPUT_FILE[$i]\" > \"$OUTPUT_FOLDER/afterSemScript$suffix.txt\"");
            $OUTPUT_FILE[$i]=$OUTPUT_FOLDER.'/'."afterSemScript$suffix.txt";
            $suffix=$i+1;
        }
        
        if ($SEM_MODE==0) {
            $FORMAT=$FORMATS{'FMT'};
        } else {
            $FORMAT=$FORMATS{'STORE'};
        }
    }
    if ($FORMAT==$FORMATS{'STORE'}) {
        die "Either wrong smc path or merge script doesn't exists in the version you specified.\n" unless (-e $MERGE);
        print "Activating merge.pl...\n";
        system("perl -w ".$MERGE." \"$OUTPUT_FILE[0]\" -w > \"$OUTPUT_FOLDER/mergeFile.txt\"");
        $OUTPUT_FILE[0]=$OUTPUT_FOLDER."/mergeFile.txt";
        $FORMAT=$FORMATS{'FMT'};
    }
    if ($LIMIT_BY_SPLIT) {
            $FMT_TEMPLATE=~s/\\/\//g;
            $FMT_OUTPUT="| ".$GAWK." -f ".$SPLITTER." -v len=$LIMIT delimiter=^DATA filename=\"".$FMT_TEMPLATE.".txt\"";   
    } else {
            $filemode=" > ";
    }
    if ($FORMAT==$FORMATS{'STORM'}) {
        if ($CHIP_NUM eq 'E1') {
            print "Activating parse_dbus_printf.pl...\n";
            system("perl -w ".$PARSE_PRINTF." \"$OUTPUT_FILE[0]\" > \"$OUTPUT_FOLDER/printfFile.txt\"");
            die "Either wrong smc path or the request memories file doesn't exists in the version you specified.\n" unless (-e $FMT_FILE);
            print "Activating formatMsg.awk...\n";
 
            system($GAWK." -f ".$formatMsg." -v warnings=$WARNINGS optionalUnionSelection=1 \"".$FMT_FILE."\" \"".$PFMT_FILE."\" \"$OUTPUT_FOLDER/printfFile.txt\" \"$FMT_OUTPUT\"");     
        } else {
            $NOSEP=1;
            print "Activating storm.pl...\n";
            system("perl -w storm.pl \"$OUTPUT_FILE[0]\" > \"$OUTPUT_FOLDER/storm.txt\"");
            
            die "Either wrong smc path or the request memories file doesn't exists in the version you specified.\n" unless (-e $FMT_FILE);
            print "Activating formatMsg.awk...\n";
            system($GAWK." -f ".$formatMsg." -v warnings=$WARNINGS optionalUnionSelection=1 \"".$FMT_FILE."\" \"".$PFMT_FILE."\" \"$OUTPUT_FOLDER/storm.txt\" \"$FMT_OUTPUT\"");
        }
    } if ($FORMAT==$FORMATS{'FMT'}){
        die "Either wrong smc path or the request memories file doesn't exists in the version you specified.\n" unless (-e $FMT_FILE);        
        print "Activating formatMsg.awk...\n";
        for(my $i=0; $i<=$#OUTPUT_FILE; $i++) {
            system($GAWK." -f ".$formatMsg." -v warnings=$WARNINGS \"".$FMT_FILE."\" \"".$FMT_FILE."\" \"$OUTPUT_FILE[$i]\" $filemode \"$FMT_OUTPUT\"");
            $FMT_OUTPUT=" > ".$FMT_TEMPLATE.($i+2).".txt";
        }
    }
}

############ MAIN ###################
# read user arguments
checkArgs;
# preparing needed files and folder
prepare;
# Setting the profile and parsing accordingly
setProfile($PROFILE_FILE,$PROFILE);
# closing open files
clean;
# passing the relevant file to succeeding scripts/tools if piping wasn't used.
if ($PIPELINE eq "NOT") {
    parseFiles;
}

